1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2018, Intel Corporation. */
3 
4 /* Intel(R) Ethernet Connection E800 Series Linux Driver */
5 
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #include <generated/utsrelease.h>
9 #include "ice.h"
10 #include "ice_base.h"
11 #include "ice_lib.h"
12 #include "ice_fltr.h"
13 #include "ice_dcb_lib.h"
14 #include "ice_dcb_nl.h"
15 #include "ice_devlink.h"
16 /* Including ice_trace.h with CREATE_TRACE_POINTS defined will generate the
17  * ice tracepoint functions. This must be done exactly once across the
18  * ice driver.
19  */
20 #define CREATE_TRACE_POINTS
21 #include "ice_trace.h"
22 #include "ice_eswitch.h"
23 #include "ice_tc_lib.h"
24 #include "ice_vsi_vlan_ops.h"
25 #include <net/xdp_sock_drv.h>
26 
27 #define DRV_SUMMARY	"Intel(R) Ethernet Connection E800 Series Linux Driver"
28 static const char ice_driver_string[] = DRV_SUMMARY;
29 static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation.";
30 
31 /* DDP Package file located in firmware search paths (e.g. /lib/firmware/) */
32 #define ICE_DDP_PKG_PATH	"intel/ice/ddp/"
33 #define ICE_DDP_PKG_FILE	ICE_DDP_PKG_PATH "ice.pkg"
34 
35 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
36 MODULE_DESCRIPTION(DRV_SUMMARY);
37 MODULE_LICENSE("GPL v2");
38 MODULE_FIRMWARE(ICE_DDP_PKG_FILE);
39 
40 static int debug = -1;
41 module_param(debug, int, 0644);
42 #ifndef CONFIG_DYNAMIC_DEBUG
43 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)");
44 #else
45 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)");
46 #endif /* !CONFIG_DYNAMIC_DEBUG */
47 
48 DEFINE_STATIC_KEY_FALSE(ice_xdp_locking_key);
49 EXPORT_SYMBOL(ice_xdp_locking_key);
50 
51 /**
52  * ice_hw_to_dev - Get device pointer from the hardware structure
53  * @hw: pointer to the device HW structure
54  *
55  * Used to access the device pointer from compilation units which can't easily
56  * include the definition of struct ice_pf without leading to circular header
57  * dependencies.
58  */
59 struct device *ice_hw_to_dev(struct ice_hw *hw)
60 {
61 	struct ice_pf *pf = container_of(hw, struct ice_pf, hw);
62 
63 	return &pf->pdev->dev;
64 }
65 
66 static struct workqueue_struct *ice_wq;
67 static const struct net_device_ops ice_netdev_safe_mode_ops;
68 static const struct net_device_ops ice_netdev_ops;
69 
70 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type);
71 
72 static void ice_vsi_release_all(struct ice_pf *pf);
73 
74 static int ice_rebuild_channels(struct ice_pf *pf);
75 static void ice_remove_q_channels(struct ice_vsi *vsi, bool rem_adv_fltr);
76 
77 static int
78 ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch,
79 		     void *cb_priv, enum tc_setup_type type, void *type_data,
80 		     void *data,
81 		     void (*cleanup)(struct flow_block_cb *block_cb));
82 
83 bool netif_is_ice(struct net_device *dev)
84 {
85 	return dev && (dev->netdev_ops == &ice_netdev_ops);
86 }
87 
88 /**
89  * ice_get_tx_pending - returns number of Tx descriptors not processed
90  * @ring: the ring of descriptors
91  */
92 static u16 ice_get_tx_pending(struct ice_tx_ring *ring)
93 {
94 	u16 head, tail;
95 
96 	head = ring->next_to_clean;
97 	tail = ring->next_to_use;
98 
99 	if (head != tail)
100 		return (head < tail) ?
101 			tail - head : (tail + ring->count - head);
102 	return 0;
103 }
104 
105 /**
106  * ice_check_for_hang_subtask - check for and recover hung queues
107  * @pf: pointer to PF struct
108  */
109 static void ice_check_for_hang_subtask(struct ice_pf *pf)
110 {
111 	struct ice_vsi *vsi = NULL;
112 	struct ice_hw *hw;
113 	unsigned int i;
114 	int packets;
115 	u32 v;
116 
117 	ice_for_each_vsi(pf, v)
118 		if (pf->vsi[v] && pf->vsi[v]->type == ICE_VSI_PF) {
119 			vsi = pf->vsi[v];
120 			break;
121 		}
122 
123 	if (!vsi || test_bit(ICE_VSI_DOWN, vsi->state))
124 		return;
125 
126 	if (!(vsi->netdev && netif_carrier_ok(vsi->netdev)))
127 		return;
128 
129 	hw = &vsi->back->hw;
130 
131 	ice_for_each_txq(vsi, i) {
132 		struct ice_tx_ring *tx_ring = vsi->tx_rings[i];
133 		struct ice_ring_stats *ring_stats;
134 
135 		if (!tx_ring)
136 			continue;
137 		if (ice_ring_ch_enabled(tx_ring))
138 			continue;
139 
140 		ring_stats = tx_ring->ring_stats;
141 		if (!ring_stats)
142 			continue;
143 
144 		if (tx_ring->desc) {
145 			/* If packet counter has not changed the queue is
146 			 * likely stalled, so force an interrupt for this
147 			 * queue.
148 			 *
149 			 * prev_pkt would be negative if there was no
150 			 * pending work.
151 			 */
152 			packets = ring_stats->stats.pkts & INT_MAX;
153 			if (ring_stats->tx_stats.prev_pkt == packets) {
154 				/* Trigger sw interrupt to revive the queue */
155 				ice_trigger_sw_intr(hw, tx_ring->q_vector);
156 				continue;
157 			}
158 
159 			/* Memory barrier between read of packet count and call
160 			 * to ice_get_tx_pending()
161 			 */
162 			smp_rmb();
163 			ring_stats->tx_stats.prev_pkt =
164 			    ice_get_tx_pending(tx_ring) ? packets : -1;
165 		}
166 	}
167 }
168 
169 /**
170  * ice_init_mac_fltr - Set initial MAC filters
171  * @pf: board private structure
172  *
173  * Set initial set of MAC filters for PF VSI; configure filters for permanent
174  * address and broadcast address. If an error is encountered, netdevice will be
175  * unregistered.
176  */
177 static int ice_init_mac_fltr(struct ice_pf *pf)
178 {
179 	struct ice_vsi *vsi;
180 	u8 *perm_addr;
181 
182 	vsi = ice_get_main_vsi(pf);
183 	if (!vsi)
184 		return -EINVAL;
185 
186 	perm_addr = vsi->port_info->mac.perm_addr;
187 	return ice_fltr_add_mac_and_broadcast(vsi, perm_addr, ICE_FWD_TO_VSI);
188 }
189 
190 /**
191  * ice_add_mac_to_sync_list - creates list of MAC addresses to be synced
192  * @netdev: the net device on which the sync is happening
193  * @addr: MAC address to sync
194  *
195  * This is a callback function which is called by the in kernel device sync
196  * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only
197  * populates the tmp_sync_list, which is later used by ice_add_mac to add the
198  * MAC filters from the hardware.
199  */
200 static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr)
201 {
202 	struct ice_netdev_priv *np = netdev_priv(netdev);
203 	struct ice_vsi *vsi = np->vsi;
204 
205 	if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr,
206 				     ICE_FWD_TO_VSI))
207 		return -EINVAL;
208 
209 	return 0;
210 }
211 
212 /**
213  * ice_add_mac_to_unsync_list - creates list of MAC addresses to be unsynced
214  * @netdev: the net device on which the unsync is happening
215  * @addr: MAC address to unsync
216  *
217  * This is a callback function which is called by the in kernel device unsync
218  * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only
219  * populates the tmp_unsync_list, which is later used by ice_remove_mac to
220  * delete the MAC filters from the hardware.
221  */
222 static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr)
223 {
224 	struct ice_netdev_priv *np = netdev_priv(netdev);
225 	struct ice_vsi *vsi = np->vsi;
226 
227 	/* Under some circumstances, we might receive a request to delete our
228 	 * own device address from our uc list. Because we store the device
229 	 * address in the VSI's MAC filter list, we need to ignore such
230 	 * requests and not delete our device address from this list.
231 	 */
232 	if (ether_addr_equal(addr, netdev->dev_addr))
233 		return 0;
234 
235 	if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr,
236 				     ICE_FWD_TO_VSI))
237 		return -EINVAL;
238 
239 	return 0;
240 }
241 
242 /**
243  * ice_vsi_fltr_changed - check if filter state changed
244  * @vsi: VSI to be checked
245  *
246  * returns true if filter state has changed, false otherwise.
247  */
248 static bool ice_vsi_fltr_changed(struct ice_vsi *vsi)
249 {
250 	return test_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state) ||
251 	       test_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state);
252 }
253 
254 /**
255  * ice_set_promisc - Enable promiscuous mode for a given PF
256  * @vsi: the VSI being configured
257  * @promisc_m: mask of promiscuous config bits
258  *
259  */
260 static int ice_set_promisc(struct ice_vsi *vsi, u8 promisc_m)
261 {
262 	int status;
263 
264 	if (vsi->type != ICE_VSI_PF)
265 		return 0;
266 
267 	if (ice_vsi_has_non_zero_vlans(vsi)) {
268 		promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX);
269 		status = ice_fltr_set_vlan_vsi_promisc(&vsi->back->hw, vsi,
270 						       promisc_m);
271 	} else {
272 		status = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
273 						  promisc_m, 0);
274 	}
275 	if (status && status != -EEXIST)
276 		return status;
277 
278 	netdev_dbg(vsi->netdev, "set promisc filter bits for VSI %i: 0x%x\n",
279 		   vsi->vsi_num, promisc_m);
280 	return 0;
281 }
282 
283 /**
284  * ice_clear_promisc - Disable promiscuous mode for a given PF
285  * @vsi: the VSI being configured
286  * @promisc_m: mask of promiscuous config bits
287  *
288  */
289 static int ice_clear_promisc(struct ice_vsi *vsi, u8 promisc_m)
290 {
291 	int status;
292 
293 	if (vsi->type != ICE_VSI_PF)
294 		return 0;
295 
296 	if (ice_vsi_has_non_zero_vlans(vsi)) {
297 		promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX);
298 		status = ice_fltr_clear_vlan_vsi_promisc(&vsi->back->hw, vsi,
299 							 promisc_m);
300 	} else {
301 		status = ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
302 						    promisc_m, 0);
303 	}
304 
305 	netdev_dbg(vsi->netdev, "clear promisc filter bits for VSI %i: 0x%x\n",
306 		   vsi->vsi_num, promisc_m);
307 	return status;
308 }
309 
310 /**
311  * ice_vsi_sync_fltr - Update the VSI filter list to the HW
312  * @vsi: ptr to the VSI
313  *
314  * Push any outstanding VSI filter changes through the AdminQ.
315  */
316 static int ice_vsi_sync_fltr(struct ice_vsi *vsi)
317 {
318 	struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
319 	struct device *dev = ice_pf_to_dev(vsi->back);
320 	struct net_device *netdev = vsi->netdev;
321 	bool promisc_forced_on = false;
322 	struct ice_pf *pf = vsi->back;
323 	struct ice_hw *hw = &pf->hw;
324 	u32 changed_flags = 0;
325 	int err;
326 
327 	if (!vsi->netdev)
328 		return -EINVAL;
329 
330 	while (test_and_set_bit(ICE_CFG_BUSY, vsi->state))
331 		usleep_range(1000, 2000);
332 
333 	changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
334 	vsi->current_netdev_flags = vsi->netdev->flags;
335 
336 	INIT_LIST_HEAD(&vsi->tmp_sync_list);
337 	INIT_LIST_HEAD(&vsi->tmp_unsync_list);
338 
339 	if (ice_vsi_fltr_changed(vsi)) {
340 		clear_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state);
341 		clear_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state);
342 
343 		/* grab the netdev's addr_list_lock */
344 		netif_addr_lock_bh(netdev);
345 		__dev_uc_sync(netdev, ice_add_mac_to_sync_list,
346 			      ice_add_mac_to_unsync_list);
347 		__dev_mc_sync(netdev, ice_add_mac_to_sync_list,
348 			      ice_add_mac_to_unsync_list);
349 		/* our temp lists are populated. release lock */
350 		netif_addr_unlock_bh(netdev);
351 	}
352 
353 	/* Remove MAC addresses in the unsync list */
354 	err = ice_fltr_remove_mac_list(vsi, &vsi->tmp_unsync_list);
355 	ice_fltr_free_list(dev, &vsi->tmp_unsync_list);
356 	if (err) {
357 		netdev_err(netdev, "Failed to delete MAC filters\n");
358 		/* if we failed because of alloc failures, just bail */
359 		if (err == -ENOMEM)
360 			goto out;
361 	}
362 
363 	/* Add MAC addresses in the sync list */
364 	err = ice_fltr_add_mac_list(vsi, &vsi->tmp_sync_list);
365 	ice_fltr_free_list(dev, &vsi->tmp_sync_list);
366 	/* If filter is added successfully or already exists, do not go into
367 	 * 'if' condition and report it as error. Instead continue processing
368 	 * rest of the function.
369 	 */
370 	if (err && err != -EEXIST) {
371 		netdev_err(netdev, "Failed to add MAC filters\n");
372 		/* If there is no more space for new umac filters, VSI
373 		 * should go into promiscuous mode. There should be some
374 		 * space reserved for promiscuous filters.
375 		 */
376 		if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC &&
377 		    !test_and_set_bit(ICE_FLTR_OVERFLOW_PROMISC,
378 				      vsi->state)) {
379 			promisc_forced_on = true;
380 			netdev_warn(netdev, "Reached MAC filter limit, forcing promisc mode on VSI %d\n",
381 				    vsi->vsi_num);
382 		} else {
383 			goto out;
384 		}
385 	}
386 	err = 0;
387 	/* check for changes in promiscuous modes */
388 	if (changed_flags & IFF_ALLMULTI) {
389 		if (vsi->current_netdev_flags & IFF_ALLMULTI) {
390 			err = ice_set_promisc(vsi, ICE_MCAST_PROMISC_BITS);
391 			if (err) {
392 				vsi->current_netdev_flags &= ~IFF_ALLMULTI;
393 				goto out_promisc;
394 			}
395 		} else {
396 			/* !(vsi->current_netdev_flags & IFF_ALLMULTI) */
397 			err = ice_clear_promisc(vsi, ICE_MCAST_PROMISC_BITS);
398 			if (err) {
399 				vsi->current_netdev_flags |= IFF_ALLMULTI;
400 				goto out_promisc;
401 			}
402 		}
403 	}
404 
405 	if (((changed_flags & IFF_PROMISC) || promisc_forced_on) ||
406 	    test_bit(ICE_VSI_PROMISC_CHANGED, vsi->state)) {
407 		clear_bit(ICE_VSI_PROMISC_CHANGED, vsi->state);
408 		if (vsi->current_netdev_flags & IFF_PROMISC) {
409 			/* Apply Rx filter rule to get traffic from wire */
410 			if (!ice_is_dflt_vsi_in_use(vsi->port_info)) {
411 				err = ice_set_dflt_vsi(vsi);
412 				if (err && err != -EEXIST) {
413 					netdev_err(netdev, "Error %d setting default VSI %i Rx rule\n",
414 						   err, vsi->vsi_num);
415 					vsi->current_netdev_flags &=
416 						~IFF_PROMISC;
417 					goto out_promisc;
418 				}
419 				err = 0;
420 				vlan_ops->dis_rx_filtering(vsi);
421 
422 				/* promiscuous mode implies allmulticast so
423 				 * that VSIs that are in promiscuous mode are
424 				 * subscribed to multicast packets coming to
425 				 * the port
426 				 */
427 				err = ice_set_promisc(vsi,
428 						      ICE_MCAST_PROMISC_BITS);
429 				if (err)
430 					goto out_promisc;
431 			}
432 		} else {
433 			/* Clear Rx filter to remove traffic from wire */
434 			if (ice_is_vsi_dflt_vsi(vsi)) {
435 				err = ice_clear_dflt_vsi(vsi);
436 				if (err) {
437 					netdev_err(netdev, "Error %d clearing default VSI %i Rx rule\n",
438 						   err, vsi->vsi_num);
439 					vsi->current_netdev_flags |=
440 						IFF_PROMISC;
441 					goto out_promisc;
442 				}
443 				if (vsi->netdev->features &
444 				    NETIF_F_HW_VLAN_CTAG_FILTER)
445 					vlan_ops->ena_rx_filtering(vsi);
446 			}
447 
448 			/* disable allmulti here, but only if allmulti is not
449 			 * still enabled for the netdev
450 			 */
451 			if (!(vsi->current_netdev_flags & IFF_ALLMULTI)) {
452 				err = ice_clear_promisc(vsi,
453 							ICE_MCAST_PROMISC_BITS);
454 				if (err) {
455 					netdev_err(netdev, "Error %d clearing multicast promiscuous on VSI %i\n",
456 						   err, vsi->vsi_num);
457 				}
458 			}
459 		}
460 	}
461 	goto exit;
462 
463 out_promisc:
464 	set_bit(ICE_VSI_PROMISC_CHANGED, vsi->state);
465 	goto exit;
466 out:
467 	/* if something went wrong then set the changed flag so we try again */
468 	set_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state);
469 	set_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state);
470 exit:
471 	clear_bit(ICE_CFG_BUSY, vsi->state);
472 	return err;
473 }
474 
475 /**
476  * ice_sync_fltr_subtask - Sync the VSI filter list with HW
477  * @pf: board private structure
478  */
479 static void ice_sync_fltr_subtask(struct ice_pf *pf)
480 {
481 	int v;
482 
483 	if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags)))
484 		return;
485 
486 	clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
487 
488 	ice_for_each_vsi(pf, v)
489 		if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) &&
490 		    ice_vsi_sync_fltr(pf->vsi[v])) {
491 			/* come back and try again later */
492 			set_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
493 			break;
494 		}
495 }
496 
497 /**
498  * ice_pf_dis_all_vsi - Pause all VSIs on a PF
499  * @pf: the PF
500  * @locked: is the rtnl_lock already held
501  */
502 static void ice_pf_dis_all_vsi(struct ice_pf *pf, bool locked)
503 {
504 	int node;
505 	int v;
506 
507 	ice_for_each_vsi(pf, v)
508 		if (pf->vsi[v])
509 			ice_dis_vsi(pf->vsi[v], locked);
510 
511 	for (node = 0; node < ICE_MAX_PF_AGG_NODES; node++)
512 		pf->pf_agg_node[node].num_vsis = 0;
513 
514 	for (node = 0; node < ICE_MAX_VF_AGG_NODES; node++)
515 		pf->vf_agg_node[node].num_vsis = 0;
516 }
517 
518 /**
519  * ice_clear_sw_switch_recipes - clear switch recipes
520  * @pf: board private structure
521  *
522  * Mark switch recipes as not created in sw structures. There are cases where
523  * rules (especially advanced rules) need to be restored, either re-read from
524  * hardware or added again. For example after the reset. 'recp_created' flag
525  * prevents from doing that and need to be cleared upfront.
526  */
527 static void ice_clear_sw_switch_recipes(struct ice_pf *pf)
528 {
529 	struct ice_sw_recipe *recp;
530 	u8 i;
531 
532 	recp = pf->hw.switch_info->recp_list;
533 	for (i = 0; i < ICE_MAX_NUM_RECIPES; i++)
534 		recp[i].recp_created = false;
535 }
536 
537 /**
538  * ice_prepare_for_reset - prep for reset
539  * @pf: board private structure
540  * @reset_type: reset type requested
541  *
542  * Inform or close all dependent features in prep for reset.
543  */
544 static void
545 ice_prepare_for_reset(struct ice_pf *pf, enum ice_reset_req reset_type)
546 {
547 	struct ice_hw *hw = &pf->hw;
548 	struct ice_vsi *vsi;
549 	struct ice_vf *vf;
550 	unsigned int bkt;
551 
552 	dev_dbg(ice_pf_to_dev(pf), "reset_type=%d\n", reset_type);
553 
554 	/* already prepared for reset */
555 	if (test_bit(ICE_PREPARED_FOR_RESET, pf->state))
556 		return;
557 
558 	ice_unplug_aux_dev(pf);
559 
560 	/* Notify VFs of impending reset */
561 	if (ice_check_sq_alive(hw, &hw->mailboxq))
562 		ice_vc_notify_reset(pf);
563 
564 	/* Disable VFs until reset is completed */
565 	mutex_lock(&pf->vfs.table_lock);
566 	ice_for_each_vf(pf, bkt, vf)
567 		ice_set_vf_state_dis(vf);
568 	mutex_unlock(&pf->vfs.table_lock);
569 
570 	if (ice_is_eswitch_mode_switchdev(pf)) {
571 		if (reset_type != ICE_RESET_PFR)
572 			ice_clear_sw_switch_recipes(pf);
573 	}
574 
575 	/* release ADQ specific HW and SW resources */
576 	vsi = ice_get_main_vsi(pf);
577 	if (!vsi)
578 		goto skip;
579 
580 	/* to be on safe side, reset orig_rss_size so that normal flow
581 	 * of deciding rss_size can take precedence
582 	 */
583 	vsi->orig_rss_size = 0;
584 
585 	if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) {
586 		if (reset_type == ICE_RESET_PFR) {
587 			vsi->old_ena_tc = vsi->all_enatc;
588 			vsi->old_numtc = vsi->all_numtc;
589 		} else {
590 			ice_remove_q_channels(vsi, true);
591 
592 			/* for other reset type, do not support channel rebuild
593 			 * hence reset needed info
594 			 */
595 			vsi->old_ena_tc = 0;
596 			vsi->all_enatc = 0;
597 			vsi->old_numtc = 0;
598 			vsi->all_numtc = 0;
599 			vsi->req_txq = 0;
600 			vsi->req_rxq = 0;
601 			clear_bit(ICE_FLAG_TC_MQPRIO, pf->flags);
602 			memset(&vsi->mqprio_qopt, 0, sizeof(vsi->mqprio_qopt));
603 		}
604 	}
605 skip:
606 
607 	/* clear SW filtering DB */
608 	ice_clear_hw_tbls(hw);
609 	/* disable the VSIs and their queues that are not already DOWN */
610 	ice_pf_dis_all_vsi(pf, false);
611 
612 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
613 		ice_ptp_prepare_for_reset(pf);
614 
615 	if (ice_is_feature_supported(pf, ICE_F_GNSS))
616 		ice_gnss_exit(pf);
617 
618 	if (hw->port_info)
619 		ice_sched_clear_port(hw->port_info);
620 
621 	ice_shutdown_all_ctrlq(hw);
622 
623 	set_bit(ICE_PREPARED_FOR_RESET, pf->state);
624 }
625 
626 /**
627  * ice_do_reset - Initiate one of many types of resets
628  * @pf: board private structure
629  * @reset_type: reset type requested before this function was called.
630  */
631 static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type)
632 {
633 	struct device *dev = ice_pf_to_dev(pf);
634 	struct ice_hw *hw = &pf->hw;
635 
636 	dev_dbg(dev, "reset_type 0x%x requested\n", reset_type);
637 
638 	ice_prepare_for_reset(pf, reset_type);
639 
640 	/* trigger the reset */
641 	if (ice_reset(hw, reset_type)) {
642 		dev_err(dev, "reset %d failed\n", reset_type);
643 		set_bit(ICE_RESET_FAILED, pf->state);
644 		clear_bit(ICE_RESET_OICR_RECV, pf->state);
645 		clear_bit(ICE_PREPARED_FOR_RESET, pf->state);
646 		clear_bit(ICE_PFR_REQ, pf->state);
647 		clear_bit(ICE_CORER_REQ, pf->state);
648 		clear_bit(ICE_GLOBR_REQ, pf->state);
649 		wake_up(&pf->reset_wait_queue);
650 		return;
651 	}
652 
653 	/* PFR is a bit of a special case because it doesn't result in an OICR
654 	 * interrupt. So for PFR, rebuild after the reset and clear the reset-
655 	 * associated state bits.
656 	 */
657 	if (reset_type == ICE_RESET_PFR) {
658 		pf->pfr_count++;
659 		ice_rebuild(pf, reset_type);
660 		clear_bit(ICE_PREPARED_FOR_RESET, pf->state);
661 		clear_bit(ICE_PFR_REQ, pf->state);
662 		wake_up(&pf->reset_wait_queue);
663 		ice_reset_all_vfs(pf);
664 	}
665 }
666 
667 /**
668  * ice_reset_subtask - Set up for resetting the device and driver
669  * @pf: board private structure
670  */
671 static void ice_reset_subtask(struct ice_pf *pf)
672 {
673 	enum ice_reset_req reset_type = ICE_RESET_INVAL;
674 
675 	/* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an
676 	 * OICR interrupt. The OICR handler (ice_misc_intr) determines what type
677 	 * of reset is pending and sets bits in pf->state indicating the reset
678 	 * type and ICE_RESET_OICR_RECV. So, if the latter bit is set
679 	 * prepare for pending reset if not already (for PF software-initiated
680 	 * global resets the software should already be prepared for it as
681 	 * indicated by ICE_PREPARED_FOR_RESET; for global resets initiated
682 	 * by firmware or software on other PFs, that bit is not set so prepare
683 	 * for the reset now), poll for reset done, rebuild and return.
684 	 */
685 	if (test_bit(ICE_RESET_OICR_RECV, pf->state)) {
686 		/* Perform the largest reset requested */
687 		if (test_and_clear_bit(ICE_CORER_RECV, pf->state))
688 			reset_type = ICE_RESET_CORER;
689 		if (test_and_clear_bit(ICE_GLOBR_RECV, pf->state))
690 			reset_type = ICE_RESET_GLOBR;
691 		if (test_and_clear_bit(ICE_EMPR_RECV, pf->state))
692 			reset_type = ICE_RESET_EMPR;
693 		/* return if no valid reset type requested */
694 		if (reset_type == ICE_RESET_INVAL)
695 			return;
696 		ice_prepare_for_reset(pf, reset_type);
697 
698 		/* make sure we are ready to rebuild */
699 		if (ice_check_reset(&pf->hw)) {
700 			set_bit(ICE_RESET_FAILED, pf->state);
701 		} else {
702 			/* done with reset. start rebuild */
703 			pf->hw.reset_ongoing = false;
704 			ice_rebuild(pf, reset_type);
705 			/* clear bit to resume normal operations, but
706 			 * ICE_NEEDS_RESTART bit is set in case rebuild failed
707 			 */
708 			clear_bit(ICE_RESET_OICR_RECV, pf->state);
709 			clear_bit(ICE_PREPARED_FOR_RESET, pf->state);
710 			clear_bit(ICE_PFR_REQ, pf->state);
711 			clear_bit(ICE_CORER_REQ, pf->state);
712 			clear_bit(ICE_GLOBR_REQ, pf->state);
713 			wake_up(&pf->reset_wait_queue);
714 			ice_reset_all_vfs(pf);
715 		}
716 
717 		return;
718 	}
719 
720 	/* No pending resets to finish processing. Check for new resets */
721 	if (test_bit(ICE_PFR_REQ, pf->state))
722 		reset_type = ICE_RESET_PFR;
723 	if (test_bit(ICE_CORER_REQ, pf->state))
724 		reset_type = ICE_RESET_CORER;
725 	if (test_bit(ICE_GLOBR_REQ, pf->state))
726 		reset_type = ICE_RESET_GLOBR;
727 	/* If no valid reset type requested just return */
728 	if (reset_type == ICE_RESET_INVAL)
729 		return;
730 
731 	/* reset if not already down or busy */
732 	if (!test_bit(ICE_DOWN, pf->state) &&
733 	    !test_bit(ICE_CFG_BUSY, pf->state)) {
734 		ice_do_reset(pf, reset_type);
735 	}
736 }
737 
738 /**
739  * ice_print_topo_conflict - print topology conflict message
740  * @vsi: the VSI whose topology status is being checked
741  */
742 static void ice_print_topo_conflict(struct ice_vsi *vsi)
743 {
744 	switch (vsi->port_info->phy.link_info.topo_media_conflict) {
745 	case ICE_AQ_LINK_TOPO_CONFLICT:
746 	case ICE_AQ_LINK_MEDIA_CONFLICT:
747 	case ICE_AQ_LINK_TOPO_UNREACH_PRT:
748 	case ICE_AQ_LINK_TOPO_UNDRUTIL_PRT:
749 	case ICE_AQ_LINK_TOPO_UNDRUTIL_MEDIA:
750 		netdev_info(vsi->netdev, "Potential misconfiguration of the Ethernet port detected. If it was not intended, please use the Intel (R) Ethernet Port Configuration Tool to address the issue.\n");
751 		break;
752 	case ICE_AQ_LINK_TOPO_UNSUPP_MEDIA:
753 		if (test_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, vsi->back->flags))
754 			netdev_warn(vsi->netdev, "An unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules\n");
755 		else
756 			netdev_err(vsi->netdev, "Rx/Tx is disabled on this device because an unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
757 		break;
758 	default:
759 		break;
760 	}
761 }
762 
763 /**
764  * ice_print_link_msg - print link up or down message
765  * @vsi: the VSI whose link status is being queried
766  * @isup: boolean for if the link is now up or down
767  */
768 void ice_print_link_msg(struct ice_vsi *vsi, bool isup)
769 {
770 	struct ice_aqc_get_phy_caps_data *caps;
771 	const char *an_advertised;
772 	const char *fec_req;
773 	const char *speed;
774 	const char *fec;
775 	const char *fc;
776 	const char *an;
777 	int status;
778 
779 	if (!vsi)
780 		return;
781 
782 	if (vsi->current_isup == isup)
783 		return;
784 
785 	vsi->current_isup = isup;
786 
787 	if (!isup) {
788 		netdev_info(vsi->netdev, "NIC Link is Down\n");
789 		return;
790 	}
791 
792 	switch (vsi->port_info->phy.link_info.link_speed) {
793 	case ICE_AQ_LINK_SPEED_100GB:
794 		speed = "100 G";
795 		break;
796 	case ICE_AQ_LINK_SPEED_50GB:
797 		speed = "50 G";
798 		break;
799 	case ICE_AQ_LINK_SPEED_40GB:
800 		speed = "40 G";
801 		break;
802 	case ICE_AQ_LINK_SPEED_25GB:
803 		speed = "25 G";
804 		break;
805 	case ICE_AQ_LINK_SPEED_20GB:
806 		speed = "20 G";
807 		break;
808 	case ICE_AQ_LINK_SPEED_10GB:
809 		speed = "10 G";
810 		break;
811 	case ICE_AQ_LINK_SPEED_5GB:
812 		speed = "5 G";
813 		break;
814 	case ICE_AQ_LINK_SPEED_2500MB:
815 		speed = "2.5 G";
816 		break;
817 	case ICE_AQ_LINK_SPEED_1000MB:
818 		speed = "1 G";
819 		break;
820 	case ICE_AQ_LINK_SPEED_100MB:
821 		speed = "100 M";
822 		break;
823 	default:
824 		speed = "Unknown ";
825 		break;
826 	}
827 
828 	switch (vsi->port_info->fc.current_mode) {
829 	case ICE_FC_FULL:
830 		fc = "Rx/Tx";
831 		break;
832 	case ICE_FC_TX_PAUSE:
833 		fc = "Tx";
834 		break;
835 	case ICE_FC_RX_PAUSE:
836 		fc = "Rx";
837 		break;
838 	case ICE_FC_NONE:
839 		fc = "None";
840 		break;
841 	default:
842 		fc = "Unknown";
843 		break;
844 	}
845 
846 	/* Get FEC mode based on negotiated link info */
847 	switch (vsi->port_info->phy.link_info.fec_info) {
848 	case ICE_AQ_LINK_25G_RS_528_FEC_EN:
849 	case ICE_AQ_LINK_25G_RS_544_FEC_EN:
850 		fec = "RS-FEC";
851 		break;
852 	case ICE_AQ_LINK_25G_KR_FEC_EN:
853 		fec = "FC-FEC/BASE-R";
854 		break;
855 	default:
856 		fec = "NONE";
857 		break;
858 	}
859 
860 	/* check if autoneg completed, might be false due to not supported */
861 	if (vsi->port_info->phy.link_info.an_info & ICE_AQ_AN_COMPLETED)
862 		an = "True";
863 	else
864 		an = "False";
865 
866 	/* Get FEC mode requested based on PHY caps last SW configuration */
867 	caps = kzalloc(sizeof(*caps), GFP_KERNEL);
868 	if (!caps) {
869 		fec_req = "Unknown";
870 		an_advertised = "Unknown";
871 		goto done;
872 	}
873 
874 	status = ice_aq_get_phy_caps(vsi->port_info, false,
875 				     ICE_AQC_REPORT_ACTIVE_CFG, caps, NULL);
876 	if (status)
877 		netdev_info(vsi->netdev, "Get phy capability failed.\n");
878 
879 	an_advertised = ice_is_phy_caps_an_enabled(caps) ? "On" : "Off";
880 
881 	if (caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_528_REQ ||
882 	    caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_544_REQ)
883 		fec_req = "RS-FEC";
884 	else if (caps->link_fec_options & ICE_AQC_PHY_FEC_10G_KR_40G_KR4_REQ ||
885 		 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_KR_REQ)
886 		fec_req = "FC-FEC/BASE-R";
887 	else
888 		fec_req = "NONE";
889 
890 	kfree(caps);
891 
892 done:
893 	netdev_info(vsi->netdev, "NIC Link is up %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg Advertised: %s, Autoneg Negotiated: %s, Flow Control: %s\n",
894 		    speed, fec_req, fec, an_advertised, an, fc);
895 	ice_print_topo_conflict(vsi);
896 }
897 
898 /**
899  * ice_vsi_link_event - update the VSI's netdev
900  * @vsi: the VSI on which the link event occurred
901  * @link_up: whether or not the VSI needs to be set up or down
902  */
903 static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up)
904 {
905 	if (!vsi)
906 		return;
907 
908 	if (test_bit(ICE_VSI_DOWN, vsi->state) || !vsi->netdev)
909 		return;
910 
911 	if (vsi->type == ICE_VSI_PF) {
912 		if (link_up == netif_carrier_ok(vsi->netdev))
913 			return;
914 
915 		if (link_up) {
916 			netif_carrier_on(vsi->netdev);
917 			netif_tx_wake_all_queues(vsi->netdev);
918 		} else {
919 			netif_carrier_off(vsi->netdev);
920 			netif_tx_stop_all_queues(vsi->netdev);
921 		}
922 	}
923 }
924 
925 /**
926  * ice_set_dflt_mib - send a default config MIB to the FW
927  * @pf: private PF struct
928  *
929  * This function sends a default configuration MIB to the FW.
930  *
931  * If this function errors out at any point, the driver is still able to
932  * function.  The main impact is that LFC may not operate as expected.
933  * Therefore an error state in this function should be treated with a DBG
934  * message and continue on with driver rebuild/reenable.
935  */
936 static void ice_set_dflt_mib(struct ice_pf *pf)
937 {
938 	struct device *dev = ice_pf_to_dev(pf);
939 	u8 mib_type, *buf, *lldpmib = NULL;
940 	u16 len, typelen, offset = 0;
941 	struct ice_lldp_org_tlv *tlv;
942 	struct ice_hw *hw = &pf->hw;
943 	u32 ouisubtype;
944 
945 	mib_type = SET_LOCAL_MIB_TYPE_LOCAL_MIB;
946 	lldpmib = kzalloc(ICE_LLDPDU_SIZE, GFP_KERNEL);
947 	if (!lldpmib) {
948 		dev_dbg(dev, "%s Failed to allocate MIB memory\n",
949 			__func__);
950 		return;
951 	}
952 
953 	/* Add ETS CFG TLV */
954 	tlv = (struct ice_lldp_org_tlv *)lldpmib;
955 	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
956 		   ICE_IEEE_ETS_TLV_LEN);
957 	tlv->typelen = htons(typelen);
958 	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
959 		      ICE_IEEE_SUBTYPE_ETS_CFG);
960 	tlv->ouisubtype = htonl(ouisubtype);
961 
962 	buf = tlv->tlvinfo;
963 	buf[0] = 0;
964 
965 	/* ETS CFG all UPs map to TC 0. Next 4 (1 - 4) Octets = 0.
966 	 * Octets 5 - 12 are BW values, set octet 5 to 100% BW.
967 	 * Octets 13 - 20 are TSA values - leave as zeros
968 	 */
969 	buf[5] = 0x64;
970 	len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S;
971 	offset += len + 2;
972 	tlv = (struct ice_lldp_org_tlv *)
973 		((char *)tlv + sizeof(tlv->typelen) + len);
974 
975 	/* Add ETS REC TLV */
976 	buf = tlv->tlvinfo;
977 	tlv->typelen = htons(typelen);
978 
979 	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
980 		      ICE_IEEE_SUBTYPE_ETS_REC);
981 	tlv->ouisubtype = htonl(ouisubtype);
982 
983 	/* First octet of buf is reserved
984 	 * Octets 1 - 4 map UP to TC - all UPs map to zero
985 	 * Octets 5 - 12 are BW values - set TC 0 to 100%.
986 	 * Octets 13 - 20 are TSA value - leave as zeros
987 	 */
988 	buf[5] = 0x64;
989 	offset += len + 2;
990 	tlv = (struct ice_lldp_org_tlv *)
991 		((char *)tlv + sizeof(tlv->typelen) + len);
992 
993 	/* Add PFC CFG TLV */
994 	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
995 		   ICE_IEEE_PFC_TLV_LEN);
996 	tlv->typelen = htons(typelen);
997 
998 	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
999 		      ICE_IEEE_SUBTYPE_PFC_CFG);
1000 	tlv->ouisubtype = htonl(ouisubtype);
1001 
1002 	/* Octet 1 left as all zeros - PFC disabled */
1003 	buf[0] = 0x08;
1004 	len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S;
1005 	offset += len + 2;
1006 
1007 	if (ice_aq_set_lldp_mib(hw, mib_type, (void *)lldpmib, offset, NULL))
1008 		dev_dbg(dev, "%s Failed to set default LLDP MIB\n", __func__);
1009 
1010 	kfree(lldpmib);
1011 }
1012 
1013 /**
1014  * ice_check_phy_fw_load - check if PHY FW load failed
1015  * @pf: pointer to PF struct
1016  * @link_cfg_err: bitmap from the link info structure
1017  *
1018  * check if external PHY FW load failed and print an error message if it did
1019  */
1020 static void ice_check_phy_fw_load(struct ice_pf *pf, u8 link_cfg_err)
1021 {
1022 	if (!(link_cfg_err & ICE_AQ_LINK_EXTERNAL_PHY_LOAD_FAILURE)) {
1023 		clear_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags);
1024 		return;
1025 	}
1026 
1027 	if (test_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags))
1028 		return;
1029 
1030 	if (link_cfg_err & ICE_AQ_LINK_EXTERNAL_PHY_LOAD_FAILURE) {
1031 		dev_err(ice_pf_to_dev(pf), "Device failed to load the FW for the external PHY. Please download and install the latest NVM for your device and try again\n");
1032 		set_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags);
1033 	}
1034 }
1035 
1036 /**
1037  * ice_check_module_power
1038  * @pf: pointer to PF struct
1039  * @link_cfg_err: bitmap from the link info structure
1040  *
1041  * check module power level returned by a previous call to aq_get_link_info
1042  * and print error messages if module power level is not supported
1043  */
1044 static void ice_check_module_power(struct ice_pf *pf, u8 link_cfg_err)
1045 {
1046 	/* if module power level is supported, clear the flag */
1047 	if (!(link_cfg_err & (ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT |
1048 			      ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED))) {
1049 		clear_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags);
1050 		return;
1051 	}
1052 
1053 	/* if ICE_FLAG_MOD_POWER_UNSUPPORTED was previously set and the
1054 	 * above block didn't clear this bit, there's nothing to do
1055 	 */
1056 	if (test_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags))
1057 		return;
1058 
1059 	if (link_cfg_err & ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT) {
1060 		dev_err(ice_pf_to_dev(pf), "The installed module is incompatible with the device's NVM image. Cannot start link\n");
1061 		set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags);
1062 	} else if (link_cfg_err & ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED) {
1063 		dev_err(ice_pf_to_dev(pf), "The module's power requirements exceed the device's power supply. Cannot start link\n");
1064 		set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags);
1065 	}
1066 }
1067 
1068 /**
1069  * ice_check_link_cfg_err - check if link configuration failed
1070  * @pf: pointer to the PF struct
1071  * @link_cfg_err: bitmap from the link info structure
1072  *
1073  * print if any link configuration failure happens due to the value in the
1074  * link_cfg_err parameter in the link info structure
1075  */
1076 static void ice_check_link_cfg_err(struct ice_pf *pf, u8 link_cfg_err)
1077 {
1078 	ice_check_module_power(pf, link_cfg_err);
1079 	ice_check_phy_fw_load(pf, link_cfg_err);
1080 }
1081 
1082 /**
1083  * ice_link_event - process the link event
1084  * @pf: PF that the link event is associated with
1085  * @pi: port_info for the port that the link event is associated with
1086  * @link_up: true if the physical link is up and false if it is down
1087  * @link_speed: current link speed received from the link event
1088  *
1089  * Returns 0 on success and negative on failure
1090  */
1091 static int
1092 ice_link_event(struct ice_pf *pf, struct ice_port_info *pi, bool link_up,
1093 	       u16 link_speed)
1094 {
1095 	struct device *dev = ice_pf_to_dev(pf);
1096 	struct ice_phy_info *phy_info;
1097 	struct ice_vsi *vsi;
1098 	u16 old_link_speed;
1099 	bool old_link;
1100 	int status;
1101 
1102 	phy_info = &pi->phy;
1103 	phy_info->link_info_old = phy_info->link_info;
1104 
1105 	old_link = !!(phy_info->link_info_old.link_info & ICE_AQ_LINK_UP);
1106 	old_link_speed = phy_info->link_info_old.link_speed;
1107 
1108 	/* update the link info structures and re-enable link events,
1109 	 * don't bail on failure due to other book keeping needed
1110 	 */
1111 	status = ice_update_link_info(pi);
1112 	if (status)
1113 		dev_dbg(dev, "Failed to update link status on port %d, err %d aq_err %s\n",
1114 			pi->lport, status,
1115 			ice_aq_str(pi->hw->adminq.sq_last_status));
1116 
1117 	ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err);
1118 
1119 	/* Check if the link state is up after updating link info, and treat
1120 	 * this event as an UP event since the link is actually UP now.
1121 	 */
1122 	if (phy_info->link_info.link_info & ICE_AQ_LINK_UP)
1123 		link_up = true;
1124 
1125 	vsi = ice_get_main_vsi(pf);
1126 	if (!vsi || !vsi->port_info)
1127 		return -EINVAL;
1128 
1129 	/* turn off PHY if media was removed */
1130 	if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags) &&
1131 	    !(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) {
1132 		set_bit(ICE_FLAG_NO_MEDIA, pf->flags);
1133 		ice_set_link(vsi, false);
1134 	}
1135 
1136 	/* if the old link up/down and speed is the same as the new */
1137 	if (link_up == old_link && link_speed == old_link_speed)
1138 		return 0;
1139 
1140 	ice_ptp_link_change(pf, pf->hw.pf_id, link_up);
1141 
1142 	if (ice_is_dcb_active(pf)) {
1143 		if (test_bit(ICE_FLAG_DCB_ENA, pf->flags))
1144 			ice_dcb_rebuild(pf);
1145 	} else {
1146 		if (link_up)
1147 			ice_set_dflt_mib(pf);
1148 	}
1149 	ice_vsi_link_event(vsi, link_up);
1150 	ice_print_link_msg(vsi, link_up);
1151 
1152 	ice_vc_notify_link_state(pf);
1153 
1154 	return 0;
1155 }
1156 
1157 /**
1158  * ice_watchdog_subtask - periodic tasks not using event driven scheduling
1159  * @pf: board private structure
1160  */
1161 static void ice_watchdog_subtask(struct ice_pf *pf)
1162 {
1163 	int i;
1164 
1165 	/* if interface is down do nothing */
1166 	if (test_bit(ICE_DOWN, pf->state) ||
1167 	    test_bit(ICE_CFG_BUSY, pf->state))
1168 		return;
1169 
1170 	/* make sure we don't do these things too often */
1171 	if (time_before(jiffies,
1172 			pf->serv_tmr_prev + pf->serv_tmr_period))
1173 		return;
1174 
1175 	pf->serv_tmr_prev = jiffies;
1176 
1177 	/* Update the stats for active netdevs so the network stack
1178 	 * can look at updated numbers whenever it cares to
1179 	 */
1180 	ice_update_pf_stats(pf);
1181 	ice_for_each_vsi(pf, i)
1182 		if (pf->vsi[i] && pf->vsi[i]->netdev)
1183 			ice_update_vsi_stats(pf->vsi[i]);
1184 }
1185 
1186 /**
1187  * ice_init_link_events - enable/initialize link events
1188  * @pi: pointer to the port_info instance
1189  *
1190  * Returns -EIO on failure, 0 on success
1191  */
1192 static int ice_init_link_events(struct ice_port_info *pi)
1193 {
1194 	u16 mask;
1195 
1196 	mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA |
1197 		       ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL |
1198 		       ICE_AQ_LINK_EVENT_PHY_FW_LOAD_FAIL));
1199 
1200 	if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) {
1201 		dev_dbg(ice_hw_to_dev(pi->hw), "Failed to set link event mask for port %d\n",
1202 			pi->lport);
1203 		return -EIO;
1204 	}
1205 
1206 	if (ice_aq_get_link_info(pi, true, NULL, NULL)) {
1207 		dev_dbg(ice_hw_to_dev(pi->hw), "Failed to enable link events for port %d\n",
1208 			pi->lport);
1209 		return -EIO;
1210 	}
1211 
1212 	return 0;
1213 }
1214 
1215 /**
1216  * ice_handle_link_event - handle link event via ARQ
1217  * @pf: PF that the link event is associated with
1218  * @event: event structure containing link status info
1219  */
1220 static int
1221 ice_handle_link_event(struct ice_pf *pf, struct ice_rq_event_info *event)
1222 {
1223 	struct ice_aqc_get_link_status_data *link_data;
1224 	struct ice_port_info *port_info;
1225 	int status;
1226 
1227 	link_data = (struct ice_aqc_get_link_status_data *)event->msg_buf;
1228 	port_info = pf->hw.port_info;
1229 	if (!port_info)
1230 		return -EINVAL;
1231 
1232 	status = ice_link_event(pf, port_info,
1233 				!!(link_data->link_info & ICE_AQ_LINK_UP),
1234 				le16_to_cpu(link_data->link_speed));
1235 	if (status)
1236 		dev_dbg(ice_pf_to_dev(pf), "Could not process link event, error %d\n",
1237 			status);
1238 
1239 	return status;
1240 }
1241 
1242 enum ice_aq_task_state {
1243 	ICE_AQ_TASK_WAITING = 0,
1244 	ICE_AQ_TASK_COMPLETE,
1245 	ICE_AQ_TASK_CANCELED,
1246 };
1247 
1248 struct ice_aq_task {
1249 	struct hlist_node entry;
1250 
1251 	u16 opcode;
1252 	struct ice_rq_event_info *event;
1253 	enum ice_aq_task_state state;
1254 };
1255 
1256 /**
1257  * ice_aq_wait_for_event - Wait for an AdminQ event from firmware
1258  * @pf: pointer to the PF private structure
1259  * @opcode: the opcode to wait for
1260  * @timeout: how long to wait, in jiffies
1261  * @event: storage for the event info
1262  *
1263  * Waits for a specific AdminQ completion event on the ARQ for a given PF. The
1264  * current thread will be put to sleep until the specified event occurs or
1265  * until the given timeout is reached.
1266  *
1267  * To obtain only the descriptor contents, pass an event without an allocated
1268  * msg_buf. If the complete data buffer is desired, allocate the
1269  * event->msg_buf with enough space ahead of time.
1270  *
1271  * Returns: zero on success, or a negative error code on failure.
1272  */
1273 int ice_aq_wait_for_event(struct ice_pf *pf, u16 opcode, unsigned long timeout,
1274 			  struct ice_rq_event_info *event)
1275 {
1276 	struct device *dev = ice_pf_to_dev(pf);
1277 	struct ice_aq_task *task;
1278 	unsigned long start;
1279 	long ret;
1280 	int err;
1281 
1282 	task = kzalloc(sizeof(*task), GFP_KERNEL);
1283 	if (!task)
1284 		return -ENOMEM;
1285 
1286 	INIT_HLIST_NODE(&task->entry);
1287 	task->opcode = opcode;
1288 	task->event = event;
1289 	task->state = ICE_AQ_TASK_WAITING;
1290 
1291 	spin_lock_bh(&pf->aq_wait_lock);
1292 	hlist_add_head(&task->entry, &pf->aq_wait_list);
1293 	spin_unlock_bh(&pf->aq_wait_lock);
1294 
1295 	start = jiffies;
1296 
1297 	ret = wait_event_interruptible_timeout(pf->aq_wait_queue, task->state,
1298 					       timeout);
1299 	switch (task->state) {
1300 	case ICE_AQ_TASK_WAITING:
1301 		err = ret < 0 ? ret : -ETIMEDOUT;
1302 		break;
1303 	case ICE_AQ_TASK_CANCELED:
1304 		err = ret < 0 ? ret : -ECANCELED;
1305 		break;
1306 	case ICE_AQ_TASK_COMPLETE:
1307 		err = ret < 0 ? ret : 0;
1308 		break;
1309 	default:
1310 		WARN(1, "Unexpected AdminQ wait task state %u", task->state);
1311 		err = -EINVAL;
1312 		break;
1313 	}
1314 
1315 	dev_dbg(dev, "Waited %u msecs (max %u msecs) for firmware response to op 0x%04x\n",
1316 		jiffies_to_msecs(jiffies - start),
1317 		jiffies_to_msecs(timeout),
1318 		opcode);
1319 
1320 	spin_lock_bh(&pf->aq_wait_lock);
1321 	hlist_del(&task->entry);
1322 	spin_unlock_bh(&pf->aq_wait_lock);
1323 	kfree(task);
1324 
1325 	return err;
1326 }
1327 
1328 /**
1329  * ice_aq_check_events - Check if any thread is waiting for an AdminQ event
1330  * @pf: pointer to the PF private structure
1331  * @opcode: the opcode of the event
1332  * @event: the event to check
1333  *
1334  * Loops over the current list of pending threads waiting for an AdminQ event.
1335  * For each matching task, copy the contents of the event into the task
1336  * structure and wake up the thread.
1337  *
1338  * If multiple threads wait for the same opcode, they will all be woken up.
1339  *
1340  * Note that event->msg_buf will only be duplicated if the event has a buffer
1341  * with enough space already allocated. Otherwise, only the descriptor and
1342  * message length will be copied.
1343  *
1344  * Returns: true if an event was found, false otherwise
1345  */
1346 static void ice_aq_check_events(struct ice_pf *pf, u16 opcode,
1347 				struct ice_rq_event_info *event)
1348 {
1349 	struct ice_aq_task *task;
1350 	bool found = false;
1351 
1352 	spin_lock_bh(&pf->aq_wait_lock);
1353 	hlist_for_each_entry(task, &pf->aq_wait_list, entry) {
1354 		if (task->state || task->opcode != opcode)
1355 			continue;
1356 
1357 		memcpy(&task->event->desc, &event->desc, sizeof(event->desc));
1358 		task->event->msg_len = event->msg_len;
1359 
1360 		/* Only copy the data buffer if a destination was set */
1361 		if (task->event->msg_buf &&
1362 		    task->event->buf_len > event->buf_len) {
1363 			memcpy(task->event->msg_buf, event->msg_buf,
1364 			       event->buf_len);
1365 			task->event->buf_len = event->buf_len;
1366 		}
1367 
1368 		task->state = ICE_AQ_TASK_COMPLETE;
1369 		found = true;
1370 	}
1371 	spin_unlock_bh(&pf->aq_wait_lock);
1372 
1373 	if (found)
1374 		wake_up(&pf->aq_wait_queue);
1375 }
1376 
1377 /**
1378  * ice_aq_cancel_waiting_tasks - Immediately cancel all waiting tasks
1379  * @pf: the PF private structure
1380  *
1381  * Set all waiting tasks to ICE_AQ_TASK_CANCELED, and wake up their threads.
1382  * This will then cause ice_aq_wait_for_event to exit with -ECANCELED.
1383  */
1384 static void ice_aq_cancel_waiting_tasks(struct ice_pf *pf)
1385 {
1386 	struct ice_aq_task *task;
1387 
1388 	spin_lock_bh(&pf->aq_wait_lock);
1389 	hlist_for_each_entry(task, &pf->aq_wait_list, entry)
1390 		task->state = ICE_AQ_TASK_CANCELED;
1391 	spin_unlock_bh(&pf->aq_wait_lock);
1392 
1393 	wake_up(&pf->aq_wait_queue);
1394 }
1395 
1396 #define ICE_MBX_OVERFLOW_WATERMARK 64
1397 
1398 /**
1399  * __ice_clean_ctrlq - helper function to clean controlq rings
1400  * @pf: ptr to struct ice_pf
1401  * @q_type: specific Control queue type
1402  */
1403 static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type)
1404 {
1405 	struct device *dev = ice_pf_to_dev(pf);
1406 	struct ice_rq_event_info event;
1407 	struct ice_hw *hw = &pf->hw;
1408 	struct ice_ctl_q_info *cq;
1409 	u16 pending, i = 0;
1410 	const char *qtype;
1411 	u32 oldval, val;
1412 
1413 	/* Do not clean control queue if/when PF reset fails */
1414 	if (test_bit(ICE_RESET_FAILED, pf->state))
1415 		return 0;
1416 
1417 	switch (q_type) {
1418 	case ICE_CTL_Q_ADMIN:
1419 		cq = &hw->adminq;
1420 		qtype = "Admin";
1421 		break;
1422 	case ICE_CTL_Q_SB:
1423 		cq = &hw->sbq;
1424 		qtype = "Sideband";
1425 		break;
1426 	case ICE_CTL_Q_MAILBOX:
1427 		cq = &hw->mailboxq;
1428 		qtype = "Mailbox";
1429 		/* we are going to try to detect a malicious VF, so set the
1430 		 * state to begin detection
1431 		 */
1432 		hw->mbx_snapshot.mbx_buf.state = ICE_MAL_VF_DETECT_STATE_NEW_SNAPSHOT;
1433 		break;
1434 	default:
1435 		dev_warn(dev, "Unknown control queue type 0x%x\n", q_type);
1436 		return 0;
1437 	}
1438 
1439 	/* check for error indications - PF_xx_AxQLEN register layout for
1440 	 * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN.
1441 	 */
1442 	val = rd32(hw, cq->rq.len);
1443 	if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
1444 		   PF_FW_ARQLEN_ARQCRIT_M)) {
1445 		oldval = val;
1446 		if (val & PF_FW_ARQLEN_ARQVFE_M)
1447 			dev_dbg(dev, "%s Receive Queue VF Error detected\n",
1448 				qtype);
1449 		if (val & PF_FW_ARQLEN_ARQOVFL_M) {
1450 			dev_dbg(dev, "%s Receive Queue Overflow Error detected\n",
1451 				qtype);
1452 		}
1453 		if (val & PF_FW_ARQLEN_ARQCRIT_M)
1454 			dev_dbg(dev, "%s Receive Queue Critical Error detected\n",
1455 				qtype);
1456 		val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
1457 			 PF_FW_ARQLEN_ARQCRIT_M);
1458 		if (oldval != val)
1459 			wr32(hw, cq->rq.len, val);
1460 	}
1461 
1462 	val = rd32(hw, cq->sq.len);
1463 	if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
1464 		   PF_FW_ATQLEN_ATQCRIT_M)) {
1465 		oldval = val;
1466 		if (val & PF_FW_ATQLEN_ATQVFE_M)
1467 			dev_dbg(dev, "%s Send Queue VF Error detected\n",
1468 				qtype);
1469 		if (val & PF_FW_ATQLEN_ATQOVFL_M) {
1470 			dev_dbg(dev, "%s Send Queue Overflow Error detected\n",
1471 				qtype);
1472 		}
1473 		if (val & PF_FW_ATQLEN_ATQCRIT_M)
1474 			dev_dbg(dev, "%s Send Queue Critical Error detected\n",
1475 				qtype);
1476 		val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
1477 			 PF_FW_ATQLEN_ATQCRIT_M);
1478 		if (oldval != val)
1479 			wr32(hw, cq->sq.len, val);
1480 	}
1481 
1482 	event.buf_len = cq->rq_buf_size;
1483 	event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
1484 	if (!event.msg_buf)
1485 		return 0;
1486 
1487 	do {
1488 		struct ice_mbx_data data = {};
1489 		u16 opcode;
1490 		int ret;
1491 
1492 		ret = ice_clean_rq_elem(hw, cq, &event, &pending);
1493 		if (ret == -EALREADY)
1494 			break;
1495 		if (ret) {
1496 			dev_err(dev, "%s Receive Queue event error %d\n", qtype,
1497 				ret);
1498 			break;
1499 		}
1500 
1501 		opcode = le16_to_cpu(event.desc.opcode);
1502 
1503 		/* Notify any thread that might be waiting for this event */
1504 		ice_aq_check_events(pf, opcode, &event);
1505 
1506 		switch (opcode) {
1507 		case ice_aqc_opc_get_link_status:
1508 			if (ice_handle_link_event(pf, &event))
1509 				dev_err(dev, "Could not handle link event\n");
1510 			break;
1511 		case ice_aqc_opc_event_lan_overflow:
1512 			ice_vf_lan_overflow_event(pf, &event);
1513 			break;
1514 		case ice_mbx_opc_send_msg_to_pf:
1515 			data.num_msg_proc = i;
1516 			data.num_pending_arq = pending;
1517 			data.max_num_msgs_mbx = hw->mailboxq.num_rq_entries;
1518 			data.async_watermark_val = ICE_MBX_OVERFLOW_WATERMARK;
1519 
1520 			ice_vc_process_vf_msg(pf, &event, &data);
1521 			break;
1522 		case ice_aqc_opc_fw_logging:
1523 			ice_output_fw_log(hw, &event.desc, event.msg_buf);
1524 			break;
1525 		case ice_aqc_opc_lldp_set_mib_change:
1526 			ice_dcb_process_lldp_set_mib_change(pf, &event);
1527 			break;
1528 		default:
1529 			dev_dbg(dev, "%s Receive Queue unknown event 0x%04x ignored\n",
1530 				qtype, opcode);
1531 			break;
1532 		}
1533 	} while (pending && (i++ < ICE_DFLT_IRQ_WORK));
1534 
1535 	kfree(event.msg_buf);
1536 
1537 	return pending && (i == ICE_DFLT_IRQ_WORK);
1538 }
1539 
1540 /**
1541  * ice_ctrlq_pending - check if there is a difference between ntc and ntu
1542  * @hw: pointer to hardware info
1543  * @cq: control queue information
1544  *
1545  * returns true if there are pending messages in a queue, false if there aren't
1546  */
1547 static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq)
1548 {
1549 	u16 ntu;
1550 
1551 	ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1552 	return cq->rq.next_to_clean != ntu;
1553 }
1554 
1555 /**
1556  * ice_clean_adminq_subtask - clean the AdminQ rings
1557  * @pf: board private structure
1558  */
1559 static void ice_clean_adminq_subtask(struct ice_pf *pf)
1560 {
1561 	struct ice_hw *hw = &pf->hw;
1562 
1563 	if (!test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state))
1564 		return;
1565 
1566 	if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN))
1567 		return;
1568 
1569 	clear_bit(ICE_ADMINQ_EVENT_PENDING, pf->state);
1570 
1571 	/* There might be a situation where new messages arrive to a control
1572 	 * queue between processing the last message and clearing the
1573 	 * EVENT_PENDING bit. So before exiting, check queue head again (using
1574 	 * ice_ctrlq_pending) and process new messages if any.
1575 	 */
1576 	if (ice_ctrlq_pending(hw, &hw->adminq))
1577 		__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN);
1578 
1579 	ice_flush(hw);
1580 }
1581 
1582 /**
1583  * ice_clean_mailboxq_subtask - clean the MailboxQ rings
1584  * @pf: board private structure
1585  */
1586 static void ice_clean_mailboxq_subtask(struct ice_pf *pf)
1587 {
1588 	struct ice_hw *hw = &pf->hw;
1589 
1590 	if (!test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state))
1591 		return;
1592 
1593 	if (__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX))
1594 		return;
1595 
1596 	clear_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state);
1597 
1598 	if (ice_ctrlq_pending(hw, &hw->mailboxq))
1599 		__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX);
1600 
1601 	ice_flush(hw);
1602 }
1603 
1604 /**
1605  * ice_clean_sbq_subtask - clean the Sideband Queue rings
1606  * @pf: board private structure
1607  */
1608 static void ice_clean_sbq_subtask(struct ice_pf *pf)
1609 {
1610 	struct ice_hw *hw = &pf->hw;
1611 
1612 	/* Nothing to do here if sideband queue is not supported */
1613 	if (!ice_is_sbq_supported(hw)) {
1614 		clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state);
1615 		return;
1616 	}
1617 
1618 	if (!test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state))
1619 		return;
1620 
1621 	if (__ice_clean_ctrlq(pf, ICE_CTL_Q_SB))
1622 		return;
1623 
1624 	clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state);
1625 
1626 	if (ice_ctrlq_pending(hw, &hw->sbq))
1627 		__ice_clean_ctrlq(pf, ICE_CTL_Q_SB);
1628 
1629 	ice_flush(hw);
1630 }
1631 
1632 /**
1633  * ice_service_task_schedule - schedule the service task to wake up
1634  * @pf: board private structure
1635  *
1636  * If not already scheduled, this puts the task into the work queue.
1637  */
1638 void ice_service_task_schedule(struct ice_pf *pf)
1639 {
1640 	if (!test_bit(ICE_SERVICE_DIS, pf->state) &&
1641 	    !test_and_set_bit(ICE_SERVICE_SCHED, pf->state) &&
1642 	    !test_bit(ICE_NEEDS_RESTART, pf->state))
1643 		queue_work(ice_wq, &pf->serv_task);
1644 }
1645 
1646 /**
1647  * ice_service_task_complete - finish up the service task
1648  * @pf: board private structure
1649  */
1650 static void ice_service_task_complete(struct ice_pf *pf)
1651 {
1652 	WARN_ON(!test_bit(ICE_SERVICE_SCHED, pf->state));
1653 
1654 	/* force memory (pf->state) to sync before next service task */
1655 	smp_mb__before_atomic();
1656 	clear_bit(ICE_SERVICE_SCHED, pf->state);
1657 }
1658 
1659 /**
1660  * ice_service_task_stop - stop service task and cancel works
1661  * @pf: board private structure
1662  *
1663  * Return 0 if the ICE_SERVICE_DIS bit was not already set,
1664  * 1 otherwise.
1665  */
1666 static int ice_service_task_stop(struct ice_pf *pf)
1667 {
1668 	int ret;
1669 
1670 	ret = test_and_set_bit(ICE_SERVICE_DIS, pf->state);
1671 
1672 	if (pf->serv_tmr.function)
1673 		del_timer_sync(&pf->serv_tmr);
1674 	if (pf->serv_task.func)
1675 		cancel_work_sync(&pf->serv_task);
1676 
1677 	clear_bit(ICE_SERVICE_SCHED, pf->state);
1678 	return ret;
1679 }
1680 
1681 /**
1682  * ice_service_task_restart - restart service task and schedule works
1683  * @pf: board private structure
1684  *
1685  * This function is needed for suspend and resume works (e.g WoL scenario)
1686  */
1687 static void ice_service_task_restart(struct ice_pf *pf)
1688 {
1689 	clear_bit(ICE_SERVICE_DIS, pf->state);
1690 	ice_service_task_schedule(pf);
1691 }
1692 
1693 /**
1694  * ice_service_timer - timer callback to schedule service task
1695  * @t: pointer to timer_list
1696  */
1697 static void ice_service_timer(struct timer_list *t)
1698 {
1699 	struct ice_pf *pf = from_timer(pf, t, serv_tmr);
1700 
1701 	mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies));
1702 	ice_service_task_schedule(pf);
1703 }
1704 
1705 /**
1706  * ice_handle_mdd_event - handle malicious driver detect event
1707  * @pf: pointer to the PF structure
1708  *
1709  * Called from service task. OICR interrupt handler indicates MDD event.
1710  * VF MDD logging is guarded by net_ratelimit. Additional PF and VF log
1711  * messages are wrapped by netif_msg_[rx|tx]_err. Since VF Rx MDD events
1712  * disable the queue, the PF can be configured to reset the VF using ethtool
1713  * private flag mdd-auto-reset-vf.
1714  */
1715 static void ice_handle_mdd_event(struct ice_pf *pf)
1716 {
1717 	struct device *dev = ice_pf_to_dev(pf);
1718 	struct ice_hw *hw = &pf->hw;
1719 	struct ice_vf *vf;
1720 	unsigned int bkt;
1721 	u32 reg;
1722 
1723 	if (!test_and_clear_bit(ICE_MDD_EVENT_PENDING, pf->state)) {
1724 		/* Since the VF MDD event logging is rate limited, check if
1725 		 * there are pending MDD events.
1726 		 */
1727 		ice_print_vfs_mdd_events(pf);
1728 		return;
1729 	}
1730 
1731 	/* find what triggered an MDD event */
1732 	reg = rd32(hw, GL_MDET_TX_PQM);
1733 	if (reg & GL_MDET_TX_PQM_VALID_M) {
1734 		u8 pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >>
1735 				GL_MDET_TX_PQM_PF_NUM_S;
1736 		u16 vf_num = (reg & GL_MDET_TX_PQM_VF_NUM_M) >>
1737 				GL_MDET_TX_PQM_VF_NUM_S;
1738 		u8 event = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >>
1739 				GL_MDET_TX_PQM_MAL_TYPE_S;
1740 		u16 queue = ((reg & GL_MDET_TX_PQM_QNUM_M) >>
1741 				GL_MDET_TX_PQM_QNUM_S);
1742 
1743 		if (netif_msg_tx_err(pf))
1744 			dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n",
1745 				 event, queue, pf_num, vf_num);
1746 		wr32(hw, GL_MDET_TX_PQM, 0xffffffff);
1747 	}
1748 
1749 	reg = rd32(hw, GL_MDET_TX_TCLAN);
1750 	if (reg & GL_MDET_TX_TCLAN_VALID_M) {
1751 		u8 pf_num = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >>
1752 				GL_MDET_TX_TCLAN_PF_NUM_S;
1753 		u16 vf_num = (reg & GL_MDET_TX_TCLAN_VF_NUM_M) >>
1754 				GL_MDET_TX_TCLAN_VF_NUM_S;
1755 		u8 event = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >>
1756 				GL_MDET_TX_TCLAN_MAL_TYPE_S;
1757 		u16 queue = ((reg & GL_MDET_TX_TCLAN_QNUM_M) >>
1758 				GL_MDET_TX_TCLAN_QNUM_S);
1759 
1760 		if (netif_msg_tx_err(pf))
1761 			dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n",
1762 				 event, queue, pf_num, vf_num);
1763 		wr32(hw, GL_MDET_TX_TCLAN, 0xffffffff);
1764 	}
1765 
1766 	reg = rd32(hw, GL_MDET_RX);
1767 	if (reg & GL_MDET_RX_VALID_M) {
1768 		u8 pf_num = (reg & GL_MDET_RX_PF_NUM_M) >>
1769 				GL_MDET_RX_PF_NUM_S;
1770 		u16 vf_num = (reg & GL_MDET_RX_VF_NUM_M) >>
1771 				GL_MDET_RX_VF_NUM_S;
1772 		u8 event = (reg & GL_MDET_RX_MAL_TYPE_M) >>
1773 				GL_MDET_RX_MAL_TYPE_S;
1774 		u16 queue = ((reg & GL_MDET_RX_QNUM_M) >>
1775 				GL_MDET_RX_QNUM_S);
1776 
1777 		if (netif_msg_rx_err(pf))
1778 			dev_info(dev, "Malicious Driver Detection event %d on RX queue %d PF# %d VF# %d\n",
1779 				 event, queue, pf_num, vf_num);
1780 		wr32(hw, GL_MDET_RX, 0xffffffff);
1781 	}
1782 
1783 	/* check to see if this PF caused an MDD event */
1784 	reg = rd32(hw, PF_MDET_TX_PQM);
1785 	if (reg & PF_MDET_TX_PQM_VALID_M) {
1786 		wr32(hw, PF_MDET_TX_PQM, 0xFFFF);
1787 		if (netif_msg_tx_err(pf))
1788 			dev_info(dev, "Malicious Driver Detection event TX_PQM detected on PF\n");
1789 	}
1790 
1791 	reg = rd32(hw, PF_MDET_TX_TCLAN);
1792 	if (reg & PF_MDET_TX_TCLAN_VALID_M) {
1793 		wr32(hw, PF_MDET_TX_TCLAN, 0xFFFF);
1794 		if (netif_msg_tx_err(pf))
1795 			dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on PF\n");
1796 	}
1797 
1798 	reg = rd32(hw, PF_MDET_RX);
1799 	if (reg & PF_MDET_RX_VALID_M) {
1800 		wr32(hw, PF_MDET_RX, 0xFFFF);
1801 		if (netif_msg_rx_err(pf))
1802 			dev_info(dev, "Malicious Driver Detection event RX detected on PF\n");
1803 	}
1804 
1805 	/* Check to see if one of the VFs caused an MDD event, and then
1806 	 * increment counters and set print pending
1807 	 */
1808 	mutex_lock(&pf->vfs.table_lock);
1809 	ice_for_each_vf(pf, bkt, vf) {
1810 		reg = rd32(hw, VP_MDET_TX_PQM(vf->vf_id));
1811 		if (reg & VP_MDET_TX_PQM_VALID_M) {
1812 			wr32(hw, VP_MDET_TX_PQM(vf->vf_id), 0xFFFF);
1813 			vf->mdd_tx_events.count++;
1814 			set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state);
1815 			if (netif_msg_tx_err(pf))
1816 				dev_info(dev, "Malicious Driver Detection event TX_PQM detected on VF %d\n",
1817 					 vf->vf_id);
1818 		}
1819 
1820 		reg = rd32(hw, VP_MDET_TX_TCLAN(vf->vf_id));
1821 		if (reg & VP_MDET_TX_TCLAN_VALID_M) {
1822 			wr32(hw, VP_MDET_TX_TCLAN(vf->vf_id), 0xFFFF);
1823 			vf->mdd_tx_events.count++;
1824 			set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state);
1825 			if (netif_msg_tx_err(pf))
1826 				dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on VF %d\n",
1827 					 vf->vf_id);
1828 		}
1829 
1830 		reg = rd32(hw, VP_MDET_TX_TDPU(vf->vf_id));
1831 		if (reg & VP_MDET_TX_TDPU_VALID_M) {
1832 			wr32(hw, VP_MDET_TX_TDPU(vf->vf_id), 0xFFFF);
1833 			vf->mdd_tx_events.count++;
1834 			set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state);
1835 			if (netif_msg_tx_err(pf))
1836 				dev_info(dev, "Malicious Driver Detection event TX_TDPU detected on VF %d\n",
1837 					 vf->vf_id);
1838 		}
1839 
1840 		reg = rd32(hw, VP_MDET_RX(vf->vf_id));
1841 		if (reg & VP_MDET_RX_VALID_M) {
1842 			wr32(hw, VP_MDET_RX(vf->vf_id), 0xFFFF);
1843 			vf->mdd_rx_events.count++;
1844 			set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state);
1845 			if (netif_msg_rx_err(pf))
1846 				dev_info(dev, "Malicious Driver Detection event RX detected on VF %d\n",
1847 					 vf->vf_id);
1848 
1849 			/* Since the queue is disabled on VF Rx MDD events, the
1850 			 * PF can be configured to reset the VF through ethtool
1851 			 * private flag mdd-auto-reset-vf.
1852 			 */
1853 			if (test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags)) {
1854 				/* VF MDD event counters will be cleared by
1855 				 * reset, so print the event prior to reset.
1856 				 */
1857 				ice_print_vf_rx_mdd_event(vf);
1858 				ice_reset_vf(vf, ICE_VF_RESET_LOCK);
1859 			}
1860 		}
1861 	}
1862 	mutex_unlock(&pf->vfs.table_lock);
1863 
1864 	ice_print_vfs_mdd_events(pf);
1865 }
1866 
1867 /**
1868  * ice_force_phys_link_state - Force the physical link state
1869  * @vsi: VSI to force the physical link state to up/down
1870  * @link_up: true/false indicates to set the physical link to up/down
1871  *
1872  * Force the physical link state by getting the current PHY capabilities from
1873  * hardware and setting the PHY config based on the determined capabilities. If
1874  * link changes a link event will be triggered because both the Enable Automatic
1875  * Link Update and LESM Enable bits are set when setting the PHY capabilities.
1876  *
1877  * Returns 0 on success, negative on failure
1878  */
1879 static int ice_force_phys_link_state(struct ice_vsi *vsi, bool link_up)
1880 {
1881 	struct ice_aqc_get_phy_caps_data *pcaps;
1882 	struct ice_aqc_set_phy_cfg_data *cfg;
1883 	struct ice_port_info *pi;
1884 	struct device *dev;
1885 	int retcode;
1886 
1887 	if (!vsi || !vsi->port_info || !vsi->back)
1888 		return -EINVAL;
1889 	if (vsi->type != ICE_VSI_PF)
1890 		return 0;
1891 
1892 	dev = ice_pf_to_dev(vsi->back);
1893 
1894 	pi = vsi->port_info;
1895 
1896 	pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL);
1897 	if (!pcaps)
1898 		return -ENOMEM;
1899 
1900 	retcode = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps,
1901 				      NULL);
1902 	if (retcode) {
1903 		dev_err(dev, "Failed to get phy capabilities, VSI %d error %d\n",
1904 			vsi->vsi_num, retcode);
1905 		retcode = -EIO;
1906 		goto out;
1907 	}
1908 
1909 	/* No change in link */
1910 	if (link_up == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) &&
1911 	    link_up == !!(pi->phy.link_info.link_info & ICE_AQ_LINK_UP))
1912 		goto out;
1913 
1914 	/* Use the current user PHY configuration. The current user PHY
1915 	 * configuration is initialized during probe from PHY capabilities
1916 	 * software mode, and updated on set PHY configuration.
1917 	 */
1918 	cfg = kmemdup(&pi->phy.curr_user_phy_cfg, sizeof(*cfg), GFP_KERNEL);
1919 	if (!cfg) {
1920 		retcode = -ENOMEM;
1921 		goto out;
1922 	}
1923 
1924 	cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT;
1925 	if (link_up)
1926 		cfg->caps |= ICE_AQ_PHY_ENA_LINK;
1927 	else
1928 		cfg->caps &= ~ICE_AQ_PHY_ENA_LINK;
1929 
1930 	retcode = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL);
1931 	if (retcode) {
1932 		dev_err(dev, "Failed to set phy config, VSI %d error %d\n",
1933 			vsi->vsi_num, retcode);
1934 		retcode = -EIO;
1935 	}
1936 
1937 	kfree(cfg);
1938 out:
1939 	kfree(pcaps);
1940 	return retcode;
1941 }
1942 
1943 /**
1944  * ice_init_nvm_phy_type - Initialize the NVM PHY type
1945  * @pi: port info structure
1946  *
1947  * Initialize nvm_phy_type_[low|high] for link lenient mode support
1948  */
1949 static int ice_init_nvm_phy_type(struct ice_port_info *pi)
1950 {
1951 	struct ice_aqc_get_phy_caps_data *pcaps;
1952 	struct ice_pf *pf = pi->hw->back;
1953 	int err;
1954 
1955 	pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL);
1956 	if (!pcaps)
1957 		return -ENOMEM;
1958 
1959 	err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA,
1960 				  pcaps, NULL);
1961 
1962 	if (err) {
1963 		dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n");
1964 		goto out;
1965 	}
1966 
1967 	pf->nvm_phy_type_hi = pcaps->phy_type_high;
1968 	pf->nvm_phy_type_lo = pcaps->phy_type_low;
1969 
1970 out:
1971 	kfree(pcaps);
1972 	return err;
1973 }
1974 
1975 /**
1976  * ice_init_link_dflt_override - Initialize link default override
1977  * @pi: port info structure
1978  *
1979  * Initialize link default override and PHY total port shutdown during probe
1980  */
1981 static void ice_init_link_dflt_override(struct ice_port_info *pi)
1982 {
1983 	struct ice_link_default_override_tlv *ldo;
1984 	struct ice_pf *pf = pi->hw->back;
1985 
1986 	ldo = &pf->link_dflt_override;
1987 	if (ice_get_link_default_override(ldo, pi))
1988 		return;
1989 
1990 	if (!(ldo->options & ICE_LINK_OVERRIDE_PORT_DIS))
1991 		return;
1992 
1993 	/* Enable Total Port Shutdown (override/replace link-down-on-close
1994 	 * ethtool private flag) for ports with Port Disable bit set.
1995 	 */
1996 	set_bit(ICE_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags);
1997 	set_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags);
1998 }
1999 
2000 /**
2001  * ice_init_phy_cfg_dflt_override - Initialize PHY cfg default override settings
2002  * @pi: port info structure
2003  *
2004  * If default override is enabled, initialize the user PHY cfg speed and FEC
2005  * settings using the default override mask from the NVM.
2006  *
2007  * The PHY should only be configured with the default override settings the
2008  * first time media is available. The ICE_LINK_DEFAULT_OVERRIDE_PENDING state
2009  * is used to indicate that the user PHY cfg default override is initialized
2010  * and the PHY has not been configured with the default override settings. The
2011  * state is set here, and cleared in ice_configure_phy the first time the PHY is
2012  * configured.
2013  *
2014  * This function should be called only if the FW doesn't support default
2015  * configuration mode, as reported by ice_fw_supports_report_dflt_cfg.
2016  */
2017 static void ice_init_phy_cfg_dflt_override(struct ice_port_info *pi)
2018 {
2019 	struct ice_link_default_override_tlv *ldo;
2020 	struct ice_aqc_set_phy_cfg_data *cfg;
2021 	struct ice_phy_info *phy = &pi->phy;
2022 	struct ice_pf *pf = pi->hw->back;
2023 
2024 	ldo = &pf->link_dflt_override;
2025 
2026 	/* If link default override is enabled, use to mask NVM PHY capabilities
2027 	 * for speed and FEC default configuration.
2028 	 */
2029 	cfg = &phy->curr_user_phy_cfg;
2030 
2031 	if (ldo->phy_type_low || ldo->phy_type_high) {
2032 		cfg->phy_type_low = pf->nvm_phy_type_lo &
2033 				    cpu_to_le64(ldo->phy_type_low);
2034 		cfg->phy_type_high = pf->nvm_phy_type_hi &
2035 				     cpu_to_le64(ldo->phy_type_high);
2036 	}
2037 	cfg->link_fec_opt = ldo->fec_options;
2038 	phy->curr_user_fec_req = ICE_FEC_AUTO;
2039 
2040 	set_bit(ICE_LINK_DEFAULT_OVERRIDE_PENDING, pf->state);
2041 }
2042 
2043 /**
2044  * ice_init_phy_user_cfg - Initialize the PHY user configuration
2045  * @pi: port info structure
2046  *
2047  * Initialize the current user PHY configuration, speed, FEC, and FC requested
2048  * mode to default. The PHY defaults are from get PHY capabilities topology
2049  * with media so call when media is first available. An error is returned if
2050  * called when media is not available. The PHY initialization completed state is
2051  * set here.
2052  *
2053  * These configurations are used when setting PHY
2054  * configuration. The user PHY configuration is updated on set PHY
2055  * configuration. Returns 0 on success, negative on failure
2056  */
2057 static int ice_init_phy_user_cfg(struct ice_port_info *pi)
2058 {
2059 	struct ice_aqc_get_phy_caps_data *pcaps;
2060 	struct ice_phy_info *phy = &pi->phy;
2061 	struct ice_pf *pf = pi->hw->back;
2062 	int err;
2063 
2064 	if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE))
2065 		return -EIO;
2066 
2067 	pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL);
2068 	if (!pcaps)
2069 		return -ENOMEM;
2070 
2071 	if (ice_fw_supports_report_dflt_cfg(pi->hw))
2072 		err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_DFLT_CFG,
2073 					  pcaps, NULL);
2074 	else
2075 		err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_MEDIA,
2076 					  pcaps, NULL);
2077 	if (err) {
2078 		dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n");
2079 		goto err_out;
2080 	}
2081 
2082 	ice_copy_phy_caps_to_cfg(pi, pcaps, &pi->phy.curr_user_phy_cfg);
2083 
2084 	/* check if lenient mode is supported and enabled */
2085 	if (ice_fw_supports_link_override(pi->hw) &&
2086 	    !(pcaps->module_compliance_enforcement &
2087 	      ICE_AQC_MOD_ENFORCE_STRICT_MODE)) {
2088 		set_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags);
2089 
2090 		/* if the FW supports default PHY configuration mode, then the driver
2091 		 * does not have to apply link override settings. If not,
2092 		 * initialize user PHY configuration with link override values
2093 		 */
2094 		if (!ice_fw_supports_report_dflt_cfg(pi->hw) &&
2095 		    (pf->link_dflt_override.options & ICE_LINK_OVERRIDE_EN)) {
2096 			ice_init_phy_cfg_dflt_override(pi);
2097 			goto out;
2098 		}
2099 	}
2100 
2101 	/* if link default override is not enabled, set user flow control and
2102 	 * FEC settings based on what get_phy_caps returned
2103 	 */
2104 	phy->curr_user_fec_req = ice_caps_to_fec_mode(pcaps->caps,
2105 						      pcaps->link_fec_options);
2106 	phy->curr_user_fc_req = ice_caps_to_fc_mode(pcaps->caps);
2107 
2108 out:
2109 	phy->curr_user_speed_req = ICE_AQ_LINK_SPEED_M;
2110 	set_bit(ICE_PHY_INIT_COMPLETE, pf->state);
2111 err_out:
2112 	kfree(pcaps);
2113 	return err;
2114 }
2115 
2116 /**
2117  * ice_configure_phy - configure PHY
2118  * @vsi: VSI of PHY
2119  *
2120  * Set the PHY configuration. If the current PHY configuration is the same as
2121  * the curr_user_phy_cfg, then do nothing to avoid link flap. Otherwise
2122  * configure the based get PHY capabilities for topology with media.
2123  */
2124 static int ice_configure_phy(struct ice_vsi *vsi)
2125 {
2126 	struct device *dev = ice_pf_to_dev(vsi->back);
2127 	struct ice_port_info *pi = vsi->port_info;
2128 	struct ice_aqc_get_phy_caps_data *pcaps;
2129 	struct ice_aqc_set_phy_cfg_data *cfg;
2130 	struct ice_phy_info *phy = &pi->phy;
2131 	struct ice_pf *pf = vsi->back;
2132 	int err;
2133 
2134 	/* Ensure we have media as we cannot configure a medialess port */
2135 	if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE))
2136 		return -EPERM;
2137 
2138 	ice_print_topo_conflict(vsi);
2139 
2140 	if (!test_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags) &&
2141 	    phy->link_info.topo_media_conflict == ICE_AQ_LINK_TOPO_UNSUPP_MEDIA)
2142 		return -EPERM;
2143 
2144 	if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))
2145 		return ice_force_phys_link_state(vsi, true);
2146 
2147 	pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL);
2148 	if (!pcaps)
2149 		return -ENOMEM;
2150 
2151 	/* Get current PHY config */
2152 	err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps,
2153 				  NULL);
2154 	if (err) {
2155 		dev_err(dev, "Failed to get PHY configuration, VSI %d error %d\n",
2156 			vsi->vsi_num, err);
2157 		goto done;
2158 	}
2159 
2160 	/* If PHY enable link is configured and configuration has not changed,
2161 	 * there's nothing to do
2162 	 */
2163 	if (pcaps->caps & ICE_AQC_PHY_EN_LINK &&
2164 	    ice_phy_caps_equals_cfg(pcaps, &phy->curr_user_phy_cfg))
2165 		goto done;
2166 
2167 	/* Use PHY topology as baseline for configuration */
2168 	memset(pcaps, 0, sizeof(*pcaps));
2169 	if (ice_fw_supports_report_dflt_cfg(pi->hw))
2170 		err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_DFLT_CFG,
2171 					  pcaps, NULL);
2172 	else
2173 		err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_MEDIA,
2174 					  pcaps, NULL);
2175 	if (err) {
2176 		dev_err(dev, "Failed to get PHY caps, VSI %d error %d\n",
2177 			vsi->vsi_num, err);
2178 		goto done;
2179 	}
2180 
2181 	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
2182 	if (!cfg) {
2183 		err = -ENOMEM;
2184 		goto done;
2185 	}
2186 
2187 	ice_copy_phy_caps_to_cfg(pi, pcaps, cfg);
2188 
2189 	/* Speed - If default override pending, use curr_user_phy_cfg set in
2190 	 * ice_init_phy_user_cfg_ldo.
2191 	 */
2192 	if (test_and_clear_bit(ICE_LINK_DEFAULT_OVERRIDE_PENDING,
2193 			       vsi->back->state)) {
2194 		cfg->phy_type_low = phy->curr_user_phy_cfg.phy_type_low;
2195 		cfg->phy_type_high = phy->curr_user_phy_cfg.phy_type_high;
2196 	} else {
2197 		u64 phy_low = 0, phy_high = 0;
2198 
2199 		ice_update_phy_type(&phy_low, &phy_high,
2200 				    pi->phy.curr_user_speed_req);
2201 		cfg->phy_type_low = pcaps->phy_type_low & cpu_to_le64(phy_low);
2202 		cfg->phy_type_high = pcaps->phy_type_high &
2203 				     cpu_to_le64(phy_high);
2204 	}
2205 
2206 	/* Can't provide what was requested; use PHY capabilities */
2207 	if (!cfg->phy_type_low && !cfg->phy_type_high) {
2208 		cfg->phy_type_low = pcaps->phy_type_low;
2209 		cfg->phy_type_high = pcaps->phy_type_high;
2210 	}
2211 
2212 	/* FEC */
2213 	ice_cfg_phy_fec(pi, cfg, phy->curr_user_fec_req);
2214 
2215 	/* Can't provide what was requested; use PHY capabilities */
2216 	if (cfg->link_fec_opt !=
2217 	    (cfg->link_fec_opt & pcaps->link_fec_options)) {
2218 		cfg->caps |= pcaps->caps & ICE_AQC_PHY_EN_AUTO_FEC;
2219 		cfg->link_fec_opt = pcaps->link_fec_options;
2220 	}
2221 
2222 	/* Flow Control - always supported; no need to check against
2223 	 * capabilities
2224 	 */
2225 	ice_cfg_phy_fc(pi, cfg, phy->curr_user_fc_req);
2226 
2227 	/* Enable link and link update */
2228 	cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT | ICE_AQ_PHY_ENA_LINK;
2229 
2230 	err = ice_aq_set_phy_cfg(&pf->hw, pi, cfg, NULL);
2231 	if (err)
2232 		dev_err(dev, "Failed to set phy config, VSI %d error %d\n",
2233 			vsi->vsi_num, err);
2234 
2235 	kfree(cfg);
2236 done:
2237 	kfree(pcaps);
2238 	return err;
2239 }
2240 
2241 /**
2242  * ice_check_media_subtask - Check for media
2243  * @pf: pointer to PF struct
2244  *
2245  * If media is available, then initialize PHY user configuration if it is not
2246  * been, and configure the PHY if the interface is up.
2247  */
2248 static void ice_check_media_subtask(struct ice_pf *pf)
2249 {
2250 	struct ice_port_info *pi;
2251 	struct ice_vsi *vsi;
2252 	int err;
2253 
2254 	/* No need to check for media if it's already present */
2255 	if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags))
2256 		return;
2257 
2258 	vsi = ice_get_main_vsi(pf);
2259 	if (!vsi)
2260 		return;
2261 
2262 	/* Refresh link info and check if media is present */
2263 	pi = vsi->port_info;
2264 	err = ice_update_link_info(pi);
2265 	if (err)
2266 		return;
2267 
2268 	ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err);
2269 
2270 	if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
2271 		if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state))
2272 			ice_init_phy_user_cfg(pi);
2273 
2274 		/* PHY settings are reset on media insertion, reconfigure
2275 		 * PHY to preserve settings.
2276 		 */
2277 		if (test_bit(ICE_VSI_DOWN, vsi->state) &&
2278 		    test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags))
2279 			return;
2280 
2281 		err = ice_configure_phy(vsi);
2282 		if (!err)
2283 			clear_bit(ICE_FLAG_NO_MEDIA, pf->flags);
2284 
2285 		/* A Link Status Event will be generated; the event handler
2286 		 * will complete bringing the interface up
2287 		 */
2288 	}
2289 }
2290 
2291 /**
2292  * ice_service_task - manage and run subtasks
2293  * @work: pointer to work_struct contained by the PF struct
2294  */
2295 static void ice_service_task(struct work_struct *work)
2296 {
2297 	struct ice_pf *pf = container_of(work, struct ice_pf, serv_task);
2298 	unsigned long start_time = jiffies;
2299 
2300 	/* subtasks */
2301 
2302 	/* process reset requests first */
2303 	ice_reset_subtask(pf);
2304 
2305 	/* bail if a reset/recovery cycle is pending or rebuild failed */
2306 	if (ice_is_reset_in_progress(pf->state) ||
2307 	    test_bit(ICE_SUSPENDED, pf->state) ||
2308 	    test_bit(ICE_NEEDS_RESTART, pf->state)) {
2309 		ice_service_task_complete(pf);
2310 		return;
2311 	}
2312 
2313 	if (test_and_clear_bit(ICE_AUX_ERR_PENDING, pf->state)) {
2314 		struct iidc_event *event;
2315 
2316 		event = kzalloc(sizeof(*event), GFP_KERNEL);
2317 		if (event) {
2318 			set_bit(IIDC_EVENT_CRIT_ERR, event->type);
2319 			/* report the entire OICR value to AUX driver */
2320 			swap(event->reg, pf->oicr_err_reg);
2321 			ice_send_event_to_aux(pf, event);
2322 			kfree(event);
2323 		}
2324 	}
2325 
2326 	/* unplug aux dev per request, if an unplug request came in
2327 	 * while processing a plug request, this will handle it
2328 	 */
2329 	if (test_and_clear_bit(ICE_FLAG_UNPLUG_AUX_DEV, pf->flags))
2330 		ice_unplug_aux_dev(pf);
2331 
2332 	/* Plug aux device per request */
2333 	if (test_and_clear_bit(ICE_FLAG_PLUG_AUX_DEV, pf->flags))
2334 		ice_plug_aux_dev(pf);
2335 
2336 	if (test_and_clear_bit(ICE_FLAG_MTU_CHANGED, pf->flags)) {
2337 		struct iidc_event *event;
2338 
2339 		event = kzalloc(sizeof(*event), GFP_KERNEL);
2340 		if (event) {
2341 			set_bit(IIDC_EVENT_AFTER_MTU_CHANGE, event->type);
2342 			ice_send_event_to_aux(pf, event);
2343 			kfree(event);
2344 		}
2345 	}
2346 
2347 	ice_clean_adminq_subtask(pf);
2348 	ice_check_media_subtask(pf);
2349 	ice_check_for_hang_subtask(pf);
2350 	ice_sync_fltr_subtask(pf);
2351 	ice_handle_mdd_event(pf);
2352 	ice_watchdog_subtask(pf);
2353 
2354 	if (ice_is_safe_mode(pf)) {
2355 		ice_service_task_complete(pf);
2356 		return;
2357 	}
2358 
2359 	ice_process_vflr_event(pf);
2360 	ice_clean_mailboxq_subtask(pf);
2361 	ice_clean_sbq_subtask(pf);
2362 	ice_sync_arfs_fltrs(pf);
2363 	ice_flush_fdir_ctx(pf);
2364 
2365 	/* Clear ICE_SERVICE_SCHED flag to allow scheduling next event */
2366 	ice_service_task_complete(pf);
2367 
2368 	/* If the tasks have taken longer than one service timer period
2369 	 * or there is more work to be done, reset the service timer to
2370 	 * schedule the service task now.
2371 	 */
2372 	if (time_after(jiffies, (start_time + pf->serv_tmr_period)) ||
2373 	    test_bit(ICE_MDD_EVENT_PENDING, pf->state) ||
2374 	    test_bit(ICE_VFLR_EVENT_PENDING, pf->state) ||
2375 	    test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state) ||
2376 	    test_bit(ICE_FD_VF_FLUSH_CTX, pf->state) ||
2377 	    test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state) ||
2378 	    test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state))
2379 		mod_timer(&pf->serv_tmr, jiffies);
2380 }
2381 
2382 /**
2383  * ice_set_ctrlq_len - helper function to set controlq length
2384  * @hw: pointer to the HW instance
2385  */
2386 static void ice_set_ctrlq_len(struct ice_hw *hw)
2387 {
2388 	hw->adminq.num_rq_entries = ICE_AQ_LEN;
2389 	hw->adminq.num_sq_entries = ICE_AQ_LEN;
2390 	hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN;
2391 	hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN;
2392 	hw->mailboxq.num_rq_entries = PF_MBX_ARQLEN_ARQLEN_M;
2393 	hw->mailboxq.num_sq_entries = ICE_MBXSQ_LEN;
2394 	hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
2395 	hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
2396 	hw->sbq.num_rq_entries = ICE_SBQ_LEN;
2397 	hw->sbq.num_sq_entries = ICE_SBQ_LEN;
2398 	hw->sbq.rq_buf_size = ICE_SBQ_MAX_BUF_LEN;
2399 	hw->sbq.sq_buf_size = ICE_SBQ_MAX_BUF_LEN;
2400 }
2401 
2402 /**
2403  * ice_schedule_reset - schedule a reset
2404  * @pf: board private structure
2405  * @reset: reset being requested
2406  */
2407 int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset)
2408 {
2409 	struct device *dev = ice_pf_to_dev(pf);
2410 
2411 	/* bail out if earlier reset has failed */
2412 	if (test_bit(ICE_RESET_FAILED, pf->state)) {
2413 		dev_dbg(dev, "earlier reset has failed\n");
2414 		return -EIO;
2415 	}
2416 	/* bail if reset/recovery already in progress */
2417 	if (ice_is_reset_in_progress(pf->state)) {
2418 		dev_dbg(dev, "Reset already in progress\n");
2419 		return -EBUSY;
2420 	}
2421 
2422 	switch (reset) {
2423 	case ICE_RESET_PFR:
2424 		set_bit(ICE_PFR_REQ, pf->state);
2425 		break;
2426 	case ICE_RESET_CORER:
2427 		set_bit(ICE_CORER_REQ, pf->state);
2428 		break;
2429 	case ICE_RESET_GLOBR:
2430 		set_bit(ICE_GLOBR_REQ, pf->state);
2431 		break;
2432 	default:
2433 		return -EINVAL;
2434 	}
2435 
2436 	ice_service_task_schedule(pf);
2437 	return 0;
2438 }
2439 
2440 /**
2441  * ice_irq_affinity_notify - Callback for affinity changes
2442  * @notify: context as to what irq was changed
2443  * @mask: the new affinity mask
2444  *
2445  * This is a callback function used by the irq_set_affinity_notifier function
2446  * so that we may register to receive changes to the irq affinity masks.
2447  */
2448 static void
2449 ice_irq_affinity_notify(struct irq_affinity_notify *notify,
2450 			const cpumask_t *mask)
2451 {
2452 	struct ice_q_vector *q_vector =
2453 		container_of(notify, struct ice_q_vector, affinity_notify);
2454 
2455 	cpumask_copy(&q_vector->affinity_mask, mask);
2456 }
2457 
2458 /**
2459  * ice_irq_affinity_release - Callback for affinity notifier release
2460  * @ref: internal core kernel usage
2461  *
2462  * This is a callback function used by the irq_set_affinity_notifier function
2463  * to inform the current notification subscriber that they will no longer
2464  * receive notifications.
2465  */
2466 static void ice_irq_affinity_release(struct kref __always_unused *ref) {}
2467 
2468 /**
2469  * ice_vsi_ena_irq - Enable IRQ for the given VSI
2470  * @vsi: the VSI being configured
2471  */
2472 static int ice_vsi_ena_irq(struct ice_vsi *vsi)
2473 {
2474 	struct ice_hw *hw = &vsi->back->hw;
2475 	int i;
2476 
2477 	ice_for_each_q_vector(vsi, i)
2478 		ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]);
2479 
2480 	ice_flush(hw);
2481 	return 0;
2482 }
2483 
2484 /**
2485  * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI
2486  * @vsi: the VSI being configured
2487  * @basename: name for the vector
2488  */
2489 static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename)
2490 {
2491 	int q_vectors = vsi->num_q_vectors;
2492 	struct ice_pf *pf = vsi->back;
2493 	int base = vsi->base_vector;
2494 	struct device *dev;
2495 	int rx_int_idx = 0;
2496 	int tx_int_idx = 0;
2497 	int vector, err;
2498 	int irq_num;
2499 
2500 	dev = ice_pf_to_dev(pf);
2501 	for (vector = 0; vector < q_vectors; vector++) {
2502 		struct ice_q_vector *q_vector = vsi->q_vectors[vector];
2503 
2504 		irq_num = pf->msix_entries[base + vector].vector;
2505 
2506 		if (q_vector->tx.tx_ring && q_vector->rx.rx_ring) {
2507 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2508 				 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
2509 			tx_int_idx++;
2510 		} else if (q_vector->rx.rx_ring) {
2511 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2512 				 "%s-%s-%d", basename, "rx", rx_int_idx++);
2513 		} else if (q_vector->tx.tx_ring) {
2514 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2515 				 "%s-%s-%d", basename, "tx", tx_int_idx++);
2516 		} else {
2517 			/* skip this unused q_vector */
2518 			continue;
2519 		}
2520 		if (vsi->type == ICE_VSI_CTRL && vsi->vf)
2521 			err = devm_request_irq(dev, irq_num, vsi->irq_handler,
2522 					       IRQF_SHARED, q_vector->name,
2523 					       q_vector);
2524 		else
2525 			err = devm_request_irq(dev, irq_num, vsi->irq_handler,
2526 					       0, q_vector->name, q_vector);
2527 		if (err) {
2528 			netdev_err(vsi->netdev, "MSIX request_irq failed, error: %d\n",
2529 				   err);
2530 			goto free_q_irqs;
2531 		}
2532 
2533 		/* register for affinity change notifications */
2534 		if (!IS_ENABLED(CONFIG_RFS_ACCEL)) {
2535 			struct irq_affinity_notify *affinity_notify;
2536 
2537 			affinity_notify = &q_vector->affinity_notify;
2538 			affinity_notify->notify = ice_irq_affinity_notify;
2539 			affinity_notify->release = ice_irq_affinity_release;
2540 			irq_set_affinity_notifier(irq_num, affinity_notify);
2541 		}
2542 
2543 		/* assign the mask for this irq */
2544 		irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
2545 	}
2546 
2547 	err = ice_set_cpu_rx_rmap(vsi);
2548 	if (err) {
2549 		netdev_err(vsi->netdev, "Failed to setup CPU RMAP on VSI %u: %pe\n",
2550 			   vsi->vsi_num, ERR_PTR(err));
2551 		goto free_q_irqs;
2552 	}
2553 
2554 	vsi->irqs_ready = true;
2555 	return 0;
2556 
2557 free_q_irqs:
2558 	while (vector) {
2559 		vector--;
2560 		irq_num = pf->msix_entries[base + vector].vector;
2561 		if (!IS_ENABLED(CONFIG_RFS_ACCEL))
2562 			irq_set_affinity_notifier(irq_num, NULL);
2563 		irq_set_affinity_hint(irq_num, NULL);
2564 		devm_free_irq(dev, irq_num, &vsi->q_vectors[vector]);
2565 	}
2566 	return err;
2567 }
2568 
2569 /**
2570  * ice_xdp_alloc_setup_rings - Allocate and setup Tx rings for XDP
2571  * @vsi: VSI to setup Tx rings used by XDP
2572  *
2573  * Return 0 on success and negative value on error
2574  */
2575 static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi)
2576 {
2577 	struct device *dev = ice_pf_to_dev(vsi->back);
2578 	struct ice_tx_desc *tx_desc;
2579 	int i, j;
2580 
2581 	ice_for_each_xdp_txq(vsi, i) {
2582 		u16 xdp_q_idx = vsi->alloc_txq + i;
2583 		struct ice_ring_stats *ring_stats;
2584 		struct ice_tx_ring *xdp_ring;
2585 
2586 		xdp_ring = kzalloc(sizeof(*xdp_ring), GFP_KERNEL);
2587 		if (!xdp_ring)
2588 			goto free_xdp_rings;
2589 
2590 		ring_stats = kzalloc(sizeof(*ring_stats), GFP_KERNEL);
2591 		if (!ring_stats) {
2592 			ice_free_tx_ring(xdp_ring);
2593 			goto free_xdp_rings;
2594 		}
2595 
2596 		xdp_ring->ring_stats = ring_stats;
2597 		xdp_ring->q_index = xdp_q_idx;
2598 		xdp_ring->reg_idx = vsi->txq_map[xdp_q_idx];
2599 		xdp_ring->vsi = vsi;
2600 		xdp_ring->netdev = NULL;
2601 		xdp_ring->dev = dev;
2602 		xdp_ring->count = vsi->num_tx_desc;
2603 		WRITE_ONCE(vsi->xdp_rings[i], xdp_ring);
2604 		if (ice_setup_tx_ring(xdp_ring))
2605 			goto free_xdp_rings;
2606 		ice_set_ring_xdp(xdp_ring);
2607 		spin_lock_init(&xdp_ring->tx_lock);
2608 		for (j = 0; j < xdp_ring->count; j++) {
2609 			tx_desc = ICE_TX_DESC(xdp_ring, j);
2610 			tx_desc->cmd_type_offset_bsz = 0;
2611 		}
2612 	}
2613 
2614 	return 0;
2615 
2616 free_xdp_rings:
2617 	for (; i >= 0; i--) {
2618 		if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) {
2619 			kfree_rcu(vsi->xdp_rings[i]->ring_stats, rcu);
2620 			vsi->xdp_rings[i]->ring_stats = NULL;
2621 			ice_free_tx_ring(vsi->xdp_rings[i]);
2622 		}
2623 	}
2624 	return -ENOMEM;
2625 }
2626 
2627 /**
2628  * ice_vsi_assign_bpf_prog - set or clear bpf prog pointer on VSI
2629  * @vsi: VSI to set the bpf prog on
2630  * @prog: the bpf prog pointer
2631  */
2632 static void ice_vsi_assign_bpf_prog(struct ice_vsi *vsi, struct bpf_prog *prog)
2633 {
2634 	struct bpf_prog *old_prog;
2635 	int i;
2636 
2637 	old_prog = xchg(&vsi->xdp_prog, prog);
2638 	if (old_prog)
2639 		bpf_prog_put(old_prog);
2640 
2641 	ice_for_each_rxq(vsi, i)
2642 		WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
2643 }
2644 
2645 /**
2646  * ice_prepare_xdp_rings - Allocate, configure and setup Tx rings for XDP
2647  * @vsi: VSI to bring up Tx rings used by XDP
2648  * @prog: bpf program that will be assigned to VSI
2649  *
2650  * Return 0 on success and negative value on error
2651  */
2652 int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog)
2653 {
2654 	u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
2655 	int xdp_rings_rem = vsi->num_xdp_txq;
2656 	struct ice_pf *pf = vsi->back;
2657 	struct ice_qs_cfg xdp_qs_cfg = {
2658 		.qs_mutex = &pf->avail_q_mutex,
2659 		.pf_map = pf->avail_txqs,
2660 		.pf_map_size = pf->max_pf_txqs,
2661 		.q_count = vsi->num_xdp_txq,
2662 		.scatter_count = ICE_MAX_SCATTER_TXQS,
2663 		.vsi_map = vsi->txq_map,
2664 		.vsi_map_offset = vsi->alloc_txq,
2665 		.mapping_mode = ICE_VSI_MAP_CONTIG
2666 	};
2667 	struct device *dev;
2668 	int i, v_idx;
2669 	int status;
2670 
2671 	dev = ice_pf_to_dev(pf);
2672 	vsi->xdp_rings = devm_kcalloc(dev, vsi->num_xdp_txq,
2673 				      sizeof(*vsi->xdp_rings), GFP_KERNEL);
2674 	if (!vsi->xdp_rings)
2675 		return -ENOMEM;
2676 
2677 	vsi->xdp_mapping_mode = xdp_qs_cfg.mapping_mode;
2678 	if (__ice_vsi_get_qs(&xdp_qs_cfg))
2679 		goto err_map_xdp;
2680 
2681 	if (static_key_enabled(&ice_xdp_locking_key))
2682 		netdev_warn(vsi->netdev,
2683 			    "Could not allocate one XDP Tx ring per CPU, XDP_TX/XDP_REDIRECT actions will be slower\n");
2684 
2685 	if (ice_xdp_alloc_setup_rings(vsi))
2686 		goto clear_xdp_rings;
2687 
2688 	/* follow the logic from ice_vsi_map_rings_to_vectors */
2689 	ice_for_each_q_vector(vsi, v_idx) {
2690 		struct ice_q_vector *q_vector = vsi->q_vectors[v_idx];
2691 		int xdp_rings_per_v, q_id, q_base;
2692 
2693 		xdp_rings_per_v = DIV_ROUND_UP(xdp_rings_rem,
2694 					       vsi->num_q_vectors - v_idx);
2695 		q_base = vsi->num_xdp_txq - xdp_rings_rem;
2696 
2697 		for (q_id = q_base; q_id < (q_base + xdp_rings_per_v); q_id++) {
2698 			struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_id];
2699 
2700 			xdp_ring->q_vector = q_vector;
2701 			xdp_ring->next = q_vector->tx.tx_ring;
2702 			q_vector->tx.tx_ring = xdp_ring;
2703 		}
2704 		xdp_rings_rem -= xdp_rings_per_v;
2705 	}
2706 
2707 	ice_for_each_rxq(vsi, i) {
2708 		if (static_key_enabled(&ice_xdp_locking_key)) {
2709 			vsi->rx_rings[i]->xdp_ring = vsi->xdp_rings[i % vsi->num_xdp_txq];
2710 		} else {
2711 			struct ice_q_vector *q_vector = vsi->rx_rings[i]->q_vector;
2712 			struct ice_tx_ring *ring;
2713 
2714 			ice_for_each_tx_ring(ring, q_vector->tx) {
2715 				if (ice_ring_is_xdp(ring)) {
2716 					vsi->rx_rings[i]->xdp_ring = ring;
2717 					break;
2718 				}
2719 			}
2720 		}
2721 		ice_tx_xsk_pool(vsi, i);
2722 	}
2723 
2724 	/* omit the scheduler update if in reset path; XDP queues will be
2725 	 * taken into account at the end of ice_vsi_rebuild, where
2726 	 * ice_cfg_vsi_lan is being called
2727 	 */
2728 	if (ice_is_reset_in_progress(pf->state))
2729 		return 0;
2730 
2731 	/* tell the Tx scheduler that right now we have
2732 	 * additional queues
2733 	 */
2734 	for (i = 0; i < vsi->tc_cfg.numtc; i++)
2735 		max_txqs[i] = vsi->num_txq + vsi->num_xdp_txq;
2736 
2737 	status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc,
2738 				 max_txqs);
2739 	if (status) {
2740 		dev_err(dev, "Failed VSI LAN queue config for XDP, error: %d\n",
2741 			status);
2742 		goto clear_xdp_rings;
2743 	}
2744 
2745 	/* assign the prog only when it's not already present on VSI;
2746 	 * this flow is a subject of both ethtool -L and ndo_bpf flows;
2747 	 * VSI rebuild that happens under ethtool -L can expose us to
2748 	 * the bpf_prog refcount issues as we would be swapping same
2749 	 * bpf_prog pointers from vsi->xdp_prog and calling bpf_prog_put
2750 	 * on it as it would be treated as an 'old_prog'; for ndo_bpf
2751 	 * this is not harmful as dev_xdp_install bumps the refcount
2752 	 * before calling the op exposed by the driver;
2753 	 */
2754 	if (!ice_is_xdp_ena_vsi(vsi))
2755 		ice_vsi_assign_bpf_prog(vsi, prog);
2756 
2757 	return 0;
2758 clear_xdp_rings:
2759 	ice_for_each_xdp_txq(vsi, i)
2760 		if (vsi->xdp_rings[i]) {
2761 			kfree_rcu(vsi->xdp_rings[i], rcu);
2762 			vsi->xdp_rings[i] = NULL;
2763 		}
2764 
2765 err_map_xdp:
2766 	mutex_lock(&pf->avail_q_mutex);
2767 	ice_for_each_xdp_txq(vsi, i) {
2768 		clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs);
2769 		vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX;
2770 	}
2771 	mutex_unlock(&pf->avail_q_mutex);
2772 
2773 	devm_kfree(dev, vsi->xdp_rings);
2774 	return -ENOMEM;
2775 }
2776 
2777 /**
2778  * ice_destroy_xdp_rings - undo the configuration made by ice_prepare_xdp_rings
2779  * @vsi: VSI to remove XDP rings
2780  *
2781  * Detach XDP rings from irq vectors, clean up the PF bitmap and free
2782  * resources
2783  */
2784 int ice_destroy_xdp_rings(struct ice_vsi *vsi)
2785 {
2786 	u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
2787 	struct ice_pf *pf = vsi->back;
2788 	int i, v_idx;
2789 
2790 	/* q_vectors are freed in reset path so there's no point in detaching
2791 	 * rings; in case of rebuild being triggered not from reset bits
2792 	 * in pf->state won't be set, so additionally check first q_vector
2793 	 * against NULL
2794 	 */
2795 	if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0])
2796 		goto free_qmap;
2797 
2798 	ice_for_each_q_vector(vsi, v_idx) {
2799 		struct ice_q_vector *q_vector = vsi->q_vectors[v_idx];
2800 		struct ice_tx_ring *ring;
2801 
2802 		ice_for_each_tx_ring(ring, q_vector->tx)
2803 			if (!ring->tx_buf || !ice_ring_is_xdp(ring))
2804 				break;
2805 
2806 		/* restore the value of last node prior to XDP setup */
2807 		q_vector->tx.tx_ring = ring;
2808 	}
2809 
2810 free_qmap:
2811 	mutex_lock(&pf->avail_q_mutex);
2812 	ice_for_each_xdp_txq(vsi, i) {
2813 		clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs);
2814 		vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX;
2815 	}
2816 	mutex_unlock(&pf->avail_q_mutex);
2817 
2818 	ice_for_each_xdp_txq(vsi, i)
2819 		if (vsi->xdp_rings[i]) {
2820 			if (vsi->xdp_rings[i]->desc) {
2821 				synchronize_rcu();
2822 				ice_free_tx_ring(vsi->xdp_rings[i]);
2823 			}
2824 			kfree_rcu(vsi->xdp_rings[i]->ring_stats, rcu);
2825 			vsi->xdp_rings[i]->ring_stats = NULL;
2826 			kfree_rcu(vsi->xdp_rings[i], rcu);
2827 			vsi->xdp_rings[i] = NULL;
2828 		}
2829 
2830 	devm_kfree(ice_pf_to_dev(pf), vsi->xdp_rings);
2831 	vsi->xdp_rings = NULL;
2832 
2833 	if (static_key_enabled(&ice_xdp_locking_key))
2834 		static_branch_dec(&ice_xdp_locking_key);
2835 
2836 	if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0])
2837 		return 0;
2838 
2839 	ice_vsi_assign_bpf_prog(vsi, NULL);
2840 
2841 	/* notify Tx scheduler that we destroyed XDP queues and bring
2842 	 * back the old number of child nodes
2843 	 */
2844 	for (i = 0; i < vsi->tc_cfg.numtc; i++)
2845 		max_txqs[i] = vsi->num_txq;
2846 
2847 	/* change number of XDP Tx queues to 0 */
2848 	vsi->num_xdp_txq = 0;
2849 
2850 	return ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc,
2851 			       max_txqs);
2852 }
2853 
2854 /**
2855  * ice_vsi_rx_napi_schedule - Schedule napi on RX queues from VSI
2856  * @vsi: VSI to schedule napi on
2857  */
2858 static void ice_vsi_rx_napi_schedule(struct ice_vsi *vsi)
2859 {
2860 	int i;
2861 
2862 	ice_for_each_rxq(vsi, i) {
2863 		struct ice_rx_ring *rx_ring = vsi->rx_rings[i];
2864 
2865 		if (rx_ring->xsk_pool)
2866 			napi_schedule(&rx_ring->q_vector->napi);
2867 	}
2868 }
2869 
2870 /**
2871  * ice_vsi_determine_xdp_res - figure out how many Tx qs can XDP have
2872  * @vsi: VSI to determine the count of XDP Tx qs
2873  *
2874  * returns 0 if Tx qs count is higher than at least half of CPU count,
2875  * -ENOMEM otherwise
2876  */
2877 int ice_vsi_determine_xdp_res(struct ice_vsi *vsi)
2878 {
2879 	u16 avail = ice_get_avail_txq_count(vsi->back);
2880 	u16 cpus = num_possible_cpus();
2881 
2882 	if (avail < cpus / 2)
2883 		return -ENOMEM;
2884 
2885 	vsi->num_xdp_txq = min_t(u16, avail, cpus);
2886 
2887 	if (vsi->num_xdp_txq < cpus)
2888 		static_branch_inc(&ice_xdp_locking_key);
2889 
2890 	return 0;
2891 }
2892 
2893 /**
2894  * ice_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2895  * @vsi: Pointer to VSI structure
2896  */
2897 static int ice_max_xdp_frame_size(struct ice_vsi *vsi)
2898 {
2899 	if (test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags))
2900 		return ICE_RXBUF_1664;
2901 	else
2902 		return ICE_RXBUF_3072;
2903 }
2904 
2905 /**
2906  * ice_xdp_setup_prog - Add or remove XDP eBPF program
2907  * @vsi: VSI to setup XDP for
2908  * @prog: XDP program
2909  * @extack: netlink extended ack
2910  */
2911 static int
2912 ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog,
2913 		   struct netlink_ext_ack *extack)
2914 {
2915 	unsigned int frame_size = vsi->netdev->mtu + ICE_ETH_PKT_HDR_PAD;
2916 	bool if_running = netif_running(vsi->netdev);
2917 	int ret = 0, xdp_ring_err = 0;
2918 
2919 	if (prog && !prog->aux->xdp_has_frags) {
2920 		if (frame_size > ice_max_xdp_frame_size(vsi)) {
2921 			NL_SET_ERR_MSG_MOD(extack,
2922 					   "MTU is too large for linear frames and XDP prog does not support frags");
2923 			return -EOPNOTSUPP;
2924 		}
2925 	}
2926 
2927 	/* need to stop netdev while setting up the program for Rx rings */
2928 	if (if_running && !test_and_set_bit(ICE_VSI_DOWN, vsi->state)) {
2929 		ret = ice_down(vsi);
2930 		if (ret) {
2931 			NL_SET_ERR_MSG_MOD(extack, "Preparing device for XDP attach failed");
2932 			return ret;
2933 		}
2934 	}
2935 
2936 	if (!ice_is_xdp_ena_vsi(vsi) && prog) {
2937 		xdp_ring_err = ice_vsi_determine_xdp_res(vsi);
2938 		if (xdp_ring_err) {
2939 			NL_SET_ERR_MSG_MOD(extack, "Not enough Tx resources for XDP");
2940 		} else {
2941 			xdp_ring_err = ice_prepare_xdp_rings(vsi, prog);
2942 			if (xdp_ring_err)
2943 				NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Tx resources failed");
2944 		}
2945 		xdp_features_set_redirect_target(vsi->netdev, true);
2946 		/* reallocate Rx queues that are used for zero-copy */
2947 		xdp_ring_err = ice_realloc_zc_buf(vsi, true);
2948 		if (xdp_ring_err)
2949 			NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Rx resources failed");
2950 	} else if (ice_is_xdp_ena_vsi(vsi) && !prog) {
2951 		xdp_features_clear_redirect_target(vsi->netdev);
2952 		xdp_ring_err = ice_destroy_xdp_rings(vsi);
2953 		if (xdp_ring_err)
2954 			NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Tx resources failed");
2955 		/* reallocate Rx queues that were used for zero-copy */
2956 		xdp_ring_err = ice_realloc_zc_buf(vsi, false);
2957 		if (xdp_ring_err)
2958 			NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Rx resources failed");
2959 	} else {
2960 		/* safe to call even when prog == vsi->xdp_prog as
2961 		 * dev_xdp_install in net/core/dev.c incremented prog's
2962 		 * refcount so corresponding bpf_prog_put won't cause
2963 		 * underflow
2964 		 */
2965 		ice_vsi_assign_bpf_prog(vsi, prog);
2966 	}
2967 
2968 	if (if_running)
2969 		ret = ice_up(vsi);
2970 
2971 	if (!ret && prog)
2972 		ice_vsi_rx_napi_schedule(vsi);
2973 
2974 	return (ret || xdp_ring_err) ? -ENOMEM : 0;
2975 }
2976 
2977 /**
2978  * ice_xdp_safe_mode - XDP handler for safe mode
2979  * @dev: netdevice
2980  * @xdp: XDP command
2981  */
2982 static int ice_xdp_safe_mode(struct net_device __always_unused *dev,
2983 			     struct netdev_bpf *xdp)
2984 {
2985 	NL_SET_ERR_MSG_MOD(xdp->extack,
2986 			   "Please provide working DDP firmware package in order to use XDP\n"
2987 			   "Refer to Documentation/networking/device_drivers/ethernet/intel/ice.rst");
2988 	return -EOPNOTSUPP;
2989 }
2990 
2991 /**
2992  * ice_xdp - implements XDP handler
2993  * @dev: netdevice
2994  * @xdp: XDP command
2995  */
2996 static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp)
2997 {
2998 	struct ice_netdev_priv *np = netdev_priv(dev);
2999 	struct ice_vsi *vsi = np->vsi;
3000 
3001 	if (vsi->type != ICE_VSI_PF) {
3002 		NL_SET_ERR_MSG_MOD(xdp->extack, "XDP can be loaded only on PF VSI");
3003 		return -EINVAL;
3004 	}
3005 
3006 	switch (xdp->command) {
3007 	case XDP_SETUP_PROG:
3008 		return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack);
3009 	case XDP_SETUP_XSK_POOL:
3010 		return ice_xsk_pool_setup(vsi, xdp->xsk.pool,
3011 					  xdp->xsk.queue_id);
3012 	default:
3013 		return -EINVAL;
3014 	}
3015 }
3016 
3017 /**
3018  * ice_ena_misc_vector - enable the non-queue interrupts
3019  * @pf: board private structure
3020  */
3021 static void ice_ena_misc_vector(struct ice_pf *pf)
3022 {
3023 	struct ice_hw *hw = &pf->hw;
3024 	u32 val;
3025 
3026 	/* Disable anti-spoof detection interrupt to prevent spurious event
3027 	 * interrupts during a function reset. Anti-spoof functionally is
3028 	 * still supported.
3029 	 */
3030 	val = rd32(hw, GL_MDCK_TX_TDPU);
3031 	val |= GL_MDCK_TX_TDPU_RCU_ANTISPOOF_ITR_DIS_M;
3032 	wr32(hw, GL_MDCK_TX_TDPU, val);
3033 
3034 	/* clear things first */
3035 	wr32(hw, PFINT_OICR_ENA, 0);	/* disable all */
3036 	rd32(hw, PFINT_OICR);		/* read to clear */
3037 
3038 	val = (PFINT_OICR_ECC_ERR_M |
3039 	       PFINT_OICR_MAL_DETECT_M |
3040 	       PFINT_OICR_GRST_M |
3041 	       PFINT_OICR_PCI_EXCEPTION_M |
3042 	       PFINT_OICR_VFLR_M |
3043 	       PFINT_OICR_HMC_ERR_M |
3044 	       PFINT_OICR_PE_PUSH_M |
3045 	       PFINT_OICR_PE_CRITERR_M);
3046 
3047 	wr32(hw, PFINT_OICR_ENA, val);
3048 
3049 	/* SW_ITR_IDX = 0, but don't change INTENA */
3050 	wr32(hw, GLINT_DYN_CTL(pf->oicr_idx),
3051 	     GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M);
3052 }
3053 
3054 /**
3055  * ice_misc_intr - misc interrupt handler
3056  * @irq: interrupt number
3057  * @data: pointer to a q_vector
3058  */
3059 static irqreturn_t ice_misc_intr(int __always_unused irq, void *data)
3060 {
3061 	struct ice_pf *pf = (struct ice_pf *)data;
3062 	struct ice_hw *hw = &pf->hw;
3063 	irqreturn_t ret = IRQ_NONE;
3064 	struct device *dev;
3065 	u32 oicr, ena_mask;
3066 
3067 	dev = ice_pf_to_dev(pf);
3068 	set_bit(ICE_ADMINQ_EVENT_PENDING, pf->state);
3069 	set_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state);
3070 	set_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state);
3071 
3072 	oicr = rd32(hw, PFINT_OICR);
3073 	ena_mask = rd32(hw, PFINT_OICR_ENA);
3074 
3075 	if (oicr & PFINT_OICR_SWINT_M) {
3076 		ena_mask &= ~PFINT_OICR_SWINT_M;
3077 		pf->sw_int_count++;
3078 	}
3079 
3080 	if (oicr & PFINT_OICR_MAL_DETECT_M) {
3081 		ena_mask &= ~PFINT_OICR_MAL_DETECT_M;
3082 		set_bit(ICE_MDD_EVENT_PENDING, pf->state);
3083 	}
3084 	if (oicr & PFINT_OICR_VFLR_M) {
3085 		/* disable any further VFLR event notifications */
3086 		if (test_bit(ICE_VF_RESETS_DISABLED, pf->state)) {
3087 			u32 reg = rd32(hw, PFINT_OICR_ENA);
3088 
3089 			reg &= ~PFINT_OICR_VFLR_M;
3090 			wr32(hw, PFINT_OICR_ENA, reg);
3091 		} else {
3092 			ena_mask &= ~PFINT_OICR_VFLR_M;
3093 			set_bit(ICE_VFLR_EVENT_PENDING, pf->state);
3094 		}
3095 	}
3096 
3097 	if (oicr & PFINT_OICR_GRST_M) {
3098 		u32 reset;
3099 
3100 		/* we have a reset warning */
3101 		ena_mask &= ~PFINT_OICR_GRST_M;
3102 		reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >>
3103 			GLGEN_RSTAT_RESET_TYPE_S;
3104 
3105 		if (reset == ICE_RESET_CORER)
3106 			pf->corer_count++;
3107 		else if (reset == ICE_RESET_GLOBR)
3108 			pf->globr_count++;
3109 		else if (reset == ICE_RESET_EMPR)
3110 			pf->empr_count++;
3111 		else
3112 			dev_dbg(dev, "Invalid reset type %d\n", reset);
3113 
3114 		/* If a reset cycle isn't already in progress, we set a bit in
3115 		 * pf->state so that the service task can start a reset/rebuild.
3116 		 */
3117 		if (!test_and_set_bit(ICE_RESET_OICR_RECV, pf->state)) {
3118 			if (reset == ICE_RESET_CORER)
3119 				set_bit(ICE_CORER_RECV, pf->state);
3120 			else if (reset == ICE_RESET_GLOBR)
3121 				set_bit(ICE_GLOBR_RECV, pf->state);
3122 			else
3123 				set_bit(ICE_EMPR_RECV, pf->state);
3124 
3125 			/* There are couple of different bits at play here.
3126 			 * hw->reset_ongoing indicates whether the hardware is
3127 			 * in reset. This is set to true when a reset interrupt
3128 			 * is received and set back to false after the driver
3129 			 * has determined that the hardware is out of reset.
3130 			 *
3131 			 * ICE_RESET_OICR_RECV in pf->state indicates
3132 			 * that a post reset rebuild is required before the
3133 			 * driver is operational again. This is set above.
3134 			 *
3135 			 * As this is the start of the reset/rebuild cycle, set
3136 			 * both to indicate that.
3137 			 */
3138 			hw->reset_ongoing = true;
3139 		}
3140 	}
3141 
3142 	if (oicr & PFINT_OICR_TSYN_TX_M) {
3143 		ena_mask &= ~PFINT_OICR_TSYN_TX_M;
3144 		if (!hw->reset_ongoing)
3145 			ret = IRQ_WAKE_THREAD;
3146 	}
3147 
3148 	if (oicr & PFINT_OICR_TSYN_EVNT_M) {
3149 		u8 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned;
3150 		u32 gltsyn_stat = rd32(hw, GLTSYN_STAT(tmr_idx));
3151 
3152 		/* Save EVENTs from GTSYN register */
3153 		pf->ptp.ext_ts_irq |= gltsyn_stat & (GLTSYN_STAT_EVENT0_M |
3154 						     GLTSYN_STAT_EVENT1_M |
3155 						     GLTSYN_STAT_EVENT2_M);
3156 		ena_mask &= ~PFINT_OICR_TSYN_EVNT_M;
3157 		kthread_queue_work(pf->ptp.kworker, &pf->ptp.extts_work);
3158 	}
3159 
3160 #define ICE_AUX_CRIT_ERR (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M | PFINT_OICR_PE_PUSH_M)
3161 	if (oicr & ICE_AUX_CRIT_ERR) {
3162 		pf->oicr_err_reg |= oicr;
3163 		set_bit(ICE_AUX_ERR_PENDING, pf->state);
3164 		ena_mask &= ~ICE_AUX_CRIT_ERR;
3165 	}
3166 
3167 	/* Report any remaining unexpected interrupts */
3168 	oicr &= ena_mask;
3169 	if (oicr) {
3170 		dev_dbg(dev, "unhandled interrupt oicr=0x%08x\n", oicr);
3171 		/* If a critical error is pending there is no choice but to
3172 		 * reset the device.
3173 		 */
3174 		if (oicr & (PFINT_OICR_PCI_EXCEPTION_M |
3175 			    PFINT_OICR_ECC_ERR_M)) {
3176 			set_bit(ICE_PFR_REQ, pf->state);
3177 			ice_service_task_schedule(pf);
3178 		}
3179 	}
3180 	if (!ret)
3181 		ret = IRQ_HANDLED;
3182 
3183 	ice_service_task_schedule(pf);
3184 	ice_irq_dynamic_ena(hw, NULL, NULL);
3185 
3186 	return ret;
3187 }
3188 
3189 /**
3190  * ice_misc_intr_thread_fn - misc interrupt thread function
3191  * @irq: interrupt number
3192  * @data: pointer to a q_vector
3193  */
3194 static irqreturn_t ice_misc_intr_thread_fn(int __always_unused irq, void *data)
3195 {
3196 	struct ice_pf *pf = data;
3197 
3198 	if (ice_is_reset_in_progress(pf->state))
3199 		return IRQ_HANDLED;
3200 
3201 	while (!ice_ptp_process_ts(pf))
3202 		usleep_range(50, 100);
3203 
3204 	return IRQ_HANDLED;
3205 }
3206 
3207 /**
3208  * ice_dis_ctrlq_interrupts - disable control queue interrupts
3209  * @hw: pointer to HW structure
3210  */
3211 static void ice_dis_ctrlq_interrupts(struct ice_hw *hw)
3212 {
3213 	/* disable Admin queue Interrupt causes */
3214 	wr32(hw, PFINT_FW_CTL,
3215 	     rd32(hw, PFINT_FW_CTL) & ~PFINT_FW_CTL_CAUSE_ENA_M);
3216 
3217 	/* disable Mailbox queue Interrupt causes */
3218 	wr32(hw, PFINT_MBX_CTL,
3219 	     rd32(hw, PFINT_MBX_CTL) & ~PFINT_MBX_CTL_CAUSE_ENA_M);
3220 
3221 	wr32(hw, PFINT_SB_CTL,
3222 	     rd32(hw, PFINT_SB_CTL) & ~PFINT_SB_CTL_CAUSE_ENA_M);
3223 
3224 	/* disable Control queue Interrupt causes */
3225 	wr32(hw, PFINT_OICR_CTL,
3226 	     rd32(hw, PFINT_OICR_CTL) & ~PFINT_OICR_CTL_CAUSE_ENA_M);
3227 
3228 	ice_flush(hw);
3229 }
3230 
3231 /**
3232  * ice_free_irq_msix_misc - Unroll misc vector setup
3233  * @pf: board private structure
3234  */
3235 static void ice_free_irq_msix_misc(struct ice_pf *pf)
3236 {
3237 	struct ice_hw *hw = &pf->hw;
3238 
3239 	ice_dis_ctrlq_interrupts(hw);
3240 
3241 	/* disable OICR interrupt */
3242 	wr32(hw, PFINT_OICR_ENA, 0);
3243 	ice_flush(hw);
3244 
3245 	if (pf->msix_entries) {
3246 		synchronize_irq(pf->msix_entries[pf->oicr_idx].vector);
3247 		devm_free_irq(ice_pf_to_dev(pf),
3248 			      pf->msix_entries[pf->oicr_idx].vector, pf);
3249 	}
3250 
3251 	pf->num_avail_sw_msix += 1;
3252 	ice_free_res(pf->irq_tracker, pf->oicr_idx, ICE_RES_MISC_VEC_ID);
3253 }
3254 
3255 /**
3256  * ice_ena_ctrlq_interrupts - enable control queue interrupts
3257  * @hw: pointer to HW structure
3258  * @reg_idx: HW vector index to associate the control queue interrupts with
3259  */
3260 static void ice_ena_ctrlq_interrupts(struct ice_hw *hw, u16 reg_idx)
3261 {
3262 	u32 val;
3263 
3264 	val = ((reg_idx & PFINT_OICR_CTL_MSIX_INDX_M) |
3265 	       PFINT_OICR_CTL_CAUSE_ENA_M);
3266 	wr32(hw, PFINT_OICR_CTL, val);
3267 
3268 	/* enable Admin queue Interrupt causes */
3269 	val = ((reg_idx & PFINT_FW_CTL_MSIX_INDX_M) |
3270 	       PFINT_FW_CTL_CAUSE_ENA_M);
3271 	wr32(hw, PFINT_FW_CTL, val);
3272 
3273 	/* enable Mailbox queue Interrupt causes */
3274 	val = ((reg_idx & PFINT_MBX_CTL_MSIX_INDX_M) |
3275 	       PFINT_MBX_CTL_CAUSE_ENA_M);
3276 	wr32(hw, PFINT_MBX_CTL, val);
3277 
3278 	/* This enables Sideband queue Interrupt causes */
3279 	val = ((reg_idx & PFINT_SB_CTL_MSIX_INDX_M) |
3280 	       PFINT_SB_CTL_CAUSE_ENA_M);
3281 	wr32(hw, PFINT_SB_CTL, val);
3282 
3283 	ice_flush(hw);
3284 }
3285 
3286 /**
3287  * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events
3288  * @pf: board private structure
3289  *
3290  * This sets up the handler for MSIX 0, which is used to manage the
3291  * non-queue interrupts, e.g. AdminQ and errors. This is not used
3292  * when in MSI or Legacy interrupt mode.
3293  */
3294 static int ice_req_irq_msix_misc(struct ice_pf *pf)
3295 {
3296 	struct device *dev = ice_pf_to_dev(pf);
3297 	struct ice_hw *hw = &pf->hw;
3298 	int oicr_idx, err = 0;
3299 
3300 	if (!pf->int_name[0])
3301 		snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc",
3302 			 dev_driver_string(dev), dev_name(dev));
3303 
3304 	/* Do not request IRQ but do enable OICR interrupt since settings are
3305 	 * lost during reset. Note that this function is called only during
3306 	 * rebuild path and not while reset is in progress.
3307 	 */
3308 	if (ice_is_reset_in_progress(pf->state))
3309 		goto skip_req_irq;
3310 
3311 	/* reserve one vector in irq_tracker for misc interrupts */
3312 	oicr_idx = ice_get_res(pf, pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID);
3313 	if (oicr_idx < 0)
3314 		return oicr_idx;
3315 
3316 	pf->num_avail_sw_msix -= 1;
3317 	pf->oicr_idx = (u16)oicr_idx;
3318 
3319 	err = devm_request_threaded_irq(dev,
3320 					pf->msix_entries[pf->oicr_idx].vector,
3321 					ice_misc_intr, ice_misc_intr_thread_fn,
3322 					0, pf->int_name, pf);
3323 	if (err) {
3324 		dev_err(dev, "devm_request_threaded_irq for %s failed: %d\n",
3325 			pf->int_name, err);
3326 		ice_free_res(pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID);
3327 		pf->num_avail_sw_msix += 1;
3328 		return err;
3329 	}
3330 
3331 skip_req_irq:
3332 	ice_ena_misc_vector(pf);
3333 
3334 	ice_ena_ctrlq_interrupts(hw, pf->oicr_idx);
3335 	wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->oicr_idx),
3336 	     ITR_REG_ALIGN(ICE_ITR_8K) >> ICE_ITR_GRAN_S);
3337 
3338 	ice_flush(hw);
3339 	ice_irq_dynamic_ena(hw, NULL, NULL);
3340 
3341 	return 0;
3342 }
3343 
3344 /**
3345  * ice_napi_add - register NAPI handler for the VSI
3346  * @vsi: VSI for which NAPI handler is to be registered
3347  *
3348  * This function is only called in the driver's load path. Registering the NAPI
3349  * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume,
3350  * reset/rebuild, etc.)
3351  */
3352 static void ice_napi_add(struct ice_vsi *vsi)
3353 {
3354 	int v_idx;
3355 
3356 	if (!vsi->netdev)
3357 		return;
3358 
3359 	ice_for_each_q_vector(vsi, v_idx)
3360 		netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi,
3361 			       ice_napi_poll);
3362 }
3363 
3364 /**
3365  * ice_set_ops - set netdev and ethtools ops for the given netdev
3366  * @vsi: the VSI associated with the new netdev
3367  */
3368 static void ice_set_ops(struct ice_vsi *vsi)
3369 {
3370 	struct net_device *netdev = vsi->netdev;
3371 	struct ice_pf *pf = ice_netdev_to_pf(netdev);
3372 
3373 	if (ice_is_safe_mode(pf)) {
3374 		netdev->netdev_ops = &ice_netdev_safe_mode_ops;
3375 		ice_set_ethtool_safe_mode_ops(netdev);
3376 		return;
3377 	}
3378 
3379 	netdev->netdev_ops = &ice_netdev_ops;
3380 	netdev->udp_tunnel_nic_info = &pf->hw.udp_tunnel_nic;
3381 	ice_set_ethtool_ops(netdev);
3382 
3383 	if (vsi->type != ICE_VSI_PF)
3384 		return;
3385 
3386 	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
3387 			       NETDEV_XDP_ACT_XSK_ZEROCOPY |
3388 			       NETDEV_XDP_ACT_RX_SG;
3389 }
3390 
3391 /**
3392  * ice_set_netdev_features - set features for the given netdev
3393  * @netdev: netdev instance
3394  */
3395 static void ice_set_netdev_features(struct net_device *netdev)
3396 {
3397 	struct ice_pf *pf = ice_netdev_to_pf(netdev);
3398 	bool is_dvm_ena = ice_is_dvm_ena(&pf->hw);
3399 	netdev_features_t csumo_features;
3400 	netdev_features_t vlano_features;
3401 	netdev_features_t dflt_features;
3402 	netdev_features_t tso_features;
3403 
3404 	if (ice_is_safe_mode(pf)) {
3405 		/* safe mode */
3406 		netdev->features = NETIF_F_SG | NETIF_F_HIGHDMA;
3407 		netdev->hw_features = netdev->features;
3408 		return;
3409 	}
3410 
3411 	dflt_features = NETIF_F_SG	|
3412 			NETIF_F_HIGHDMA	|
3413 			NETIF_F_NTUPLE	|
3414 			NETIF_F_RXHASH;
3415 
3416 	csumo_features = NETIF_F_RXCSUM	  |
3417 			 NETIF_F_IP_CSUM  |
3418 			 NETIF_F_SCTP_CRC |
3419 			 NETIF_F_IPV6_CSUM;
3420 
3421 	vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER |
3422 			 NETIF_F_HW_VLAN_CTAG_TX     |
3423 			 NETIF_F_HW_VLAN_CTAG_RX;
3424 
3425 	/* Enable CTAG/STAG filtering by default in Double VLAN Mode (DVM) */
3426 	if (is_dvm_ena)
3427 		vlano_features |= NETIF_F_HW_VLAN_STAG_FILTER;
3428 
3429 	tso_features = NETIF_F_TSO			|
3430 		       NETIF_F_TSO_ECN			|
3431 		       NETIF_F_TSO6			|
3432 		       NETIF_F_GSO_GRE			|
3433 		       NETIF_F_GSO_UDP_TUNNEL		|
3434 		       NETIF_F_GSO_GRE_CSUM		|
3435 		       NETIF_F_GSO_UDP_TUNNEL_CSUM	|
3436 		       NETIF_F_GSO_PARTIAL		|
3437 		       NETIF_F_GSO_IPXIP4		|
3438 		       NETIF_F_GSO_IPXIP6		|
3439 		       NETIF_F_GSO_UDP_L4;
3440 
3441 	netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM |
3442 					NETIF_F_GSO_GRE_CSUM;
3443 	/* set features that user can change */
3444 	netdev->hw_features = dflt_features | csumo_features |
3445 			      vlano_features | tso_features;
3446 
3447 	/* add support for HW_CSUM on packets with MPLS header */
3448 	netdev->mpls_features =  NETIF_F_HW_CSUM |
3449 				 NETIF_F_TSO     |
3450 				 NETIF_F_TSO6;
3451 
3452 	/* enable features */
3453 	netdev->features |= netdev->hw_features;
3454 
3455 	netdev->hw_features |= NETIF_F_HW_TC;
3456 	netdev->hw_features |= NETIF_F_LOOPBACK;
3457 
3458 	/* encap and VLAN devices inherit default, csumo and tso features */
3459 	netdev->hw_enc_features |= dflt_features | csumo_features |
3460 				   tso_features;
3461 	netdev->vlan_features |= dflt_features | csumo_features |
3462 				 tso_features;
3463 
3464 	/* advertise support but don't enable by default since only one type of
3465 	 * VLAN offload can be enabled at a time (i.e. CTAG or STAG). When one
3466 	 * type turns on the other has to be turned off. This is enforced by the
3467 	 * ice_fix_features() ndo callback.
3468 	 */
3469 	if (is_dvm_ena)
3470 		netdev->hw_features |= NETIF_F_HW_VLAN_STAG_RX |
3471 			NETIF_F_HW_VLAN_STAG_TX;
3472 
3473 	/* Leave CRC / FCS stripping enabled by default, but allow the value to
3474 	 * be changed at runtime
3475 	 */
3476 	netdev->hw_features |= NETIF_F_RXFCS;
3477 
3478 	netif_set_tso_max_size(netdev, ICE_MAX_TSO_SIZE);
3479 }
3480 
3481 /**
3482  * ice_fill_rss_lut - Fill the RSS lookup table with default values
3483  * @lut: Lookup table
3484  * @rss_table_size: Lookup table size
3485  * @rss_size: Range of queue number for hashing
3486  */
3487 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size)
3488 {
3489 	u16 i;
3490 
3491 	for (i = 0; i < rss_table_size; i++)
3492 		lut[i] = i % rss_size;
3493 }
3494 
3495 /**
3496  * ice_pf_vsi_setup - Set up a PF VSI
3497  * @pf: board private structure
3498  * @pi: pointer to the port_info instance
3499  *
3500  * Returns pointer to the successfully allocated VSI software struct
3501  * on success, otherwise returns NULL on failure.
3502  */
3503 static struct ice_vsi *
3504 ice_pf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi)
3505 {
3506 	struct ice_vsi_cfg_params params = {};
3507 
3508 	params.type = ICE_VSI_PF;
3509 	params.pi = pi;
3510 	params.flags = ICE_VSI_FLAG_INIT;
3511 
3512 	return ice_vsi_setup(pf, &params);
3513 }
3514 
3515 static struct ice_vsi *
3516 ice_chnl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi,
3517 		   struct ice_channel *ch)
3518 {
3519 	struct ice_vsi_cfg_params params = {};
3520 
3521 	params.type = ICE_VSI_CHNL;
3522 	params.pi = pi;
3523 	params.ch = ch;
3524 	params.flags = ICE_VSI_FLAG_INIT;
3525 
3526 	return ice_vsi_setup(pf, &params);
3527 }
3528 
3529 /**
3530  * ice_ctrl_vsi_setup - Set up a control VSI
3531  * @pf: board private structure
3532  * @pi: pointer to the port_info instance
3533  *
3534  * Returns pointer to the successfully allocated VSI software struct
3535  * on success, otherwise returns NULL on failure.
3536  */
3537 static struct ice_vsi *
3538 ice_ctrl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi)
3539 {
3540 	struct ice_vsi_cfg_params params = {};
3541 
3542 	params.type = ICE_VSI_CTRL;
3543 	params.pi = pi;
3544 	params.flags = ICE_VSI_FLAG_INIT;
3545 
3546 	return ice_vsi_setup(pf, &params);
3547 }
3548 
3549 /**
3550  * ice_lb_vsi_setup - Set up a loopback VSI
3551  * @pf: board private structure
3552  * @pi: pointer to the port_info instance
3553  *
3554  * Returns pointer to the successfully allocated VSI software struct
3555  * on success, otherwise returns NULL on failure.
3556  */
3557 struct ice_vsi *
3558 ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi)
3559 {
3560 	struct ice_vsi_cfg_params params = {};
3561 
3562 	params.type = ICE_VSI_LB;
3563 	params.pi = pi;
3564 	params.flags = ICE_VSI_FLAG_INIT;
3565 
3566 	return ice_vsi_setup(pf, &params);
3567 }
3568 
3569 /**
3570  * ice_vlan_rx_add_vid - Add a VLAN ID filter to HW offload
3571  * @netdev: network interface to be adjusted
3572  * @proto: VLAN TPID
3573  * @vid: VLAN ID to be added
3574  *
3575  * net_device_ops implementation for adding VLAN IDs
3576  */
3577 static int
3578 ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
3579 {
3580 	struct ice_netdev_priv *np = netdev_priv(netdev);
3581 	struct ice_vsi_vlan_ops *vlan_ops;
3582 	struct ice_vsi *vsi = np->vsi;
3583 	struct ice_vlan vlan;
3584 	int ret;
3585 
3586 	/* VLAN 0 is added by default during load/reset */
3587 	if (!vid)
3588 		return 0;
3589 
3590 	while (test_and_set_bit(ICE_CFG_BUSY, vsi->state))
3591 		usleep_range(1000, 2000);
3592 
3593 	/* Add multicast promisc rule for the VLAN ID to be added if
3594 	 * all-multicast is currently enabled.
3595 	 */
3596 	if (vsi->current_netdev_flags & IFF_ALLMULTI) {
3597 		ret = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
3598 					       ICE_MCAST_VLAN_PROMISC_BITS,
3599 					       vid);
3600 		if (ret)
3601 			goto finish;
3602 	}
3603 
3604 	vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
3605 
3606 	/* Add a switch rule for this VLAN ID so its corresponding VLAN tagged
3607 	 * packets aren't pruned by the device's internal switch on Rx
3608 	 */
3609 	vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0);
3610 	ret = vlan_ops->add_vlan(vsi, &vlan);
3611 	if (ret)
3612 		goto finish;
3613 
3614 	/* If all-multicast is currently enabled and this VLAN ID is only one
3615 	 * besides VLAN-0 we have to update look-up type of multicast promisc
3616 	 * rule for VLAN-0 from ICE_SW_LKUP_PROMISC to ICE_SW_LKUP_PROMISC_VLAN.
3617 	 */
3618 	if ((vsi->current_netdev_flags & IFF_ALLMULTI) &&
3619 	    ice_vsi_num_non_zero_vlans(vsi) == 1) {
3620 		ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
3621 					   ICE_MCAST_PROMISC_BITS, 0);
3622 		ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
3623 					 ICE_MCAST_VLAN_PROMISC_BITS, 0);
3624 	}
3625 
3626 finish:
3627 	clear_bit(ICE_CFG_BUSY, vsi->state);
3628 
3629 	return ret;
3630 }
3631 
3632 /**
3633  * ice_vlan_rx_kill_vid - Remove a VLAN ID filter from HW offload
3634  * @netdev: network interface to be adjusted
3635  * @proto: VLAN TPID
3636  * @vid: VLAN ID to be removed
3637  *
3638  * net_device_ops implementation for removing VLAN IDs
3639  */
3640 static int
3641 ice_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
3642 {
3643 	struct ice_netdev_priv *np = netdev_priv(netdev);
3644 	struct ice_vsi_vlan_ops *vlan_ops;
3645 	struct ice_vsi *vsi = np->vsi;
3646 	struct ice_vlan vlan;
3647 	int ret;
3648 
3649 	/* don't allow removal of VLAN 0 */
3650 	if (!vid)
3651 		return 0;
3652 
3653 	while (test_and_set_bit(ICE_CFG_BUSY, vsi->state))
3654 		usleep_range(1000, 2000);
3655 
3656 	ret = ice_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
3657 				    ICE_MCAST_VLAN_PROMISC_BITS, vid);
3658 	if (ret) {
3659 		netdev_err(netdev, "Error clearing multicast promiscuous mode on VSI %i\n",
3660 			   vsi->vsi_num);
3661 		vsi->current_netdev_flags |= IFF_ALLMULTI;
3662 	}
3663 
3664 	vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
3665 
3666 	/* Make sure VLAN delete is successful before updating VLAN
3667 	 * information
3668 	 */
3669 	vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0);
3670 	ret = vlan_ops->del_vlan(vsi, &vlan);
3671 	if (ret)
3672 		goto finish;
3673 
3674 	/* Remove multicast promisc rule for the removed VLAN ID if
3675 	 * all-multicast is enabled.
3676 	 */
3677 	if (vsi->current_netdev_flags & IFF_ALLMULTI)
3678 		ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
3679 					   ICE_MCAST_VLAN_PROMISC_BITS, vid);
3680 
3681 	if (!ice_vsi_has_non_zero_vlans(vsi)) {
3682 		/* Update look-up type of multicast promisc rule for VLAN 0
3683 		 * from ICE_SW_LKUP_PROMISC_VLAN to ICE_SW_LKUP_PROMISC when
3684 		 * all-multicast is enabled and VLAN 0 is the only VLAN rule.
3685 		 */
3686 		if (vsi->current_netdev_flags & IFF_ALLMULTI) {
3687 			ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
3688 						   ICE_MCAST_VLAN_PROMISC_BITS,
3689 						   0);
3690 			ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
3691 						 ICE_MCAST_PROMISC_BITS, 0);
3692 		}
3693 	}
3694 
3695 finish:
3696 	clear_bit(ICE_CFG_BUSY, vsi->state);
3697 
3698 	return ret;
3699 }
3700 
3701 /**
3702  * ice_rep_indr_tc_block_unbind
3703  * @cb_priv: indirection block private data
3704  */
3705 static void ice_rep_indr_tc_block_unbind(void *cb_priv)
3706 {
3707 	struct ice_indr_block_priv *indr_priv = cb_priv;
3708 
3709 	list_del(&indr_priv->list);
3710 	kfree(indr_priv);
3711 }
3712 
3713 /**
3714  * ice_tc_indir_block_unregister - Unregister TC indirect block notifications
3715  * @vsi: VSI struct which has the netdev
3716  */
3717 static void ice_tc_indir_block_unregister(struct ice_vsi *vsi)
3718 {
3719 	struct ice_netdev_priv *np = netdev_priv(vsi->netdev);
3720 
3721 	flow_indr_dev_unregister(ice_indr_setup_tc_cb, np,
3722 				 ice_rep_indr_tc_block_unbind);
3723 }
3724 
3725 /**
3726  * ice_tc_indir_block_register - Register TC indirect block notifications
3727  * @vsi: VSI struct which has the netdev
3728  *
3729  * Returns 0 on success, negative value on failure
3730  */
3731 static int ice_tc_indir_block_register(struct ice_vsi *vsi)
3732 {
3733 	struct ice_netdev_priv *np;
3734 
3735 	if (!vsi || !vsi->netdev)
3736 		return -EINVAL;
3737 
3738 	np = netdev_priv(vsi->netdev);
3739 
3740 	INIT_LIST_HEAD(&np->tc_indr_block_priv_list);
3741 	return flow_indr_dev_register(ice_indr_setup_tc_cb, np);
3742 }
3743 
3744 /**
3745  * ice_get_avail_q_count - Get count of queues in use
3746  * @pf_qmap: bitmap to get queue use count from
3747  * @lock: pointer to a mutex that protects access to pf_qmap
3748  * @size: size of the bitmap
3749  */
3750 static u16
3751 ice_get_avail_q_count(unsigned long *pf_qmap, struct mutex *lock, u16 size)
3752 {
3753 	unsigned long bit;
3754 	u16 count = 0;
3755 
3756 	mutex_lock(lock);
3757 	for_each_clear_bit(bit, pf_qmap, size)
3758 		count++;
3759 	mutex_unlock(lock);
3760 
3761 	return count;
3762 }
3763 
3764 /**
3765  * ice_get_avail_txq_count - Get count of Tx queues in use
3766  * @pf: pointer to an ice_pf instance
3767  */
3768 u16 ice_get_avail_txq_count(struct ice_pf *pf)
3769 {
3770 	return ice_get_avail_q_count(pf->avail_txqs, &pf->avail_q_mutex,
3771 				     pf->max_pf_txqs);
3772 }
3773 
3774 /**
3775  * ice_get_avail_rxq_count - Get count of Rx queues in use
3776  * @pf: pointer to an ice_pf instance
3777  */
3778 u16 ice_get_avail_rxq_count(struct ice_pf *pf)
3779 {
3780 	return ice_get_avail_q_count(pf->avail_rxqs, &pf->avail_q_mutex,
3781 				     pf->max_pf_rxqs);
3782 }
3783 
3784 /**
3785  * ice_deinit_pf - Unrolls initialziations done by ice_init_pf
3786  * @pf: board private structure to initialize
3787  */
3788 static void ice_deinit_pf(struct ice_pf *pf)
3789 {
3790 	ice_service_task_stop(pf);
3791 	mutex_destroy(&pf->adev_mutex);
3792 	mutex_destroy(&pf->sw_mutex);
3793 	mutex_destroy(&pf->tc_mutex);
3794 	mutex_destroy(&pf->avail_q_mutex);
3795 	mutex_destroy(&pf->vfs.table_lock);
3796 
3797 	if (pf->avail_txqs) {
3798 		bitmap_free(pf->avail_txqs);
3799 		pf->avail_txqs = NULL;
3800 	}
3801 
3802 	if (pf->avail_rxqs) {
3803 		bitmap_free(pf->avail_rxqs);
3804 		pf->avail_rxqs = NULL;
3805 	}
3806 
3807 	if (pf->ptp.clock)
3808 		ptp_clock_unregister(pf->ptp.clock);
3809 }
3810 
3811 /**
3812  * ice_set_pf_caps - set PFs capability flags
3813  * @pf: pointer to the PF instance
3814  */
3815 static void ice_set_pf_caps(struct ice_pf *pf)
3816 {
3817 	struct ice_hw_func_caps *func_caps = &pf->hw.func_caps;
3818 
3819 	clear_bit(ICE_FLAG_RDMA_ENA, pf->flags);
3820 	if (func_caps->common_cap.rdma)
3821 		set_bit(ICE_FLAG_RDMA_ENA, pf->flags);
3822 	clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags);
3823 	if (func_caps->common_cap.dcb)
3824 		set_bit(ICE_FLAG_DCB_CAPABLE, pf->flags);
3825 	clear_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags);
3826 	if (func_caps->common_cap.sr_iov_1_1) {
3827 		set_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags);
3828 		pf->vfs.num_supported = min_t(int, func_caps->num_allocd_vfs,
3829 					      ICE_MAX_SRIOV_VFS);
3830 	}
3831 	clear_bit(ICE_FLAG_RSS_ENA, pf->flags);
3832 	if (func_caps->common_cap.rss_table_size)
3833 		set_bit(ICE_FLAG_RSS_ENA, pf->flags);
3834 
3835 	clear_bit(ICE_FLAG_FD_ENA, pf->flags);
3836 	if (func_caps->fd_fltr_guar > 0 || func_caps->fd_fltr_best_effort > 0) {
3837 		u16 unused;
3838 
3839 		/* ctrl_vsi_idx will be set to a valid value when flow director
3840 		 * is setup by ice_init_fdir
3841 		 */
3842 		pf->ctrl_vsi_idx = ICE_NO_VSI;
3843 		set_bit(ICE_FLAG_FD_ENA, pf->flags);
3844 		/* force guaranteed filter pool for PF */
3845 		ice_alloc_fd_guar_item(&pf->hw, &unused,
3846 				       func_caps->fd_fltr_guar);
3847 		/* force shared filter pool for PF */
3848 		ice_alloc_fd_shrd_item(&pf->hw, &unused,
3849 				       func_caps->fd_fltr_best_effort);
3850 	}
3851 
3852 	clear_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags);
3853 	if (func_caps->common_cap.ieee_1588)
3854 		set_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags);
3855 
3856 	pf->max_pf_txqs = func_caps->common_cap.num_txq;
3857 	pf->max_pf_rxqs = func_caps->common_cap.num_rxq;
3858 }
3859 
3860 /**
3861  * ice_init_pf - Initialize general software structures (struct ice_pf)
3862  * @pf: board private structure to initialize
3863  */
3864 static int ice_init_pf(struct ice_pf *pf)
3865 {
3866 	ice_set_pf_caps(pf);
3867 
3868 	mutex_init(&pf->sw_mutex);
3869 	mutex_init(&pf->tc_mutex);
3870 	mutex_init(&pf->adev_mutex);
3871 
3872 	INIT_HLIST_HEAD(&pf->aq_wait_list);
3873 	spin_lock_init(&pf->aq_wait_lock);
3874 	init_waitqueue_head(&pf->aq_wait_queue);
3875 
3876 	init_waitqueue_head(&pf->reset_wait_queue);
3877 
3878 	/* setup service timer and periodic service task */
3879 	timer_setup(&pf->serv_tmr, ice_service_timer, 0);
3880 	pf->serv_tmr_period = HZ;
3881 	INIT_WORK(&pf->serv_task, ice_service_task);
3882 	clear_bit(ICE_SERVICE_SCHED, pf->state);
3883 
3884 	mutex_init(&pf->avail_q_mutex);
3885 	pf->avail_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL);
3886 	if (!pf->avail_txqs)
3887 		return -ENOMEM;
3888 
3889 	pf->avail_rxqs = bitmap_zalloc(pf->max_pf_rxqs, GFP_KERNEL);
3890 	if (!pf->avail_rxqs) {
3891 		bitmap_free(pf->avail_txqs);
3892 		pf->avail_txqs = NULL;
3893 		return -ENOMEM;
3894 	}
3895 
3896 	mutex_init(&pf->vfs.table_lock);
3897 	hash_init(pf->vfs.table);
3898 	ice_mbx_init_snapshot(&pf->hw);
3899 
3900 	return 0;
3901 }
3902 
3903 /**
3904  * ice_reduce_msix_usage - Reduce usage of MSI-X vectors
3905  * @pf: board private structure
3906  * @v_remain: number of remaining MSI-X vectors to be distributed
3907  *
3908  * Reduce the usage of MSI-X vectors when entire request cannot be fulfilled.
3909  * pf->num_lan_msix and pf->num_rdma_msix values are set based on number of
3910  * remaining vectors.
3911  */
3912 static void ice_reduce_msix_usage(struct ice_pf *pf, int v_remain)
3913 {
3914 	int v_rdma;
3915 
3916 	if (!ice_is_rdma_ena(pf)) {
3917 		pf->num_lan_msix = v_remain;
3918 		return;
3919 	}
3920 
3921 	/* RDMA needs at least 1 interrupt in addition to AEQ MSIX */
3922 	v_rdma = ICE_RDMA_NUM_AEQ_MSIX + 1;
3923 
3924 	if (v_remain < ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_RDMA_MSIX) {
3925 		dev_warn(ice_pf_to_dev(pf), "Not enough MSI-X vectors to support RDMA.\n");
3926 		clear_bit(ICE_FLAG_RDMA_ENA, pf->flags);
3927 
3928 		pf->num_rdma_msix = 0;
3929 		pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX;
3930 	} else if ((v_remain < ICE_MIN_LAN_TXRX_MSIX + v_rdma) ||
3931 		   (v_remain - v_rdma < v_rdma)) {
3932 		/* Support minimum RDMA and give remaining vectors to LAN MSIX */
3933 		pf->num_rdma_msix = ICE_MIN_RDMA_MSIX;
3934 		pf->num_lan_msix = v_remain - ICE_MIN_RDMA_MSIX;
3935 	} else {
3936 		/* Split remaining MSIX with RDMA after accounting for AEQ MSIX
3937 		 */
3938 		pf->num_rdma_msix = (v_remain - ICE_RDMA_NUM_AEQ_MSIX) / 2 +
3939 				    ICE_RDMA_NUM_AEQ_MSIX;
3940 		pf->num_lan_msix = v_remain - pf->num_rdma_msix;
3941 	}
3942 }
3943 
3944 /**
3945  * ice_ena_msix_range - Request a range of MSIX vectors from the OS
3946  * @pf: board private structure
3947  *
3948  * Compute the number of MSIX vectors wanted and request from the OS. Adjust
3949  * device usage if there are not enough vectors. Return the number of vectors
3950  * reserved or negative on failure.
3951  */
3952 static int ice_ena_msix_range(struct ice_pf *pf)
3953 {
3954 	int num_cpus, hw_num_msix, v_other, v_wanted, v_actual;
3955 	struct device *dev = ice_pf_to_dev(pf);
3956 	int err, i;
3957 
3958 	hw_num_msix = pf->hw.func_caps.common_cap.num_msix_vectors;
3959 	num_cpus = num_online_cpus();
3960 
3961 	/* LAN miscellaneous handler */
3962 	v_other = ICE_MIN_LAN_OICR_MSIX;
3963 
3964 	/* Flow Director */
3965 	if (test_bit(ICE_FLAG_FD_ENA, pf->flags))
3966 		v_other += ICE_FDIR_MSIX;
3967 
3968 	/* switchdev */
3969 	v_other += ICE_ESWITCH_MSIX;
3970 
3971 	v_wanted = v_other;
3972 
3973 	/* LAN traffic */
3974 	pf->num_lan_msix = num_cpus;
3975 	v_wanted += pf->num_lan_msix;
3976 
3977 	/* RDMA auxiliary driver */
3978 	if (ice_is_rdma_ena(pf)) {
3979 		pf->num_rdma_msix = num_cpus + ICE_RDMA_NUM_AEQ_MSIX;
3980 		v_wanted += pf->num_rdma_msix;
3981 	}
3982 
3983 	if (v_wanted > hw_num_msix) {
3984 		int v_remain;
3985 
3986 		dev_warn(dev, "not enough device MSI-X vectors. wanted = %d, available = %d\n",
3987 			 v_wanted, hw_num_msix);
3988 
3989 		if (hw_num_msix < ICE_MIN_MSIX) {
3990 			err = -ERANGE;
3991 			goto exit_err;
3992 		}
3993 
3994 		v_remain = hw_num_msix - v_other;
3995 		if (v_remain < ICE_MIN_LAN_TXRX_MSIX) {
3996 			v_other = ICE_MIN_MSIX - ICE_MIN_LAN_TXRX_MSIX;
3997 			v_remain = ICE_MIN_LAN_TXRX_MSIX;
3998 		}
3999 
4000 		ice_reduce_msix_usage(pf, v_remain);
4001 		v_wanted = pf->num_lan_msix + pf->num_rdma_msix + v_other;
4002 
4003 		dev_notice(dev, "Reducing request to %d MSI-X vectors for LAN traffic.\n",
4004 			   pf->num_lan_msix);
4005 		if (ice_is_rdma_ena(pf))
4006 			dev_notice(dev, "Reducing request to %d MSI-X vectors for RDMA.\n",
4007 				   pf->num_rdma_msix);
4008 	}
4009 
4010 	pf->msix_entries = devm_kcalloc(dev, v_wanted,
4011 					sizeof(*pf->msix_entries), GFP_KERNEL);
4012 	if (!pf->msix_entries) {
4013 		err = -ENOMEM;
4014 		goto exit_err;
4015 	}
4016 
4017 	for (i = 0; i < v_wanted; i++)
4018 		pf->msix_entries[i].entry = i;
4019 
4020 	/* actually reserve the vectors */
4021 	v_actual = pci_enable_msix_range(pf->pdev, pf->msix_entries,
4022 					 ICE_MIN_MSIX, v_wanted);
4023 	if (v_actual < 0) {
4024 		dev_err(dev, "unable to reserve MSI-X vectors\n");
4025 		err = v_actual;
4026 		goto msix_err;
4027 	}
4028 
4029 	if (v_actual < v_wanted) {
4030 		dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n",
4031 			 v_wanted, v_actual);
4032 
4033 		if (v_actual < ICE_MIN_MSIX) {
4034 			/* error if we can't get minimum vectors */
4035 			pci_disable_msix(pf->pdev);
4036 			err = -ERANGE;
4037 			goto msix_err;
4038 		} else {
4039 			int v_remain = v_actual - v_other;
4040 
4041 			if (v_remain < ICE_MIN_LAN_TXRX_MSIX)
4042 				v_remain = ICE_MIN_LAN_TXRX_MSIX;
4043 
4044 			ice_reduce_msix_usage(pf, v_remain);
4045 
4046 			dev_notice(dev, "Enabled %d MSI-X vectors for LAN traffic.\n",
4047 				   pf->num_lan_msix);
4048 
4049 			if (ice_is_rdma_ena(pf))
4050 				dev_notice(dev, "Enabled %d MSI-X vectors for RDMA.\n",
4051 					   pf->num_rdma_msix);
4052 		}
4053 	}
4054 
4055 	return v_actual;
4056 
4057 msix_err:
4058 	devm_kfree(dev, pf->msix_entries);
4059 
4060 exit_err:
4061 	pf->num_rdma_msix = 0;
4062 	pf->num_lan_msix = 0;
4063 	return err;
4064 }
4065 
4066 /**
4067  * ice_dis_msix - Disable MSI-X interrupt setup in OS
4068  * @pf: board private structure
4069  */
4070 static void ice_dis_msix(struct ice_pf *pf)
4071 {
4072 	pci_disable_msix(pf->pdev);
4073 	devm_kfree(ice_pf_to_dev(pf), pf->msix_entries);
4074 	pf->msix_entries = NULL;
4075 }
4076 
4077 /**
4078  * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme
4079  * @pf: board private structure
4080  */
4081 static void ice_clear_interrupt_scheme(struct ice_pf *pf)
4082 {
4083 	ice_dis_msix(pf);
4084 
4085 	if (pf->irq_tracker) {
4086 		devm_kfree(ice_pf_to_dev(pf), pf->irq_tracker);
4087 		pf->irq_tracker = NULL;
4088 	}
4089 }
4090 
4091 /**
4092  * ice_init_interrupt_scheme - Determine proper interrupt scheme
4093  * @pf: board private structure to initialize
4094  */
4095 static int ice_init_interrupt_scheme(struct ice_pf *pf)
4096 {
4097 	int vectors;
4098 
4099 	vectors = ice_ena_msix_range(pf);
4100 
4101 	if (vectors < 0)
4102 		return vectors;
4103 
4104 	/* set up vector assignment tracking */
4105 	pf->irq_tracker = devm_kzalloc(ice_pf_to_dev(pf),
4106 				       struct_size(pf->irq_tracker, list, vectors),
4107 				       GFP_KERNEL);
4108 	if (!pf->irq_tracker) {
4109 		ice_dis_msix(pf);
4110 		return -ENOMEM;
4111 	}
4112 
4113 	/* populate SW interrupts pool with number of OS granted IRQs. */
4114 	pf->num_avail_sw_msix = (u16)vectors;
4115 	pf->irq_tracker->num_entries = (u16)vectors;
4116 	pf->irq_tracker->end = pf->irq_tracker->num_entries;
4117 
4118 	return 0;
4119 }
4120 
4121 /**
4122  * ice_is_wol_supported - check if WoL is supported
4123  * @hw: pointer to hardware info
4124  *
4125  * Check if WoL is supported based on the HW configuration.
4126  * Returns true if NVM supports and enables WoL for this port, false otherwise
4127  */
4128 bool ice_is_wol_supported(struct ice_hw *hw)
4129 {
4130 	u16 wol_ctrl;
4131 
4132 	/* A bit set to 1 in the NVM Software Reserved Word 2 (WoL control
4133 	 * word) indicates WoL is not supported on the corresponding PF ID.
4134 	 */
4135 	if (ice_read_sr_word(hw, ICE_SR_NVM_WOL_CFG, &wol_ctrl))
4136 		return false;
4137 
4138 	return !(BIT(hw->port_info->lport) & wol_ctrl);
4139 }
4140 
4141 /**
4142  * ice_vsi_recfg_qs - Change the number of queues on a VSI
4143  * @vsi: VSI being changed
4144  * @new_rx: new number of Rx queues
4145  * @new_tx: new number of Tx queues
4146  * @locked: is adev device_lock held
4147  *
4148  * Only change the number of queues if new_tx, or new_rx is non-0.
4149  *
4150  * Returns 0 on success.
4151  */
4152 int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx, bool locked)
4153 {
4154 	struct ice_pf *pf = vsi->back;
4155 	int err = 0, timeout = 50;
4156 
4157 	if (!new_rx && !new_tx)
4158 		return -EINVAL;
4159 
4160 	while (test_and_set_bit(ICE_CFG_BUSY, pf->state)) {
4161 		timeout--;
4162 		if (!timeout)
4163 			return -EBUSY;
4164 		usleep_range(1000, 2000);
4165 	}
4166 
4167 	if (new_tx)
4168 		vsi->req_txq = (u16)new_tx;
4169 	if (new_rx)
4170 		vsi->req_rxq = (u16)new_rx;
4171 
4172 	/* set for the next time the netdev is started */
4173 	if (!netif_running(vsi->netdev)) {
4174 		ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT);
4175 		dev_dbg(ice_pf_to_dev(pf), "Link is down, queue count change happens when link is brought up\n");
4176 		goto done;
4177 	}
4178 
4179 	ice_vsi_close(vsi);
4180 	ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT);
4181 	ice_pf_dcb_recfg(pf, locked);
4182 	ice_vsi_open(vsi);
4183 done:
4184 	clear_bit(ICE_CFG_BUSY, pf->state);
4185 	return err;
4186 }
4187 
4188 /**
4189  * ice_set_safe_mode_vlan_cfg - configure PF VSI to allow all VLANs in safe mode
4190  * @pf: PF to configure
4191  *
4192  * No VLAN offloads/filtering are advertised in safe mode so make sure the PF
4193  * VSI can still Tx/Rx VLAN tagged packets.
4194  */
4195 static void ice_set_safe_mode_vlan_cfg(struct ice_pf *pf)
4196 {
4197 	struct ice_vsi *vsi = ice_get_main_vsi(pf);
4198 	struct ice_vsi_ctx *ctxt;
4199 	struct ice_hw *hw;
4200 	int status;
4201 
4202 	if (!vsi)
4203 		return;
4204 
4205 	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
4206 	if (!ctxt)
4207 		return;
4208 
4209 	hw = &pf->hw;
4210 	ctxt->info = vsi->info;
4211 
4212 	ctxt->info.valid_sections =
4213 		cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID |
4214 			    ICE_AQ_VSI_PROP_SECURITY_VALID |
4215 			    ICE_AQ_VSI_PROP_SW_VALID);
4216 
4217 	/* disable VLAN anti-spoof */
4218 	ctxt->info.sec_flags &= ~(ICE_AQ_VSI_SEC_TX_VLAN_PRUNE_ENA <<
4219 				  ICE_AQ_VSI_SEC_TX_PRUNE_ENA_S);
4220 
4221 	/* disable VLAN pruning and keep all other settings */
4222 	ctxt->info.sw_flags2 &= ~ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA;
4223 
4224 	/* allow all VLANs on Tx and don't strip on Rx */
4225 	ctxt->info.inner_vlan_flags = ICE_AQ_VSI_INNER_VLAN_TX_MODE_ALL |
4226 		ICE_AQ_VSI_INNER_VLAN_EMODE_NOTHING;
4227 
4228 	status = ice_update_vsi(hw, vsi->idx, ctxt, NULL);
4229 	if (status) {
4230 		dev_err(ice_pf_to_dev(vsi->back), "Failed to update VSI for safe mode VLANs, err %d aq_err %s\n",
4231 			status, ice_aq_str(hw->adminq.sq_last_status));
4232 	} else {
4233 		vsi->info.sec_flags = ctxt->info.sec_flags;
4234 		vsi->info.sw_flags2 = ctxt->info.sw_flags2;
4235 		vsi->info.inner_vlan_flags = ctxt->info.inner_vlan_flags;
4236 	}
4237 
4238 	kfree(ctxt);
4239 }
4240 
4241 /**
4242  * ice_log_pkg_init - log result of DDP package load
4243  * @hw: pointer to hardware info
4244  * @state: state of package load
4245  */
4246 static void ice_log_pkg_init(struct ice_hw *hw, enum ice_ddp_state state)
4247 {
4248 	struct ice_pf *pf = hw->back;
4249 	struct device *dev;
4250 
4251 	dev = ice_pf_to_dev(pf);
4252 
4253 	switch (state) {
4254 	case ICE_DDP_PKG_SUCCESS:
4255 		dev_info(dev, "The DDP package was successfully loaded: %s version %d.%d.%d.%d\n",
4256 			 hw->active_pkg_name,
4257 			 hw->active_pkg_ver.major,
4258 			 hw->active_pkg_ver.minor,
4259 			 hw->active_pkg_ver.update,
4260 			 hw->active_pkg_ver.draft);
4261 		break;
4262 	case ICE_DDP_PKG_SAME_VERSION_ALREADY_LOADED:
4263 		dev_info(dev, "DDP package already present on device: %s version %d.%d.%d.%d\n",
4264 			 hw->active_pkg_name,
4265 			 hw->active_pkg_ver.major,
4266 			 hw->active_pkg_ver.minor,
4267 			 hw->active_pkg_ver.update,
4268 			 hw->active_pkg_ver.draft);
4269 		break;
4270 	case ICE_DDP_PKG_ALREADY_LOADED_NOT_SUPPORTED:
4271 		dev_err(dev, "The device has a DDP package that is not supported by the driver.  The device has package '%s' version %d.%d.x.x.  The driver requires version %d.%d.x.x.  Entering Safe Mode.\n",
4272 			hw->active_pkg_name,
4273 			hw->active_pkg_ver.major,
4274 			hw->active_pkg_ver.minor,
4275 			ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
4276 		break;
4277 	case ICE_DDP_PKG_COMPATIBLE_ALREADY_LOADED:
4278 		dev_info(dev, "The driver could not load the DDP package file because a compatible DDP package is already present on the device.  The device has package '%s' version %d.%d.%d.%d.  The package file found by the driver: '%s' version %d.%d.%d.%d.\n",
4279 			 hw->active_pkg_name,
4280 			 hw->active_pkg_ver.major,
4281 			 hw->active_pkg_ver.minor,
4282 			 hw->active_pkg_ver.update,
4283 			 hw->active_pkg_ver.draft,
4284 			 hw->pkg_name,
4285 			 hw->pkg_ver.major,
4286 			 hw->pkg_ver.minor,
4287 			 hw->pkg_ver.update,
4288 			 hw->pkg_ver.draft);
4289 		break;
4290 	case ICE_DDP_PKG_FW_MISMATCH:
4291 		dev_err(dev, "The firmware loaded on the device is not compatible with the DDP package.  Please update the device's NVM.  Entering safe mode.\n");
4292 		break;
4293 	case ICE_DDP_PKG_INVALID_FILE:
4294 		dev_err(dev, "The DDP package file is invalid. Entering Safe Mode.\n");
4295 		break;
4296 	case ICE_DDP_PKG_FILE_VERSION_TOO_HIGH:
4297 		dev_err(dev, "The DDP package file version is higher than the driver supports.  Please use an updated driver.  Entering Safe Mode.\n");
4298 		break;
4299 	case ICE_DDP_PKG_FILE_VERSION_TOO_LOW:
4300 		dev_err(dev, "The DDP package file version is lower than the driver supports.  The driver requires version %d.%d.x.x.  Please use an updated DDP Package file.  Entering Safe Mode.\n",
4301 			ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
4302 		break;
4303 	case ICE_DDP_PKG_FILE_SIGNATURE_INVALID:
4304 		dev_err(dev, "The DDP package could not be loaded because its signature is not valid.  Please use a valid DDP Package.  Entering Safe Mode.\n");
4305 		break;
4306 	case ICE_DDP_PKG_FILE_REVISION_TOO_LOW:
4307 		dev_err(dev, "The DDP Package could not be loaded because its security revision is too low.  Please use an updated DDP Package.  Entering Safe Mode.\n");
4308 		break;
4309 	case ICE_DDP_PKG_LOAD_ERROR:
4310 		dev_err(dev, "An error occurred on the device while loading the DDP package.  The device will be reset.\n");
4311 		/* poll for reset to complete */
4312 		if (ice_check_reset(hw))
4313 			dev_err(dev, "Error resetting device. Please reload the driver\n");
4314 		break;
4315 	case ICE_DDP_PKG_ERR:
4316 	default:
4317 		dev_err(dev, "An unknown error occurred when loading the DDP package.  Entering Safe Mode.\n");
4318 		break;
4319 	}
4320 }
4321 
4322 /**
4323  * ice_load_pkg - load/reload the DDP Package file
4324  * @firmware: firmware structure when firmware requested or NULL for reload
4325  * @pf: pointer to the PF instance
4326  *
4327  * Called on probe and post CORER/GLOBR rebuild to load DDP Package and
4328  * initialize HW tables.
4329  */
4330 static void
4331 ice_load_pkg(const struct firmware *firmware, struct ice_pf *pf)
4332 {
4333 	enum ice_ddp_state state = ICE_DDP_PKG_ERR;
4334 	struct device *dev = ice_pf_to_dev(pf);
4335 	struct ice_hw *hw = &pf->hw;
4336 
4337 	/* Load DDP Package */
4338 	if (firmware && !hw->pkg_copy) {
4339 		state = ice_copy_and_init_pkg(hw, firmware->data,
4340 					      firmware->size);
4341 		ice_log_pkg_init(hw, state);
4342 	} else if (!firmware && hw->pkg_copy) {
4343 		/* Reload package during rebuild after CORER/GLOBR reset */
4344 		state = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size);
4345 		ice_log_pkg_init(hw, state);
4346 	} else {
4347 		dev_err(dev, "The DDP package file failed to load. Entering Safe Mode.\n");
4348 	}
4349 
4350 	if (!ice_is_init_pkg_successful(state)) {
4351 		/* Safe Mode */
4352 		clear_bit(ICE_FLAG_ADV_FEATURES, pf->flags);
4353 		return;
4354 	}
4355 
4356 	/* Successful download package is the precondition for advanced
4357 	 * features, hence setting the ICE_FLAG_ADV_FEATURES flag
4358 	 */
4359 	set_bit(ICE_FLAG_ADV_FEATURES, pf->flags);
4360 }
4361 
4362 /**
4363  * ice_verify_cacheline_size - verify driver's assumption of 64 Byte cache lines
4364  * @pf: pointer to the PF structure
4365  *
4366  * There is no error returned here because the driver should be able to handle
4367  * 128 Byte cache lines, so we only print a warning in case issues are seen,
4368  * specifically with Tx.
4369  */
4370 static void ice_verify_cacheline_size(struct ice_pf *pf)
4371 {
4372 	if (rd32(&pf->hw, GLPCI_CNF2) & GLPCI_CNF2_CACHELINE_SIZE_M)
4373 		dev_warn(ice_pf_to_dev(pf), "%d Byte cache line assumption is invalid, driver may have Tx timeouts!\n",
4374 			 ICE_CACHE_LINE_BYTES);
4375 }
4376 
4377 /**
4378  * ice_send_version - update firmware with driver version
4379  * @pf: PF struct
4380  *
4381  * Returns 0 on success, else error code
4382  */
4383 static int ice_send_version(struct ice_pf *pf)
4384 {
4385 	struct ice_driver_ver dv;
4386 
4387 	dv.major_ver = 0xff;
4388 	dv.minor_ver = 0xff;
4389 	dv.build_ver = 0xff;
4390 	dv.subbuild_ver = 0;
4391 	strscpy((char *)dv.driver_string, UTS_RELEASE,
4392 		sizeof(dv.driver_string));
4393 	return ice_aq_send_driver_ver(&pf->hw, &dv, NULL);
4394 }
4395 
4396 /**
4397  * ice_init_fdir - Initialize flow director VSI and configuration
4398  * @pf: pointer to the PF instance
4399  *
4400  * returns 0 on success, negative on error
4401  */
4402 static int ice_init_fdir(struct ice_pf *pf)
4403 {
4404 	struct device *dev = ice_pf_to_dev(pf);
4405 	struct ice_vsi *ctrl_vsi;
4406 	int err;
4407 
4408 	/* Side Band Flow Director needs to have a control VSI.
4409 	 * Allocate it and store it in the PF.
4410 	 */
4411 	ctrl_vsi = ice_ctrl_vsi_setup(pf, pf->hw.port_info);
4412 	if (!ctrl_vsi) {
4413 		dev_dbg(dev, "could not create control VSI\n");
4414 		return -ENOMEM;
4415 	}
4416 
4417 	err = ice_vsi_open_ctrl(ctrl_vsi);
4418 	if (err) {
4419 		dev_dbg(dev, "could not open control VSI\n");
4420 		goto err_vsi_open;
4421 	}
4422 
4423 	mutex_init(&pf->hw.fdir_fltr_lock);
4424 
4425 	err = ice_fdir_create_dflt_rules(pf);
4426 	if (err)
4427 		goto err_fdir_rule;
4428 
4429 	return 0;
4430 
4431 err_fdir_rule:
4432 	ice_fdir_release_flows(&pf->hw);
4433 	ice_vsi_close(ctrl_vsi);
4434 err_vsi_open:
4435 	ice_vsi_release(ctrl_vsi);
4436 	if (pf->ctrl_vsi_idx != ICE_NO_VSI) {
4437 		pf->vsi[pf->ctrl_vsi_idx] = NULL;
4438 		pf->ctrl_vsi_idx = ICE_NO_VSI;
4439 	}
4440 	return err;
4441 }
4442 
4443 static void ice_deinit_fdir(struct ice_pf *pf)
4444 {
4445 	struct ice_vsi *vsi = ice_get_ctrl_vsi(pf);
4446 
4447 	if (!vsi)
4448 		return;
4449 
4450 	ice_vsi_manage_fdir(vsi, false);
4451 	ice_vsi_release(vsi);
4452 	if (pf->ctrl_vsi_idx != ICE_NO_VSI) {
4453 		pf->vsi[pf->ctrl_vsi_idx] = NULL;
4454 		pf->ctrl_vsi_idx = ICE_NO_VSI;
4455 	}
4456 
4457 	mutex_destroy(&(&pf->hw)->fdir_fltr_lock);
4458 }
4459 
4460 /**
4461  * ice_get_opt_fw_name - return optional firmware file name or NULL
4462  * @pf: pointer to the PF instance
4463  */
4464 static char *ice_get_opt_fw_name(struct ice_pf *pf)
4465 {
4466 	/* Optional firmware name same as default with additional dash
4467 	 * followed by a EUI-64 identifier (PCIe Device Serial Number)
4468 	 */
4469 	struct pci_dev *pdev = pf->pdev;
4470 	char *opt_fw_filename;
4471 	u64 dsn;
4472 
4473 	/* Determine the name of the optional file using the DSN (two
4474 	 * dwords following the start of the DSN Capability).
4475 	 */
4476 	dsn = pci_get_dsn(pdev);
4477 	if (!dsn)
4478 		return NULL;
4479 
4480 	opt_fw_filename = kzalloc(NAME_MAX, GFP_KERNEL);
4481 	if (!opt_fw_filename)
4482 		return NULL;
4483 
4484 	snprintf(opt_fw_filename, NAME_MAX, "%sice-%016llx.pkg",
4485 		 ICE_DDP_PKG_PATH, dsn);
4486 
4487 	return opt_fw_filename;
4488 }
4489 
4490 /**
4491  * ice_request_fw - Device initialization routine
4492  * @pf: pointer to the PF instance
4493  */
4494 static void ice_request_fw(struct ice_pf *pf)
4495 {
4496 	char *opt_fw_filename = ice_get_opt_fw_name(pf);
4497 	const struct firmware *firmware = NULL;
4498 	struct device *dev = ice_pf_to_dev(pf);
4499 	int err = 0;
4500 
4501 	/* optional device-specific DDP (if present) overrides the default DDP
4502 	 * package file. kernel logs a debug message if the file doesn't exist,
4503 	 * and warning messages for other errors.
4504 	 */
4505 	if (opt_fw_filename) {
4506 		err = firmware_request_nowarn(&firmware, opt_fw_filename, dev);
4507 		if (err) {
4508 			kfree(opt_fw_filename);
4509 			goto dflt_pkg_load;
4510 		}
4511 
4512 		/* request for firmware was successful. Download to device */
4513 		ice_load_pkg(firmware, pf);
4514 		kfree(opt_fw_filename);
4515 		release_firmware(firmware);
4516 		return;
4517 	}
4518 
4519 dflt_pkg_load:
4520 	err = request_firmware(&firmware, ICE_DDP_PKG_FILE, dev);
4521 	if (err) {
4522 		dev_err(dev, "The DDP package file was not found or could not be read. Entering Safe Mode\n");
4523 		return;
4524 	}
4525 
4526 	/* request for firmware was successful. Download to device */
4527 	ice_load_pkg(firmware, pf);
4528 	release_firmware(firmware);
4529 }
4530 
4531 /**
4532  * ice_print_wake_reason - show the wake up cause in the log
4533  * @pf: pointer to the PF struct
4534  */
4535 static void ice_print_wake_reason(struct ice_pf *pf)
4536 {
4537 	u32 wus = pf->wakeup_reason;
4538 	const char *wake_str;
4539 
4540 	/* if no wake event, nothing to print */
4541 	if (!wus)
4542 		return;
4543 
4544 	if (wus & PFPM_WUS_LNKC_M)
4545 		wake_str = "Link\n";
4546 	else if (wus & PFPM_WUS_MAG_M)
4547 		wake_str = "Magic Packet\n";
4548 	else if (wus & PFPM_WUS_MNG_M)
4549 		wake_str = "Management\n";
4550 	else if (wus & PFPM_WUS_FW_RST_WK_M)
4551 		wake_str = "Firmware Reset\n";
4552 	else
4553 		wake_str = "Unknown\n";
4554 
4555 	dev_info(ice_pf_to_dev(pf), "Wake reason: %s", wake_str);
4556 }
4557 
4558 /**
4559  * ice_register_netdev - register netdev
4560  * @vsi: pointer to the VSI struct
4561  */
4562 static int ice_register_netdev(struct ice_vsi *vsi)
4563 {
4564 	int err;
4565 
4566 	if (!vsi || !vsi->netdev)
4567 		return -EIO;
4568 
4569 	err = register_netdev(vsi->netdev);
4570 	if (err)
4571 		return err;
4572 
4573 	set_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state);
4574 	netif_carrier_off(vsi->netdev);
4575 	netif_tx_stop_all_queues(vsi->netdev);
4576 
4577 	return 0;
4578 }
4579 
4580 static void ice_unregister_netdev(struct ice_vsi *vsi)
4581 {
4582 	if (!vsi || !vsi->netdev)
4583 		return;
4584 
4585 	unregister_netdev(vsi->netdev);
4586 	clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state);
4587 }
4588 
4589 /**
4590  * ice_cfg_netdev - Allocate, configure and register a netdev
4591  * @vsi: the VSI associated with the new netdev
4592  *
4593  * Returns 0 on success, negative value on failure
4594  */
4595 static int ice_cfg_netdev(struct ice_vsi *vsi)
4596 {
4597 	struct ice_netdev_priv *np;
4598 	struct net_device *netdev;
4599 	u8 mac_addr[ETH_ALEN];
4600 
4601 	netdev = alloc_etherdev_mqs(sizeof(*np), vsi->alloc_txq,
4602 				    vsi->alloc_rxq);
4603 	if (!netdev)
4604 		return -ENOMEM;
4605 
4606 	set_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state);
4607 	vsi->netdev = netdev;
4608 	np = netdev_priv(netdev);
4609 	np->vsi = vsi;
4610 
4611 	ice_set_netdev_features(netdev);
4612 	ice_set_ops(vsi);
4613 
4614 	if (vsi->type == ICE_VSI_PF) {
4615 		SET_NETDEV_DEV(netdev, ice_pf_to_dev(vsi->back));
4616 		ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr);
4617 		eth_hw_addr_set(netdev, mac_addr);
4618 	}
4619 
4620 	netdev->priv_flags |= IFF_UNICAST_FLT;
4621 
4622 	/* Setup netdev TC information */
4623 	ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc);
4624 
4625 	netdev->max_mtu = ICE_MAX_MTU;
4626 
4627 	return 0;
4628 }
4629 
4630 static void ice_decfg_netdev(struct ice_vsi *vsi)
4631 {
4632 	clear_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state);
4633 	free_netdev(vsi->netdev);
4634 	vsi->netdev = NULL;
4635 }
4636 
4637 static int ice_start_eth(struct ice_vsi *vsi)
4638 {
4639 	int err;
4640 
4641 	err = ice_init_mac_fltr(vsi->back);
4642 	if (err)
4643 		return err;
4644 
4645 	rtnl_lock();
4646 	err = ice_vsi_open(vsi);
4647 	rtnl_unlock();
4648 
4649 	return err;
4650 }
4651 
4652 static void ice_stop_eth(struct ice_vsi *vsi)
4653 {
4654 	ice_fltr_remove_all(vsi);
4655 	ice_vsi_close(vsi);
4656 }
4657 
4658 static int ice_init_eth(struct ice_pf *pf)
4659 {
4660 	struct ice_vsi *vsi = ice_get_main_vsi(pf);
4661 	int err;
4662 
4663 	if (!vsi)
4664 		return -EINVAL;
4665 
4666 	/* init channel list */
4667 	INIT_LIST_HEAD(&vsi->ch_list);
4668 
4669 	err = ice_cfg_netdev(vsi);
4670 	if (err)
4671 		return err;
4672 	/* Setup DCB netlink interface */
4673 	ice_dcbnl_setup(vsi);
4674 
4675 	err = ice_init_mac_fltr(pf);
4676 	if (err)
4677 		goto err_init_mac_fltr;
4678 
4679 	err = ice_devlink_create_pf_port(pf);
4680 	if (err)
4681 		goto err_devlink_create_pf_port;
4682 
4683 	SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port);
4684 
4685 	err = ice_register_netdev(vsi);
4686 	if (err)
4687 		goto err_register_netdev;
4688 
4689 	err = ice_tc_indir_block_register(vsi);
4690 	if (err)
4691 		goto err_tc_indir_block_register;
4692 
4693 	ice_napi_add(vsi);
4694 
4695 	return 0;
4696 
4697 err_tc_indir_block_register:
4698 	ice_unregister_netdev(vsi);
4699 err_register_netdev:
4700 	ice_devlink_destroy_pf_port(pf);
4701 err_devlink_create_pf_port:
4702 err_init_mac_fltr:
4703 	ice_decfg_netdev(vsi);
4704 	return err;
4705 }
4706 
4707 static void ice_deinit_eth(struct ice_pf *pf)
4708 {
4709 	struct ice_vsi *vsi = ice_get_main_vsi(pf);
4710 
4711 	if (!vsi)
4712 		return;
4713 
4714 	ice_vsi_close(vsi);
4715 	ice_unregister_netdev(vsi);
4716 	ice_devlink_destroy_pf_port(pf);
4717 	ice_tc_indir_block_unregister(vsi);
4718 	ice_decfg_netdev(vsi);
4719 }
4720 
4721 static int ice_init_dev(struct ice_pf *pf)
4722 {
4723 	struct device *dev = ice_pf_to_dev(pf);
4724 	struct ice_hw *hw = &pf->hw;
4725 	int err;
4726 
4727 	err = ice_init_hw(hw);
4728 	if (err) {
4729 		dev_err(dev, "ice_init_hw failed: %d\n", err);
4730 		return err;
4731 	}
4732 
4733 	ice_init_feature_support(pf);
4734 
4735 	ice_request_fw(pf);
4736 
4737 	/* if ice_request_fw fails, ICE_FLAG_ADV_FEATURES bit won't be
4738 	 * set in pf->state, which will cause ice_is_safe_mode to return
4739 	 * true
4740 	 */
4741 	if (ice_is_safe_mode(pf)) {
4742 		/* we already got function/device capabilities but these don't
4743 		 * reflect what the driver needs to do in safe mode. Instead of
4744 		 * adding conditional logic everywhere to ignore these
4745 		 * device/function capabilities, override them.
4746 		 */
4747 		ice_set_safe_mode_caps(hw);
4748 	}
4749 
4750 	err = ice_init_pf(pf);
4751 	if (err) {
4752 		dev_err(dev, "ice_init_pf failed: %d\n", err);
4753 		goto err_init_pf;
4754 	}
4755 
4756 	pf->hw.udp_tunnel_nic.set_port = ice_udp_tunnel_set_port;
4757 	pf->hw.udp_tunnel_nic.unset_port = ice_udp_tunnel_unset_port;
4758 	pf->hw.udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
4759 	pf->hw.udp_tunnel_nic.shared = &pf->hw.udp_tunnel_shared;
4760 	if (pf->hw.tnl.valid_count[TNL_VXLAN]) {
4761 		pf->hw.udp_tunnel_nic.tables[0].n_entries =
4762 			pf->hw.tnl.valid_count[TNL_VXLAN];
4763 		pf->hw.udp_tunnel_nic.tables[0].tunnel_types =
4764 			UDP_TUNNEL_TYPE_VXLAN;
4765 	}
4766 	if (pf->hw.tnl.valid_count[TNL_GENEVE]) {
4767 		pf->hw.udp_tunnel_nic.tables[1].n_entries =
4768 			pf->hw.tnl.valid_count[TNL_GENEVE];
4769 		pf->hw.udp_tunnel_nic.tables[1].tunnel_types =
4770 			UDP_TUNNEL_TYPE_GENEVE;
4771 	}
4772 
4773 	err = ice_init_interrupt_scheme(pf);
4774 	if (err) {
4775 		dev_err(dev, "ice_init_interrupt_scheme failed: %d\n", err);
4776 		err = -EIO;
4777 		goto err_init_interrupt_scheme;
4778 	}
4779 
4780 	/* In case of MSIX we are going to setup the misc vector right here
4781 	 * to handle admin queue events etc. In case of legacy and MSI
4782 	 * the misc functionality and queue processing is combined in
4783 	 * the same vector and that gets setup at open.
4784 	 */
4785 	err = ice_req_irq_msix_misc(pf);
4786 	if (err) {
4787 		dev_err(dev, "setup of misc vector failed: %d\n", err);
4788 		goto err_req_irq_msix_misc;
4789 	}
4790 
4791 	return 0;
4792 
4793 err_req_irq_msix_misc:
4794 	ice_clear_interrupt_scheme(pf);
4795 err_init_interrupt_scheme:
4796 	ice_deinit_pf(pf);
4797 err_init_pf:
4798 	ice_deinit_hw(hw);
4799 	return err;
4800 }
4801 
4802 static void ice_deinit_dev(struct ice_pf *pf)
4803 {
4804 	ice_free_irq_msix_misc(pf);
4805 	ice_clear_interrupt_scheme(pf);
4806 	ice_deinit_pf(pf);
4807 	ice_deinit_hw(&pf->hw);
4808 }
4809 
4810 static void ice_init_features(struct ice_pf *pf)
4811 {
4812 	struct device *dev = ice_pf_to_dev(pf);
4813 
4814 	if (ice_is_safe_mode(pf))
4815 		return;
4816 
4817 	/* initialize DDP driven features */
4818 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
4819 		ice_ptp_init(pf);
4820 
4821 	if (ice_is_feature_supported(pf, ICE_F_GNSS))
4822 		ice_gnss_init(pf);
4823 
4824 	/* Note: Flow director init failure is non-fatal to load */
4825 	if (ice_init_fdir(pf))
4826 		dev_err(dev, "could not initialize flow director\n");
4827 
4828 	/* Note: DCB init failure is non-fatal to load */
4829 	if (ice_init_pf_dcb(pf, false)) {
4830 		clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags);
4831 		clear_bit(ICE_FLAG_DCB_ENA, pf->flags);
4832 	} else {
4833 		ice_cfg_lldp_mib_change(&pf->hw, true);
4834 	}
4835 
4836 	if (ice_init_lag(pf))
4837 		dev_warn(dev, "Failed to init link aggregation support\n");
4838 }
4839 
4840 static void ice_deinit_features(struct ice_pf *pf)
4841 {
4842 	ice_deinit_lag(pf);
4843 	if (test_bit(ICE_FLAG_DCB_CAPABLE, pf->flags))
4844 		ice_cfg_lldp_mib_change(&pf->hw, false);
4845 	ice_deinit_fdir(pf);
4846 	if (ice_is_feature_supported(pf, ICE_F_GNSS))
4847 		ice_gnss_exit(pf);
4848 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
4849 		ice_ptp_release(pf);
4850 }
4851 
4852 static void ice_init_wakeup(struct ice_pf *pf)
4853 {
4854 	/* Save wakeup reason register for later use */
4855 	pf->wakeup_reason = rd32(&pf->hw, PFPM_WUS);
4856 
4857 	/* check for a power management event */
4858 	ice_print_wake_reason(pf);
4859 
4860 	/* clear wake status, all bits */
4861 	wr32(&pf->hw, PFPM_WUS, U32_MAX);
4862 
4863 	/* Disable WoL at init, wait for user to enable */
4864 	device_set_wakeup_enable(ice_pf_to_dev(pf), false);
4865 }
4866 
4867 static int ice_init_link(struct ice_pf *pf)
4868 {
4869 	struct device *dev = ice_pf_to_dev(pf);
4870 	int err;
4871 
4872 	err = ice_init_link_events(pf->hw.port_info);
4873 	if (err) {
4874 		dev_err(dev, "ice_init_link_events failed: %d\n", err);
4875 		return err;
4876 	}
4877 
4878 	/* not a fatal error if this fails */
4879 	err = ice_init_nvm_phy_type(pf->hw.port_info);
4880 	if (err)
4881 		dev_err(dev, "ice_init_nvm_phy_type failed: %d\n", err);
4882 
4883 	/* not a fatal error if this fails */
4884 	err = ice_update_link_info(pf->hw.port_info);
4885 	if (err)
4886 		dev_err(dev, "ice_update_link_info failed: %d\n", err);
4887 
4888 	ice_init_link_dflt_override(pf->hw.port_info);
4889 
4890 	ice_check_link_cfg_err(pf,
4891 			       pf->hw.port_info->phy.link_info.link_cfg_err);
4892 
4893 	/* if media available, initialize PHY settings */
4894 	if (pf->hw.port_info->phy.link_info.link_info &
4895 	    ICE_AQ_MEDIA_AVAILABLE) {
4896 		/* not a fatal error if this fails */
4897 		err = ice_init_phy_user_cfg(pf->hw.port_info);
4898 		if (err)
4899 			dev_err(dev, "ice_init_phy_user_cfg failed: %d\n", err);
4900 
4901 		if (!test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) {
4902 			struct ice_vsi *vsi = ice_get_main_vsi(pf);
4903 
4904 			if (vsi)
4905 				ice_configure_phy(vsi);
4906 		}
4907 	} else {
4908 		set_bit(ICE_FLAG_NO_MEDIA, pf->flags);
4909 	}
4910 
4911 	return err;
4912 }
4913 
4914 static int ice_init_pf_sw(struct ice_pf *pf)
4915 {
4916 	bool dvm = ice_is_dvm_ena(&pf->hw);
4917 	struct ice_vsi *vsi;
4918 	int err;
4919 
4920 	/* create switch struct for the switch element created by FW on boot */
4921 	pf->first_sw = kzalloc(sizeof(*pf->first_sw), GFP_KERNEL);
4922 	if (!pf->first_sw)
4923 		return -ENOMEM;
4924 
4925 	if (pf->hw.evb_veb)
4926 		pf->first_sw->bridge_mode = BRIDGE_MODE_VEB;
4927 	else
4928 		pf->first_sw->bridge_mode = BRIDGE_MODE_VEPA;
4929 
4930 	pf->first_sw->pf = pf;
4931 
4932 	/* record the sw_id available for later use */
4933 	pf->first_sw->sw_id = pf->hw.port_info->sw_id;
4934 
4935 	err = ice_aq_set_port_params(pf->hw.port_info, dvm, NULL);
4936 	if (err)
4937 		goto err_aq_set_port_params;
4938 
4939 	vsi = ice_pf_vsi_setup(pf, pf->hw.port_info);
4940 	if (!vsi) {
4941 		err = -ENOMEM;
4942 		goto err_pf_vsi_setup;
4943 	}
4944 
4945 	return 0;
4946 
4947 err_pf_vsi_setup:
4948 err_aq_set_port_params:
4949 	kfree(pf->first_sw);
4950 	return err;
4951 }
4952 
4953 static void ice_deinit_pf_sw(struct ice_pf *pf)
4954 {
4955 	struct ice_vsi *vsi = ice_get_main_vsi(pf);
4956 
4957 	if (!vsi)
4958 		return;
4959 
4960 	ice_vsi_release(vsi);
4961 	kfree(pf->first_sw);
4962 }
4963 
4964 static int ice_alloc_vsis(struct ice_pf *pf)
4965 {
4966 	struct device *dev = ice_pf_to_dev(pf);
4967 
4968 	pf->num_alloc_vsi = pf->hw.func_caps.guar_num_vsi;
4969 	if (!pf->num_alloc_vsi)
4970 		return -EIO;
4971 
4972 	if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
4973 		dev_warn(dev,
4974 			 "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
4975 			 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
4976 		pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
4977 	}
4978 
4979 	pf->vsi = devm_kcalloc(dev, pf->num_alloc_vsi, sizeof(*pf->vsi),
4980 			       GFP_KERNEL);
4981 	if (!pf->vsi)
4982 		return -ENOMEM;
4983 
4984 	pf->vsi_stats = devm_kcalloc(dev, pf->num_alloc_vsi,
4985 				     sizeof(*pf->vsi_stats), GFP_KERNEL);
4986 	if (!pf->vsi_stats) {
4987 		devm_kfree(dev, pf->vsi);
4988 		return -ENOMEM;
4989 	}
4990 
4991 	return 0;
4992 }
4993 
4994 static void ice_dealloc_vsis(struct ice_pf *pf)
4995 {
4996 	devm_kfree(ice_pf_to_dev(pf), pf->vsi_stats);
4997 	pf->vsi_stats = NULL;
4998 
4999 	pf->num_alloc_vsi = 0;
5000 	devm_kfree(ice_pf_to_dev(pf), pf->vsi);
5001 	pf->vsi = NULL;
5002 }
5003 
5004 static int ice_init_devlink(struct ice_pf *pf)
5005 {
5006 	int err;
5007 
5008 	err = ice_devlink_register_params(pf);
5009 	if (err)
5010 		return err;
5011 
5012 	ice_devlink_init_regions(pf);
5013 	ice_devlink_register(pf);
5014 
5015 	return 0;
5016 }
5017 
5018 static void ice_deinit_devlink(struct ice_pf *pf)
5019 {
5020 	ice_devlink_unregister(pf);
5021 	ice_devlink_destroy_regions(pf);
5022 	ice_devlink_unregister_params(pf);
5023 }
5024 
5025 static int ice_init(struct ice_pf *pf)
5026 {
5027 	int err;
5028 
5029 	err = ice_init_dev(pf);
5030 	if (err)
5031 		return err;
5032 
5033 	err = ice_alloc_vsis(pf);
5034 	if (err)
5035 		goto err_alloc_vsis;
5036 
5037 	err = ice_init_pf_sw(pf);
5038 	if (err)
5039 		goto err_init_pf_sw;
5040 
5041 	ice_init_wakeup(pf);
5042 
5043 	err = ice_init_link(pf);
5044 	if (err)
5045 		goto err_init_link;
5046 
5047 	err = ice_send_version(pf);
5048 	if (err)
5049 		goto err_init_link;
5050 
5051 	ice_verify_cacheline_size(pf);
5052 
5053 	if (ice_is_safe_mode(pf))
5054 		ice_set_safe_mode_vlan_cfg(pf);
5055 	else
5056 		/* print PCI link speed and width */
5057 		pcie_print_link_status(pf->pdev);
5058 
5059 	/* ready to go, so clear down state bit */
5060 	clear_bit(ICE_DOWN, pf->state);
5061 	clear_bit(ICE_SERVICE_DIS, pf->state);
5062 
5063 	/* since everything is good, start the service timer */
5064 	mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
5065 
5066 	return 0;
5067 
5068 err_init_link:
5069 	ice_deinit_pf_sw(pf);
5070 err_init_pf_sw:
5071 	ice_dealloc_vsis(pf);
5072 err_alloc_vsis:
5073 	ice_deinit_dev(pf);
5074 	return err;
5075 }
5076 
5077 static void ice_deinit(struct ice_pf *pf)
5078 {
5079 	set_bit(ICE_SERVICE_DIS, pf->state);
5080 	set_bit(ICE_DOWN, pf->state);
5081 
5082 	ice_deinit_pf_sw(pf);
5083 	ice_dealloc_vsis(pf);
5084 	ice_deinit_dev(pf);
5085 }
5086 
5087 /**
5088  * ice_load - load pf by init hw and starting VSI
5089  * @pf: pointer to the pf instance
5090  */
5091 int ice_load(struct ice_pf *pf)
5092 {
5093 	struct ice_vsi_cfg_params params = {};
5094 	struct ice_vsi *vsi;
5095 	int err;
5096 
5097 	err = ice_reset(&pf->hw, ICE_RESET_PFR);
5098 	if (err)
5099 		return err;
5100 
5101 	err = ice_init_dev(pf);
5102 	if (err)
5103 		return err;
5104 
5105 	vsi = ice_get_main_vsi(pf);
5106 
5107 	params = ice_vsi_to_params(vsi);
5108 	params.flags = ICE_VSI_FLAG_INIT;
5109 
5110 	err = ice_vsi_cfg(vsi, &params);
5111 	if (err)
5112 		goto err_vsi_cfg;
5113 
5114 	err = ice_start_eth(ice_get_main_vsi(pf));
5115 	if (err)
5116 		goto err_start_eth;
5117 
5118 	err = ice_init_rdma(pf);
5119 	if (err)
5120 		goto err_init_rdma;
5121 
5122 	ice_init_features(pf);
5123 	ice_service_task_restart(pf);
5124 
5125 	clear_bit(ICE_DOWN, pf->state);
5126 
5127 	return 0;
5128 
5129 err_init_rdma:
5130 	ice_vsi_close(ice_get_main_vsi(pf));
5131 err_start_eth:
5132 	ice_vsi_decfg(ice_get_main_vsi(pf));
5133 err_vsi_cfg:
5134 	ice_deinit_dev(pf);
5135 	return err;
5136 }
5137 
5138 /**
5139  * ice_unload - unload pf by stopping VSI and deinit hw
5140  * @pf: pointer to the pf instance
5141  */
5142 void ice_unload(struct ice_pf *pf)
5143 {
5144 	ice_deinit_features(pf);
5145 	ice_deinit_rdma(pf);
5146 	ice_stop_eth(ice_get_main_vsi(pf));
5147 	ice_vsi_decfg(ice_get_main_vsi(pf));
5148 	ice_deinit_dev(pf);
5149 }
5150 
5151 /**
5152  * ice_probe - Device initialization routine
5153  * @pdev: PCI device information struct
5154  * @ent: entry in ice_pci_tbl
5155  *
5156  * Returns 0 on success, negative on failure
5157  */
5158 static int
5159 ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent)
5160 {
5161 	struct device *dev = &pdev->dev;
5162 	struct ice_pf *pf;
5163 	struct ice_hw *hw;
5164 	int err;
5165 
5166 	if (pdev->is_virtfn) {
5167 		dev_err(dev, "can't probe a virtual function\n");
5168 		return -EINVAL;
5169 	}
5170 
5171 	/* this driver uses devres, see
5172 	 * Documentation/driver-api/driver-model/devres.rst
5173 	 */
5174 	err = pcim_enable_device(pdev);
5175 	if (err)
5176 		return err;
5177 
5178 	err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), dev_driver_string(dev));
5179 	if (err) {
5180 		dev_err(dev, "BAR0 I/O map error %d\n", err);
5181 		return err;
5182 	}
5183 
5184 	pf = ice_allocate_pf(dev);
5185 	if (!pf)
5186 		return -ENOMEM;
5187 
5188 	/* initialize Auxiliary index to invalid value */
5189 	pf->aux_idx = -1;
5190 
5191 	/* set up for high or low DMA */
5192 	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
5193 	if (err) {
5194 		dev_err(dev, "DMA configuration failed: 0x%x\n", err);
5195 		return err;
5196 	}
5197 
5198 	pci_set_master(pdev);
5199 
5200 	pf->pdev = pdev;
5201 	pci_set_drvdata(pdev, pf);
5202 	set_bit(ICE_DOWN, pf->state);
5203 	/* Disable service task until DOWN bit is cleared */
5204 	set_bit(ICE_SERVICE_DIS, pf->state);
5205 
5206 	hw = &pf->hw;
5207 	hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0];
5208 	pci_save_state(pdev);
5209 
5210 	hw->back = pf;
5211 	hw->port_info = NULL;
5212 	hw->vendor_id = pdev->vendor;
5213 	hw->device_id = pdev->device;
5214 	pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
5215 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
5216 	hw->subsystem_device_id = pdev->subsystem_device;
5217 	hw->bus.device = PCI_SLOT(pdev->devfn);
5218 	hw->bus.func = PCI_FUNC(pdev->devfn);
5219 	ice_set_ctrlq_len(hw);
5220 
5221 	pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M);
5222 
5223 #ifndef CONFIG_DYNAMIC_DEBUG
5224 	if (debug < -1)
5225 		hw->debug_mask = debug;
5226 #endif
5227 
5228 	err = ice_init(pf);
5229 	if (err)
5230 		goto err_init;
5231 
5232 	err = ice_init_eth(pf);
5233 	if (err)
5234 		goto err_init_eth;
5235 
5236 	err = ice_init_rdma(pf);
5237 	if (err)
5238 		goto err_init_rdma;
5239 
5240 	err = ice_init_devlink(pf);
5241 	if (err)
5242 		goto err_init_devlink;
5243 
5244 	ice_init_features(pf);
5245 
5246 	return 0;
5247 
5248 err_init_devlink:
5249 	ice_deinit_rdma(pf);
5250 err_init_rdma:
5251 	ice_deinit_eth(pf);
5252 err_init_eth:
5253 	ice_deinit(pf);
5254 err_init:
5255 	pci_disable_device(pdev);
5256 	return err;
5257 }
5258 
5259 /**
5260  * ice_set_wake - enable or disable Wake on LAN
5261  * @pf: pointer to the PF struct
5262  *
5263  * Simple helper for WoL control
5264  */
5265 static void ice_set_wake(struct ice_pf *pf)
5266 {
5267 	struct ice_hw *hw = &pf->hw;
5268 	bool wol = pf->wol_ena;
5269 
5270 	/* clear wake state, otherwise new wake events won't fire */
5271 	wr32(hw, PFPM_WUS, U32_MAX);
5272 
5273 	/* enable / disable APM wake up, no RMW needed */
5274 	wr32(hw, PFPM_APM, wol ? PFPM_APM_APME_M : 0);
5275 
5276 	/* set magic packet filter enabled */
5277 	wr32(hw, PFPM_WUFC, wol ? PFPM_WUFC_MAG_M : 0);
5278 }
5279 
5280 /**
5281  * ice_setup_mc_magic_wake - setup device to wake on multicast magic packet
5282  * @pf: pointer to the PF struct
5283  *
5284  * Issue firmware command to enable multicast magic wake, making
5285  * sure that any locally administered address (LAA) is used for
5286  * wake, and that PF reset doesn't undo the LAA.
5287  */
5288 static void ice_setup_mc_magic_wake(struct ice_pf *pf)
5289 {
5290 	struct device *dev = ice_pf_to_dev(pf);
5291 	struct ice_hw *hw = &pf->hw;
5292 	u8 mac_addr[ETH_ALEN];
5293 	struct ice_vsi *vsi;
5294 	int status;
5295 	u8 flags;
5296 
5297 	if (!pf->wol_ena)
5298 		return;
5299 
5300 	vsi = ice_get_main_vsi(pf);
5301 	if (!vsi)
5302 		return;
5303 
5304 	/* Get current MAC address in case it's an LAA */
5305 	if (vsi->netdev)
5306 		ether_addr_copy(mac_addr, vsi->netdev->dev_addr);
5307 	else
5308 		ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr);
5309 
5310 	flags = ICE_AQC_MAN_MAC_WR_MC_MAG_EN |
5311 		ICE_AQC_MAN_MAC_UPDATE_LAA_WOL |
5312 		ICE_AQC_MAN_MAC_WR_WOL_LAA_PFR_KEEP;
5313 
5314 	status = ice_aq_manage_mac_write(hw, mac_addr, flags, NULL);
5315 	if (status)
5316 		dev_err(dev, "Failed to enable Multicast Magic Packet wake, err %d aq_err %s\n",
5317 			status, ice_aq_str(hw->adminq.sq_last_status));
5318 }
5319 
5320 /**
5321  * ice_remove - Device removal routine
5322  * @pdev: PCI device information struct
5323  */
5324 static void ice_remove(struct pci_dev *pdev)
5325 {
5326 	struct ice_pf *pf = pci_get_drvdata(pdev);
5327 	int i;
5328 
5329 	for (i = 0; i < ICE_MAX_RESET_WAIT; i++) {
5330 		if (!ice_is_reset_in_progress(pf->state))
5331 			break;
5332 		msleep(100);
5333 	}
5334 
5335 	if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) {
5336 		set_bit(ICE_VF_RESETS_DISABLED, pf->state);
5337 		ice_free_vfs(pf);
5338 	}
5339 
5340 	ice_service_task_stop(pf);
5341 	ice_aq_cancel_waiting_tasks(pf);
5342 	set_bit(ICE_DOWN, pf->state);
5343 
5344 	if (!ice_is_safe_mode(pf))
5345 		ice_remove_arfs(pf);
5346 	ice_deinit_features(pf);
5347 	ice_deinit_devlink(pf);
5348 	ice_deinit_rdma(pf);
5349 	ice_deinit_eth(pf);
5350 	ice_deinit(pf);
5351 
5352 	ice_vsi_release_all(pf);
5353 
5354 	ice_setup_mc_magic_wake(pf);
5355 	ice_set_wake(pf);
5356 
5357 	/* Issue a PFR as part of the prescribed driver unload flow.  Do not
5358 	 * do it via ice_schedule_reset() since there is no need to rebuild
5359 	 * and the service task is already stopped.
5360 	 */
5361 	ice_reset(&pf->hw, ICE_RESET_PFR);
5362 	pci_wait_for_pending_transaction(pdev);
5363 	pci_disable_device(pdev);
5364 }
5365 
5366 /**
5367  * ice_shutdown - PCI callback for shutting down device
5368  * @pdev: PCI device information struct
5369  */
5370 static void ice_shutdown(struct pci_dev *pdev)
5371 {
5372 	struct ice_pf *pf = pci_get_drvdata(pdev);
5373 
5374 	ice_remove(pdev);
5375 
5376 	if (system_state == SYSTEM_POWER_OFF) {
5377 		pci_wake_from_d3(pdev, pf->wol_ena);
5378 		pci_set_power_state(pdev, PCI_D3hot);
5379 	}
5380 }
5381 
5382 #ifdef CONFIG_PM
5383 /**
5384  * ice_prepare_for_shutdown - prep for PCI shutdown
5385  * @pf: board private structure
5386  *
5387  * Inform or close all dependent features in prep for PCI device shutdown
5388  */
5389 static void ice_prepare_for_shutdown(struct ice_pf *pf)
5390 {
5391 	struct ice_hw *hw = &pf->hw;
5392 	u32 v;
5393 
5394 	/* Notify VFs of impending reset */
5395 	if (ice_check_sq_alive(hw, &hw->mailboxq))
5396 		ice_vc_notify_reset(pf);
5397 
5398 	dev_dbg(ice_pf_to_dev(pf), "Tearing down internal switch for shutdown\n");
5399 
5400 	/* disable the VSIs and their queues that are not already DOWN */
5401 	ice_pf_dis_all_vsi(pf, false);
5402 
5403 	ice_for_each_vsi(pf, v)
5404 		if (pf->vsi[v])
5405 			pf->vsi[v]->vsi_num = 0;
5406 
5407 	ice_shutdown_all_ctrlq(hw);
5408 }
5409 
5410 /**
5411  * ice_reinit_interrupt_scheme - Reinitialize interrupt scheme
5412  * @pf: board private structure to reinitialize
5413  *
5414  * This routine reinitialize interrupt scheme that was cleared during
5415  * power management suspend callback.
5416  *
5417  * This should be called during resume routine to re-allocate the q_vectors
5418  * and reacquire interrupts.
5419  */
5420 static int ice_reinit_interrupt_scheme(struct ice_pf *pf)
5421 {
5422 	struct device *dev = ice_pf_to_dev(pf);
5423 	int ret, v;
5424 
5425 	/* Since we clear MSIX flag during suspend, we need to
5426 	 * set it back during resume...
5427 	 */
5428 
5429 	ret = ice_init_interrupt_scheme(pf);
5430 	if (ret) {
5431 		dev_err(dev, "Failed to re-initialize interrupt %d\n", ret);
5432 		return ret;
5433 	}
5434 
5435 	/* Remap vectors and rings, after successful re-init interrupts */
5436 	ice_for_each_vsi(pf, v) {
5437 		if (!pf->vsi[v])
5438 			continue;
5439 
5440 		ret = ice_vsi_alloc_q_vectors(pf->vsi[v]);
5441 		if (ret)
5442 			goto err_reinit;
5443 		ice_vsi_map_rings_to_vectors(pf->vsi[v]);
5444 	}
5445 
5446 	ret = ice_req_irq_msix_misc(pf);
5447 	if (ret) {
5448 		dev_err(dev, "Setting up misc vector failed after device suspend %d\n",
5449 			ret);
5450 		goto err_reinit;
5451 	}
5452 
5453 	return 0;
5454 
5455 err_reinit:
5456 	while (v--)
5457 		if (pf->vsi[v])
5458 			ice_vsi_free_q_vectors(pf->vsi[v]);
5459 
5460 	return ret;
5461 }
5462 
5463 /**
5464  * ice_suspend
5465  * @dev: generic device information structure
5466  *
5467  * Power Management callback to quiesce the device and prepare
5468  * for D3 transition.
5469  */
5470 static int __maybe_unused ice_suspend(struct device *dev)
5471 {
5472 	struct pci_dev *pdev = to_pci_dev(dev);
5473 	struct ice_pf *pf;
5474 	int disabled, v;
5475 
5476 	pf = pci_get_drvdata(pdev);
5477 
5478 	if (!ice_pf_state_is_nominal(pf)) {
5479 		dev_err(dev, "Device is not ready, no need to suspend it\n");
5480 		return -EBUSY;
5481 	}
5482 
5483 	/* Stop watchdog tasks until resume completion.
5484 	 * Even though it is most likely that the service task is
5485 	 * disabled if the device is suspended or down, the service task's
5486 	 * state is controlled by a different state bit, and we should
5487 	 * store and honor whatever state that bit is in at this point.
5488 	 */
5489 	disabled = ice_service_task_stop(pf);
5490 
5491 	ice_unplug_aux_dev(pf);
5492 
5493 	/* Already suspended?, then there is nothing to do */
5494 	if (test_and_set_bit(ICE_SUSPENDED, pf->state)) {
5495 		if (!disabled)
5496 			ice_service_task_restart(pf);
5497 		return 0;
5498 	}
5499 
5500 	if (test_bit(ICE_DOWN, pf->state) ||
5501 	    ice_is_reset_in_progress(pf->state)) {
5502 		dev_err(dev, "can't suspend device in reset or already down\n");
5503 		if (!disabled)
5504 			ice_service_task_restart(pf);
5505 		return 0;
5506 	}
5507 
5508 	ice_setup_mc_magic_wake(pf);
5509 
5510 	ice_prepare_for_shutdown(pf);
5511 
5512 	ice_set_wake(pf);
5513 
5514 	/* Free vectors, clear the interrupt scheme and release IRQs
5515 	 * for proper hibernation, especially with large number of CPUs.
5516 	 * Otherwise hibernation might fail when mapping all the vectors back
5517 	 * to CPU0.
5518 	 */
5519 	ice_free_irq_msix_misc(pf);
5520 	ice_for_each_vsi(pf, v) {
5521 		if (!pf->vsi[v])
5522 			continue;
5523 		ice_vsi_free_q_vectors(pf->vsi[v]);
5524 	}
5525 	ice_clear_interrupt_scheme(pf);
5526 
5527 	pci_save_state(pdev);
5528 	pci_wake_from_d3(pdev, pf->wol_ena);
5529 	pci_set_power_state(pdev, PCI_D3hot);
5530 	return 0;
5531 }
5532 
5533 /**
5534  * ice_resume - PM callback for waking up from D3
5535  * @dev: generic device information structure
5536  */
5537 static int __maybe_unused ice_resume(struct device *dev)
5538 {
5539 	struct pci_dev *pdev = to_pci_dev(dev);
5540 	enum ice_reset_req reset_type;
5541 	struct ice_pf *pf;
5542 	struct ice_hw *hw;
5543 	int ret;
5544 
5545 	pci_set_power_state(pdev, PCI_D0);
5546 	pci_restore_state(pdev);
5547 	pci_save_state(pdev);
5548 
5549 	if (!pci_device_is_present(pdev))
5550 		return -ENODEV;
5551 
5552 	ret = pci_enable_device_mem(pdev);
5553 	if (ret) {
5554 		dev_err(dev, "Cannot enable device after suspend\n");
5555 		return ret;
5556 	}
5557 
5558 	pf = pci_get_drvdata(pdev);
5559 	hw = &pf->hw;
5560 
5561 	pf->wakeup_reason = rd32(hw, PFPM_WUS);
5562 	ice_print_wake_reason(pf);
5563 
5564 	/* We cleared the interrupt scheme when we suspended, so we need to
5565 	 * restore it now to resume device functionality.
5566 	 */
5567 	ret = ice_reinit_interrupt_scheme(pf);
5568 	if (ret)
5569 		dev_err(dev, "Cannot restore interrupt scheme: %d\n", ret);
5570 
5571 	clear_bit(ICE_DOWN, pf->state);
5572 	/* Now perform PF reset and rebuild */
5573 	reset_type = ICE_RESET_PFR;
5574 	/* re-enable service task for reset, but allow reset to schedule it */
5575 	clear_bit(ICE_SERVICE_DIS, pf->state);
5576 
5577 	if (ice_schedule_reset(pf, reset_type))
5578 		dev_err(dev, "Reset during resume failed.\n");
5579 
5580 	clear_bit(ICE_SUSPENDED, pf->state);
5581 	ice_service_task_restart(pf);
5582 
5583 	/* Restart the service task */
5584 	mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
5585 
5586 	return 0;
5587 }
5588 #endif /* CONFIG_PM */
5589 
5590 /**
5591  * ice_pci_err_detected - warning that PCI error has been detected
5592  * @pdev: PCI device information struct
5593  * @err: the type of PCI error
5594  *
5595  * Called to warn that something happened on the PCI bus and the error handling
5596  * is in progress.  Allows the driver to gracefully prepare/handle PCI errors.
5597  */
5598 static pci_ers_result_t
5599 ice_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t err)
5600 {
5601 	struct ice_pf *pf = pci_get_drvdata(pdev);
5602 
5603 	if (!pf) {
5604 		dev_err(&pdev->dev, "%s: unrecoverable device error %d\n",
5605 			__func__, err);
5606 		return PCI_ERS_RESULT_DISCONNECT;
5607 	}
5608 
5609 	if (!test_bit(ICE_SUSPENDED, pf->state)) {
5610 		ice_service_task_stop(pf);
5611 
5612 		if (!test_bit(ICE_PREPARED_FOR_RESET, pf->state)) {
5613 			set_bit(ICE_PFR_REQ, pf->state);
5614 			ice_prepare_for_reset(pf, ICE_RESET_PFR);
5615 		}
5616 	}
5617 
5618 	return PCI_ERS_RESULT_NEED_RESET;
5619 }
5620 
5621 /**
5622  * ice_pci_err_slot_reset - a PCI slot reset has just happened
5623  * @pdev: PCI device information struct
5624  *
5625  * Called to determine if the driver can recover from the PCI slot reset by
5626  * using a register read to determine if the device is recoverable.
5627  */
5628 static pci_ers_result_t ice_pci_err_slot_reset(struct pci_dev *pdev)
5629 {
5630 	struct ice_pf *pf = pci_get_drvdata(pdev);
5631 	pci_ers_result_t result;
5632 	int err;
5633 	u32 reg;
5634 
5635 	err = pci_enable_device_mem(pdev);
5636 	if (err) {
5637 		dev_err(&pdev->dev, "Cannot re-enable PCI device after reset, error %d\n",
5638 			err);
5639 		result = PCI_ERS_RESULT_DISCONNECT;
5640 	} else {
5641 		pci_set_master(pdev);
5642 		pci_restore_state(pdev);
5643 		pci_save_state(pdev);
5644 		pci_wake_from_d3(pdev, false);
5645 
5646 		/* Check for life */
5647 		reg = rd32(&pf->hw, GLGEN_RTRIG);
5648 		if (!reg)
5649 			result = PCI_ERS_RESULT_RECOVERED;
5650 		else
5651 			result = PCI_ERS_RESULT_DISCONNECT;
5652 	}
5653 
5654 	return result;
5655 }
5656 
5657 /**
5658  * ice_pci_err_resume - restart operations after PCI error recovery
5659  * @pdev: PCI device information struct
5660  *
5661  * Called to allow the driver to bring things back up after PCI error and/or
5662  * reset recovery have finished
5663  */
5664 static void ice_pci_err_resume(struct pci_dev *pdev)
5665 {
5666 	struct ice_pf *pf = pci_get_drvdata(pdev);
5667 
5668 	if (!pf) {
5669 		dev_err(&pdev->dev, "%s failed, device is unrecoverable\n",
5670 			__func__);
5671 		return;
5672 	}
5673 
5674 	if (test_bit(ICE_SUSPENDED, pf->state)) {
5675 		dev_dbg(&pdev->dev, "%s failed to resume normal operations!\n",
5676 			__func__);
5677 		return;
5678 	}
5679 
5680 	ice_restore_all_vfs_msi_state(pdev);
5681 
5682 	ice_do_reset(pf, ICE_RESET_PFR);
5683 	ice_service_task_restart(pf);
5684 	mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
5685 }
5686 
5687 /**
5688  * ice_pci_err_reset_prepare - prepare device driver for PCI reset
5689  * @pdev: PCI device information struct
5690  */
5691 static void ice_pci_err_reset_prepare(struct pci_dev *pdev)
5692 {
5693 	struct ice_pf *pf = pci_get_drvdata(pdev);
5694 
5695 	if (!test_bit(ICE_SUSPENDED, pf->state)) {
5696 		ice_service_task_stop(pf);
5697 
5698 		if (!test_bit(ICE_PREPARED_FOR_RESET, pf->state)) {
5699 			set_bit(ICE_PFR_REQ, pf->state);
5700 			ice_prepare_for_reset(pf, ICE_RESET_PFR);
5701 		}
5702 	}
5703 }
5704 
5705 /**
5706  * ice_pci_err_reset_done - PCI reset done, device driver reset can begin
5707  * @pdev: PCI device information struct
5708  */
5709 static void ice_pci_err_reset_done(struct pci_dev *pdev)
5710 {
5711 	ice_pci_err_resume(pdev);
5712 }
5713 
5714 /* ice_pci_tbl - PCI Device ID Table
5715  *
5716  * Wildcard entries (PCI_ANY_ID) should come last
5717  * Last entry must be all 0s
5718  *
5719  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
5720  *   Class, Class Mask, private data (not used) }
5721  */
5722 static const struct pci_device_id ice_pci_tbl[] = {
5723 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_BACKPLANE), 0 },
5724 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_QSFP), 0 },
5725 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_SFP), 0 },
5726 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_BACKPLANE), 0 },
5727 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_QSFP), 0 },
5728 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_SFP), 0 },
5729 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_BACKPLANE), 0 },
5730 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_QSFP), 0 },
5731 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SFP), 0 },
5732 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_10G_BASE_T), 0 },
5733 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SGMII), 0 },
5734 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_BACKPLANE), 0 },
5735 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_QSFP), 0 },
5736 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SFP), 0 },
5737 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_10G_BASE_T), 0 },
5738 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SGMII), 0 },
5739 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_BACKPLANE), 0 },
5740 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SFP), 0 },
5741 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_10G_BASE_T), 0 },
5742 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SGMII), 0 },
5743 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_BACKPLANE), 0 },
5744 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_SFP), 0 },
5745 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_10G_BASE_T), 0 },
5746 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_1GBE), 0 },
5747 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_QSFP), 0 },
5748 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822_SI_DFLT), 0 },
5749 	/* required last entry */
5750 	{ 0, }
5751 };
5752 MODULE_DEVICE_TABLE(pci, ice_pci_tbl);
5753 
5754 static __maybe_unused SIMPLE_DEV_PM_OPS(ice_pm_ops, ice_suspend, ice_resume);
5755 
5756 static const struct pci_error_handlers ice_pci_err_handler = {
5757 	.error_detected = ice_pci_err_detected,
5758 	.slot_reset = ice_pci_err_slot_reset,
5759 	.reset_prepare = ice_pci_err_reset_prepare,
5760 	.reset_done = ice_pci_err_reset_done,
5761 	.resume = ice_pci_err_resume
5762 };
5763 
5764 static struct pci_driver ice_driver = {
5765 	.name = KBUILD_MODNAME,
5766 	.id_table = ice_pci_tbl,
5767 	.probe = ice_probe,
5768 	.remove = ice_remove,
5769 #ifdef CONFIG_PM
5770 	.driver.pm = &ice_pm_ops,
5771 #endif /* CONFIG_PM */
5772 	.shutdown = ice_shutdown,
5773 	.sriov_configure = ice_sriov_configure,
5774 	.err_handler = &ice_pci_err_handler
5775 };
5776 
5777 /**
5778  * ice_module_init - Driver registration routine
5779  *
5780  * ice_module_init is the first routine called when the driver is
5781  * loaded. All it does is register with the PCI subsystem.
5782  */
5783 static int __init ice_module_init(void)
5784 {
5785 	int status;
5786 
5787 	pr_info("%s\n", ice_driver_string);
5788 	pr_info("%s\n", ice_copyright);
5789 
5790 	ice_wq = alloc_workqueue("%s", 0, 0, KBUILD_MODNAME);
5791 	if (!ice_wq) {
5792 		pr_err("Failed to create workqueue\n");
5793 		return -ENOMEM;
5794 	}
5795 
5796 	status = pci_register_driver(&ice_driver);
5797 	if (status) {
5798 		pr_err("failed to register PCI driver, err %d\n", status);
5799 		destroy_workqueue(ice_wq);
5800 	}
5801 
5802 	return status;
5803 }
5804 module_init(ice_module_init);
5805 
5806 /**
5807  * ice_module_exit - Driver exit cleanup routine
5808  *
5809  * ice_module_exit is called just before the driver is removed
5810  * from memory.
5811  */
5812 static void __exit ice_module_exit(void)
5813 {
5814 	pci_unregister_driver(&ice_driver);
5815 	destroy_workqueue(ice_wq);
5816 	pr_info("module unloaded\n");
5817 }
5818 module_exit(ice_module_exit);
5819 
5820 /**
5821  * ice_set_mac_address - NDO callback to set MAC address
5822  * @netdev: network interface device structure
5823  * @pi: pointer to an address structure
5824  *
5825  * Returns 0 on success, negative on failure
5826  */
5827 static int ice_set_mac_address(struct net_device *netdev, void *pi)
5828 {
5829 	struct ice_netdev_priv *np = netdev_priv(netdev);
5830 	struct ice_vsi *vsi = np->vsi;
5831 	struct ice_pf *pf = vsi->back;
5832 	struct ice_hw *hw = &pf->hw;
5833 	struct sockaddr *addr = pi;
5834 	u8 old_mac[ETH_ALEN];
5835 	u8 flags = 0;
5836 	u8 *mac;
5837 	int err;
5838 
5839 	mac = (u8 *)addr->sa_data;
5840 
5841 	if (!is_valid_ether_addr(mac))
5842 		return -EADDRNOTAVAIL;
5843 
5844 	if (ether_addr_equal(netdev->dev_addr, mac)) {
5845 		netdev_dbg(netdev, "already using mac %pM\n", mac);
5846 		return 0;
5847 	}
5848 
5849 	if (test_bit(ICE_DOWN, pf->state) ||
5850 	    ice_is_reset_in_progress(pf->state)) {
5851 		netdev_err(netdev, "can't set mac %pM. device not ready\n",
5852 			   mac);
5853 		return -EBUSY;
5854 	}
5855 
5856 	if (ice_chnl_dmac_fltr_cnt(pf)) {
5857 		netdev_err(netdev, "can't set mac %pM. Device has tc-flower filters, delete all of them and try again\n",
5858 			   mac);
5859 		return -EAGAIN;
5860 	}
5861 
5862 	netif_addr_lock_bh(netdev);
5863 	ether_addr_copy(old_mac, netdev->dev_addr);
5864 	/* change the netdev's MAC address */
5865 	eth_hw_addr_set(netdev, mac);
5866 	netif_addr_unlock_bh(netdev);
5867 
5868 	/* Clean up old MAC filter. Not an error if old filter doesn't exist */
5869 	err = ice_fltr_remove_mac(vsi, old_mac, ICE_FWD_TO_VSI);
5870 	if (err && err != -ENOENT) {
5871 		err = -EADDRNOTAVAIL;
5872 		goto err_update_filters;
5873 	}
5874 
5875 	/* Add filter for new MAC. If filter exists, return success */
5876 	err = ice_fltr_add_mac(vsi, mac, ICE_FWD_TO_VSI);
5877 	if (err == -EEXIST) {
5878 		/* Although this MAC filter is already present in hardware it's
5879 		 * possible in some cases (e.g. bonding) that dev_addr was
5880 		 * modified outside of the driver and needs to be restored back
5881 		 * to this value.
5882 		 */
5883 		netdev_dbg(netdev, "filter for MAC %pM already exists\n", mac);
5884 
5885 		return 0;
5886 	} else if (err) {
5887 		/* error if the new filter addition failed */
5888 		err = -EADDRNOTAVAIL;
5889 	}
5890 
5891 err_update_filters:
5892 	if (err) {
5893 		netdev_err(netdev, "can't set MAC %pM. filter update failed\n",
5894 			   mac);
5895 		netif_addr_lock_bh(netdev);
5896 		eth_hw_addr_set(netdev, old_mac);
5897 		netif_addr_unlock_bh(netdev);
5898 		return err;
5899 	}
5900 
5901 	netdev_dbg(vsi->netdev, "updated MAC address to %pM\n",
5902 		   netdev->dev_addr);
5903 
5904 	/* write new MAC address to the firmware */
5905 	flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL;
5906 	err = ice_aq_manage_mac_write(hw, mac, flags, NULL);
5907 	if (err) {
5908 		netdev_err(netdev, "can't set MAC %pM. write to firmware failed error %d\n",
5909 			   mac, err);
5910 	}
5911 	return 0;
5912 }
5913 
5914 /**
5915  * ice_set_rx_mode - NDO callback to set the netdev filters
5916  * @netdev: network interface device structure
5917  */
5918 static void ice_set_rx_mode(struct net_device *netdev)
5919 {
5920 	struct ice_netdev_priv *np = netdev_priv(netdev);
5921 	struct ice_vsi *vsi = np->vsi;
5922 
5923 	if (!vsi)
5924 		return;
5925 
5926 	/* Set the flags to synchronize filters
5927 	 * ndo_set_rx_mode may be triggered even without a change in netdev
5928 	 * flags
5929 	 */
5930 	set_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state);
5931 	set_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state);
5932 	set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags);
5933 
5934 	/* schedule our worker thread which will take care of
5935 	 * applying the new filter changes
5936 	 */
5937 	ice_service_task_schedule(vsi->back);
5938 }
5939 
5940 /**
5941  * ice_set_tx_maxrate - NDO callback to set the maximum per-queue bitrate
5942  * @netdev: network interface device structure
5943  * @queue_index: Queue ID
5944  * @maxrate: maximum bandwidth in Mbps
5945  */
5946 static int
5947 ice_set_tx_maxrate(struct net_device *netdev, int queue_index, u32 maxrate)
5948 {
5949 	struct ice_netdev_priv *np = netdev_priv(netdev);
5950 	struct ice_vsi *vsi = np->vsi;
5951 	u16 q_handle;
5952 	int status;
5953 	u8 tc;
5954 
5955 	/* Validate maxrate requested is within permitted range */
5956 	if (maxrate && (maxrate > (ICE_SCHED_MAX_BW / 1000))) {
5957 		netdev_err(netdev, "Invalid max rate %d specified for the queue %d\n",
5958 			   maxrate, queue_index);
5959 		return -EINVAL;
5960 	}
5961 
5962 	q_handle = vsi->tx_rings[queue_index]->q_handle;
5963 	tc = ice_dcb_get_tc(vsi, queue_index);
5964 
5965 	/* Set BW back to default, when user set maxrate to 0 */
5966 	if (!maxrate)
5967 		status = ice_cfg_q_bw_dflt_lmt(vsi->port_info, vsi->idx, tc,
5968 					       q_handle, ICE_MAX_BW);
5969 	else
5970 		status = ice_cfg_q_bw_lmt(vsi->port_info, vsi->idx, tc,
5971 					  q_handle, ICE_MAX_BW, maxrate * 1000);
5972 	if (status)
5973 		netdev_err(netdev, "Unable to set Tx max rate, error %d\n",
5974 			   status);
5975 
5976 	return status;
5977 }
5978 
5979 /**
5980  * ice_fdb_add - add an entry to the hardware database
5981  * @ndm: the input from the stack
5982  * @tb: pointer to array of nladdr (unused)
5983  * @dev: the net device pointer
5984  * @addr: the MAC address entry being added
5985  * @vid: VLAN ID
5986  * @flags: instructions from stack about fdb operation
5987  * @extack: netlink extended ack
5988  */
5989 static int
5990 ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[],
5991 	    struct net_device *dev, const unsigned char *addr, u16 vid,
5992 	    u16 flags, struct netlink_ext_ack __always_unused *extack)
5993 {
5994 	int err;
5995 
5996 	if (vid) {
5997 		netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n");
5998 		return -EINVAL;
5999 	}
6000 	if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
6001 		netdev_err(dev, "FDB only supports static addresses\n");
6002 		return -EINVAL;
6003 	}
6004 
6005 	if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
6006 		err = dev_uc_add_excl(dev, addr);
6007 	else if (is_multicast_ether_addr(addr))
6008 		err = dev_mc_add_excl(dev, addr);
6009 	else
6010 		err = -EINVAL;
6011 
6012 	/* Only return duplicate errors if NLM_F_EXCL is set */
6013 	if (err == -EEXIST && !(flags & NLM_F_EXCL))
6014 		err = 0;
6015 
6016 	return err;
6017 }
6018 
6019 /**
6020  * ice_fdb_del - delete an entry from the hardware database
6021  * @ndm: the input from the stack
6022  * @tb: pointer to array of nladdr (unused)
6023  * @dev: the net device pointer
6024  * @addr: the MAC address entry being added
6025  * @vid: VLAN ID
6026  * @extack: netlink extended ack
6027  */
6028 static int
6029 ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[],
6030 	    struct net_device *dev, const unsigned char *addr,
6031 	    __always_unused u16 vid, struct netlink_ext_ack *extack)
6032 {
6033 	int err;
6034 
6035 	if (ndm->ndm_state & NUD_PERMANENT) {
6036 		netdev_err(dev, "FDB only supports static addresses\n");
6037 		return -EINVAL;
6038 	}
6039 
6040 	if (is_unicast_ether_addr(addr))
6041 		err = dev_uc_del(dev, addr);
6042 	else if (is_multicast_ether_addr(addr))
6043 		err = dev_mc_del(dev, addr);
6044 	else
6045 		err = -EINVAL;
6046 
6047 	return err;
6048 }
6049 
6050 #define NETIF_VLAN_OFFLOAD_FEATURES	(NETIF_F_HW_VLAN_CTAG_RX | \
6051 					 NETIF_F_HW_VLAN_CTAG_TX | \
6052 					 NETIF_F_HW_VLAN_STAG_RX | \
6053 					 NETIF_F_HW_VLAN_STAG_TX)
6054 
6055 #define NETIF_VLAN_STRIPPING_FEATURES	(NETIF_F_HW_VLAN_CTAG_RX | \
6056 					 NETIF_F_HW_VLAN_STAG_RX)
6057 
6058 #define NETIF_VLAN_FILTERING_FEATURES	(NETIF_F_HW_VLAN_CTAG_FILTER | \
6059 					 NETIF_F_HW_VLAN_STAG_FILTER)
6060 
6061 /**
6062  * ice_fix_features - fix the netdev features flags based on device limitations
6063  * @netdev: ptr to the netdev that flags are being fixed on
6064  * @features: features that need to be checked and possibly fixed
6065  *
6066  * Make sure any fixups are made to features in this callback. This enables the
6067  * driver to not have to check unsupported configurations throughout the driver
6068  * because that's the responsiblity of this callback.
6069  *
6070  * Single VLAN Mode (SVM) Supported Features:
6071  *	NETIF_F_HW_VLAN_CTAG_FILTER
6072  *	NETIF_F_HW_VLAN_CTAG_RX
6073  *	NETIF_F_HW_VLAN_CTAG_TX
6074  *
6075  * Double VLAN Mode (DVM) Supported Features:
6076  *	NETIF_F_HW_VLAN_CTAG_FILTER
6077  *	NETIF_F_HW_VLAN_CTAG_RX
6078  *	NETIF_F_HW_VLAN_CTAG_TX
6079  *
6080  *	NETIF_F_HW_VLAN_STAG_FILTER
6081  *	NETIF_HW_VLAN_STAG_RX
6082  *	NETIF_HW_VLAN_STAG_TX
6083  *
6084  * Features that need fixing:
6085  *	Cannot simultaneously enable CTAG and STAG stripping and/or insertion.
6086  *	These are mutually exlusive as the VSI context cannot support multiple
6087  *	VLAN ethertypes simultaneously for stripping and/or insertion. If this
6088  *	is not done, then default to clearing the requested STAG offload
6089  *	settings.
6090  *
6091  *	All supported filtering has to be enabled or disabled together. For
6092  *	example, in DVM, CTAG and STAG filtering have to be enabled and disabled
6093  *	together. If this is not done, then default to VLAN filtering disabled.
6094  *	These are mutually exclusive as there is currently no way to
6095  *	enable/disable VLAN filtering based on VLAN ethertype when using VLAN
6096  *	prune rules.
6097  */
6098 static netdev_features_t
6099 ice_fix_features(struct net_device *netdev, netdev_features_t features)
6100 {
6101 	struct ice_netdev_priv *np = netdev_priv(netdev);
6102 	netdev_features_t req_vlan_fltr, cur_vlan_fltr;
6103 	bool cur_ctag, cur_stag, req_ctag, req_stag;
6104 
6105 	cur_vlan_fltr = netdev->features & NETIF_VLAN_FILTERING_FEATURES;
6106 	cur_ctag = cur_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER;
6107 	cur_stag = cur_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER;
6108 
6109 	req_vlan_fltr = features & NETIF_VLAN_FILTERING_FEATURES;
6110 	req_ctag = req_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER;
6111 	req_stag = req_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER;
6112 
6113 	if (req_vlan_fltr != cur_vlan_fltr) {
6114 		if (ice_is_dvm_ena(&np->vsi->back->hw)) {
6115 			if (req_ctag && req_stag) {
6116 				features |= NETIF_VLAN_FILTERING_FEATURES;
6117 			} else if (!req_ctag && !req_stag) {
6118 				features &= ~NETIF_VLAN_FILTERING_FEATURES;
6119 			} else if ((!cur_ctag && req_ctag && !cur_stag) ||
6120 				   (!cur_stag && req_stag && !cur_ctag)) {
6121 				features |= NETIF_VLAN_FILTERING_FEATURES;
6122 				netdev_warn(netdev,  "802.1Q and 802.1ad VLAN filtering must be either both on or both off. VLAN filtering has been enabled for both types.\n");
6123 			} else if ((cur_ctag && !req_ctag && cur_stag) ||
6124 				   (cur_stag && !req_stag && cur_ctag)) {
6125 				features &= ~NETIF_VLAN_FILTERING_FEATURES;
6126 				netdev_warn(netdev,  "802.1Q and 802.1ad VLAN filtering must be either both on or both off. VLAN filtering has been disabled for both types.\n");
6127 			}
6128 		} else {
6129 			if (req_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER)
6130 				netdev_warn(netdev, "cannot support requested 802.1ad filtering setting in SVM mode\n");
6131 
6132 			if (req_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER)
6133 				features |= NETIF_F_HW_VLAN_CTAG_FILTER;
6134 		}
6135 	}
6136 
6137 	if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) &&
6138 	    (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))) {
6139 		netdev_warn(netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n");
6140 		features &= ~(NETIF_F_HW_VLAN_STAG_RX |
6141 			      NETIF_F_HW_VLAN_STAG_TX);
6142 	}
6143 
6144 	if (!(netdev->features & NETIF_F_RXFCS) &&
6145 	    (features & NETIF_F_RXFCS) &&
6146 	    (features & NETIF_VLAN_STRIPPING_FEATURES) &&
6147 	    !ice_vsi_has_non_zero_vlans(np->vsi)) {
6148 		netdev_warn(netdev, "Disabling VLAN stripping as FCS/CRC stripping is also disabled and there is no VLAN configured\n");
6149 		features &= ~NETIF_VLAN_STRIPPING_FEATURES;
6150 	}
6151 
6152 	return features;
6153 }
6154 
6155 /**
6156  * ice_set_vlan_offload_features - set VLAN offload features for the PF VSI
6157  * @vsi: PF's VSI
6158  * @features: features used to determine VLAN offload settings
6159  *
6160  * First, determine the vlan_ethertype based on the VLAN offload bits in
6161  * features. Then determine if stripping and insertion should be enabled or
6162  * disabled. Finally enable or disable VLAN stripping and insertion.
6163  */
6164 static int
6165 ice_set_vlan_offload_features(struct ice_vsi *vsi, netdev_features_t features)
6166 {
6167 	bool enable_stripping = true, enable_insertion = true;
6168 	struct ice_vsi_vlan_ops *vlan_ops;
6169 	int strip_err = 0, insert_err = 0;
6170 	u16 vlan_ethertype = 0;
6171 
6172 	vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
6173 
6174 	if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
6175 		vlan_ethertype = ETH_P_8021AD;
6176 	else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
6177 		vlan_ethertype = ETH_P_8021Q;
6178 
6179 	if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX)))
6180 		enable_stripping = false;
6181 	if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX)))
6182 		enable_insertion = false;
6183 
6184 	if (enable_stripping)
6185 		strip_err = vlan_ops->ena_stripping(vsi, vlan_ethertype);
6186 	else
6187 		strip_err = vlan_ops->dis_stripping(vsi);
6188 
6189 	if (enable_insertion)
6190 		insert_err = vlan_ops->ena_insertion(vsi, vlan_ethertype);
6191 	else
6192 		insert_err = vlan_ops->dis_insertion(vsi);
6193 
6194 	if (strip_err || insert_err)
6195 		return -EIO;
6196 
6197 	return 0;
6198 }
6199 
6200 /**
6201  * ice_set_vlan_filtering_features - set VLAN filtering features for the PF VSI
6202  * @vsi: PF's VSI
6203  * @features: features used to determine VLAN filtering settings
6204  *
6205  * Enable or disable Rx VLAN filtering based on the VLAN filtering bits in the
6206  * features.
6207  */
6208 static int
6209 ice_set_vlan_filtering_features(struct ice_vsi *vsi, netdev_features_t features)
6210 {
6211 	struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
6212 	int err = 0;
6213 
6214 	/* support Single VLAN Mode (SVM) and Double VLAN Mode (DVM) by checking
6215 	 * if either bit is set
6216 	 */
6217 	if (features &
6218 	    (NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_FILTER))
6219 		err = vlan_ops->ena_rx_filtering(vsi);
6220 	else
6221 		err = vlan_ops->dis_rx_filtering(vsi);
6222 
6223 	return err;
6224 }
6225 
6226 /**
6227  * ice_set_vlan_features - set VLAN settings based on suggested feature set
6228  * @netdev: ptr to the netdev being adjusted
6229  * @features: the feature set that the stack is suggesting
6230  *
6231  * Only update VLAN settings if the requested_vlan_features are different than
6232  * the current_vlan_features.
6233  */
6234 static int
6235 ice_set_vlan_features(struct net_device *netdev, netdev_features_t features)
6236 {
6237 	netdev_features_t current_vlan_features, requested_vlan_features;
6238 	struct ice_netdev_priv *np = netdev_priv(netdev);
6239 	struct ice_vsi *vsi = np->vsi;
6240 	int err;
6241 
6242 	current_vlan_features = netdev->features & NETIF_VLAN_OFFLOAD_FEATURES;
6243 	requested_vlan_features = features & NETIF_VLAN_OFFLOAD_FEATURES;
6244 	if (current_vlan_features ^ requested_vlan_features) {
6245 		if ((features & NETIF_F_RXFCS) &&
6246 		    (features & NETIF_VLAN_STRIPPING_FEATURES)) {
6247 			dev_err(ice_pf_to_dev(vsi->back),
6248 				"To enable VLAN stripping, you must first enable FCS/CRC stripping\n");
6249 			return -EIO;
6250 		}
6251 
6252 		err = ice_set_vlan_offload_features(vsi, features);
6253 		if (err)
6254 			return err;
6255 	}
6256 
6257 	current_vlan_features = netdev->features &
6258 		NETIF_VLAN_FILTERING_FEATURES;
6259 	requested_vlan_features = features & NETIF_VLAN_FILTERING_FEATURES;
6260 	if (current_vlan_features ^ requested_vlan_features) {
6261 		err = ice_set_vlan_filtering_features(vsi, features);
6262 		if (err)
6263 			return err;
6264 	}
6265 
6266 	return 0;
6267 }
6268 
6269 /**
6270  * ice_set_loopback - turn on/off loopback mode on underlying PF
6271  * @vsi: ptr to VSI
6272  * @ena: flag to indicate the on/off setting
6273  */
6274 static int ice_set_loopback(struct ice_vsi *vsi, bool ena)
6275 {
6276 	bool if_running = netif_running(vsi->netdev);
6277 	int ret;
6278 
6279 	if (if_running && !test_and_set_bit(ICE_VSI_DOWN, vsi->state)) {
6280 		ret = ice_down(vsi);
6281 		if (ret) {
6282 			netdev_err(vsi->netdev, "Preparing device to toggle loopback failed\n");
6283 			return ret;
6284 		}
6285 	}
6286 	ret = ice_aq_set_mac_loopback(&vsi->back->hw, ena, NULL);
6287 	if (ret)
6288 		netdev_err(vsi->netdev, "Failed to toggle loopback state\n");
6289 	if (if_running)
6290 		ret = ice_up(vsi);
6291 
6292 	return ret;
6293 }
6294 
6295 /**
6296  * ice_set_features - set the netdev feature flags
6297  * @netdev: ptr to the netdev being adjusted
6298  * @features: the feature set that the stack is suggesting
6299  */
6300 static int
6301 ice_set_features(struct net_device *netdev, netdev_features_t features)
6302 {
6303 	netdev_features_t changed = netdev->features ^ features;
6304 	struct ice_netdev_priv *np = netdev_priv(netdev);
6305 	struct ice_vsi *vsi = np->vsi;
6306 	struct ice_pf *pf = vsi->back;
6307 	int ret = 0;
6308 
6309 	/* Don't set any netdev advanced features with device in Safe Mode */
6310 	if (ice_is_safe_mode(pf)) {
6311 		dev_err(ice_pf_to_dev(pf),
6312 			"Device is in Safe Mode - not enabling advanced netdev features\n");
6313 		return ret;
6314 	}
6315 
6316 	/* Do not change setting during reset */
6317 	if (ice_is_reset_in_progress(pf->state)) {
6318 		dev_err(ice_pf_to_dev(pf),
6319 			"Device is resetting, changing advanced netdev features temporarily unavailable.\n");
6320 		return -EBUSY;
6321 	}
6322 
6323 	/* Multiple features can be changed in one call so keep features in
6324 	 * separate if/else statements to guarantee each feature is checked
6325 	 */
6326 	if (changed & NETIF_F_RXHASH)
6327 		ice_vsi_manage_rss_lut(vsi, !!(features & NETIF_F_RXHASH));
6328 
6329 	ret = ice_set_vlan_features(netdev, features);
6330 	if (ret)
6331 		return ret;
6332 
6333 	/* Turn on receive of FCS aka CRC, and after setting this
6334 	 * flag the packet data will have the 4 byte CRC appended
6335 	 */
6336 	if (changed & NETIF_F_RXFCS) {
6337 		if ((features & NETIF_F_RXFCS) &&
6338 		    (features & NETIF_VLAN_STRIPPING_FEATURES)) {
6339 			dev_err(ice_pf_to_dev(vsi->back),
6340 				"To disable FCS/CRC stripping, you must first disable VLAN stripping\n");
6341 			return -EIO;
6342 		}
6343 
6344 		ice_vsi_cfg_crc_strip(vsi, !!(features & NETIF_F_RXFCS));
6345 		ret = ice_down_up(vsi);
6346 		if (ret)
6347 			return ret;
6348 	}
6349 
6350 	if (changed & NETIF_F_NTUPLE) {
6351 		bool ena = !!(features & NETIF_F_NTUPLE);
6352 
6353 		ice_vsi_manage_fdir(vsi, ena);
6354 		ena ? ice_init_arfs(vsi) : ice_clear_arfs(vsi);
6355 	}
6356 
6357 	/* don't turn off hw_tc_offload when ADQ is already enabled */
6358 	if (!(features & NETIF_F_HW_TC) && ice_is_adq_active(pf)) {
6359 		dev_err(ice_pf_to_dev(pf), "ADQ is active, can't turn hw_tc_offload off\n");
6360 		return -EACCES;
6361 	}
6362 
6363 	if (changed & NETIF_F_HW_TC) {
6364 		bool ena = !!(features & NETIF_F_HW_TC);
6365 
6366 		ena ? set_bit(ICE_FLAG_CLS_FLOWER, pf->flags) :
6367 		      clear_bit(ICE_FLAG_CLS_FLOWER, pf->flags);
6368 	}
6369 
6370 	if (changed & NETIF_F_LOOPBACK)
6371 		ret = ice_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK));
6372 
6373 	return ret;
6374 }
6375 
6376 /**
6377  * ice_vsi_vlan_setup - Setup VLAN offload properties on a PF VSI
6378  * @vsi: VSI to setup VLAN properties for
6379  */
6380 static int ice_vsi_vlan_setup(struct ice_vsi *vsi)
6381 {
6382 	int err;
6383 
6384 	err = ice_set_vlan_offload_features(vsi, vsi->netdev->features);
6385 	if (err)
6386 		return err;
6387 
6388 	err = ice_set_vlan_filtering_features(vsi, vsi->netdev->features);
6389 	if (err)
6390 		return err;
6391 
6392 	return ice_vsi_add_vlan_zero(vsi);
6393 }
6394 
6395 /**
6396  * ice_vsi_cfg_lan - Setup the VSI lan related config
6397  * @vsi: the VSI being configured
6398  *
6399  * Return 0 on success and negative value on error
6400  */
6401 int ice_vsi_cfg_lan(struct ice_vsi *vsi)
6402 {
6403 	int err;
6404 
6405 	if (vsi->netdev && vsi->type == ICE_VSI_PF) {
6406 		ice_set_rx_mode(vsi->netdev);
6407 
6408 		err = ice_vsi_vlan_setup(vsi);
6409 		if (err)
6410 			return err;
6411 	}
6412 	ice_vsi_cfg_dcb_rings(vsi);
6413 
6414 	err = ice_vsi_cfg_lan_txqs(vsi);
6415 	if (!err && ice_is_xdp_ena_vsi(vsi))
6416 		err = ice_vsi_cfg_xdp_txqs(vsi);
6417 	if (!err)
6418 		err = ice_vsi_cfg_rxqs(vsi);
6419 
6420 	return err;
6421 }
6422 
6423 /* THEORY OF MODERATION:
6424  * The ice driver hardware works differently than the hardware that DIMLIB was
6425  * originally made for. ice hardware doesn't have packet count limits that
6426  * can trigger an interrupt, but it *does* have interrupt rate limit support,
6427  * which is hard-coded to a limit of 250,000 ints/second.
6428  * If not using dynamic moderation, the INTRL value can be modified
6429  * by ethtool rx-usecs-high.
6430  */
6431 struct ice_dim {
6432 	/* the throttle rate for interrupts, basically worst case delay before
6433 	 * an initial interrupt fires, value is stored in microseconds.
6434 	 */
6435 	u16 itr;
6436 };
6437 
6438 /* Make a different profile for Rx that doesn't allow quite so aggressive
6439  * moderation at the high end (it maxes out at 126us or about 8k interrupts a
6440  * second.
6441  */
6442 static const struct ice_dim rx_profile[] = {
6443 	{2},    /* 500,000 ints/s, capped at 250K by INTRL */
6444 	{8},    /* 125,000 ints/s */
6445 	{16},   /*  62,500 ints/s */
6446 	{62},   /*  16,129 ints/s */
6447 	{126}   /*   7,936 ints/s */
6448 };
6449 
6450 /* The transmit profile, which has the same sorts of values
6451  * as the previous struct
6452  */
6453 static const struct ice_dim tx_profile[] = {
6454 	{2},    /* 500,000 ints/s, capped at 250K by INTRL */
6455 	{8},    /* 125,000 ints/s */
6456 	{40},   /*  16,125 ints/s */
6457 	{128},  /*   7,812 ints/s */
6458 	{256}   /*   3,906 ints/s */
6459 };
6460 
6461 static void ice_tx_dim_work(struct work_struct *work)
6462 {
6463 	struct ice_ring_container *rc;
6464 	struct dim *dim;
6465 	u16 itr;
6466 
6467 	dim = container_of(work, struct dim, work);
6468 	rc = (struct ice_ring_container *)dim->priv;
6469 
6470 	WARN_ON(dim->profile_ix >= ARRAY_SIZE(tx_profile));
6471 
6472 	/* look up the values in our local table */
6473 	itr = tx_profile[dim->profile_ix].itr;
6474 
6475 	ice_trace(tx_dim_work, container_of(rc, struct ice_q_vector, tx), dim);
6476 	ice_write_itr(rc, itr);
6477 
6478 	dim->state = DIM_START_MEASURE;
6479 }
6480 
6481 static void ice_rx_dim_work(struct work_struct *work)
6482 {
6483 	struct ice_ring_container *rc;
6484 	struct dim *dim;
6485 	u16 itr;
6486 
6487 	dim = container_of(work, struct dim, work);
6488 	rc = (struct ice_ring_container *)dim->priv;
6489 
6490 	WARN_ON(dim->profile_ix >= ARRAY_SIZE(rx_profile));
6491 
6492 	/* look up the values in our local table */
6493 	itr = rx_profile[dim->profile_ix].itr;
6494 
6495 	ice_trace(rx_dim_work, container_of(rc, struct ice_q_vector, rx), dim);
6496 	ice_write_itr(rc, itr);
6497 
6498 	dim->state = DIM_START_MEASURE;
6499 }
6500 
6501 #define ICE_DIM_DEFAULT_PROFILE_IX 1
6502 
6503 /**
6504  * ice_init_moderation - set up interrupt moderation
6505  * @q_vector: the vector containing rings to be configured
6506  *
6507  * Set up interrupt moderation registers, with the intent to do the right thing
6508  * when called from reset or from probe, and whether or not dynamic moderation
6509  * is enabled or not. Take special care to write all the registers in both
6510  * dynamic moderation mode or not in order to make sure hardware is in a known
6511  * state.
6512  */
6513 static void ice_init_moderation(struct ice_q_vector *q_vector)
6514 {
6515 	struct ice_ring_container *rc;
6516 	bool tx_dynamic, rx_dynamic;
6517 
6518 	rc = &q_vector->tx;
6519 	INIT_WORK(&rc->dim.work, ice_tx_dim_work);
6520 	rc->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
6521 	rc->dim.profile_ix = ICE_DIM_DEFAULT_PROFILE_IX;
6522 	rc->dim.priv = rc;
6523 	tx_dynamic = ITR_IS_DYNAMIC(rc);
6524 
6525 	/* set the initial TX ITR to match the above */
6526 	ice_write_itr(rc, tx_dynamic ?
6527 		      tx_profile[rc->dim.profile_ix].itr : rc->itr_setting);
6528 
6529 	rc = &q_vector->rx;
6530 	INIT_WORK(&rc->dim.work, ice_rx_dim_work);
6531 	rc->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
6532 	rc->dim.profile_ix = ICE_DIM_DEFAULT_PROFILE_IX;
6533 	rc->dim.priv = rc;
6534 	rx_dynamic = ITR_IS_DYNAMIC(rc);
6535 
6536 	/* set the initial RX ITR to match the above */
6537 	ice_write_itr(rc, rx_dynamic ? rx_profile[rc->dim.profile_ix].itr :
6538 				       rc->itr_setting);
6539 
6540 	ice_set_q_vector_intrl(q_vector);
6541 }
6542 
6543 /**
6544  * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI
6545  * @vsi: the VSI being configured
6546  */
6547 static void ice_napi_enable_all(struct ice_vsi *vsi)
6548 {
6549 	int q_idx;
6550 
6551 	if (!vsi->netdev)
6552 		return;
6553 
6554 	ice_for_each_q_vector(vsi, q_idx) {
6555 		struct ice_q_vector *q_vector = vsi->q_vectors[q_idx];
6556 
6557 		ice_init_moderation(q_vector);
6558 
6559 		if (q_vector->rx.rx_ring || q_vector->tx.tx_ring)
6560 			napi_enable(&q_vector->napi);
6561 	}
6562 }
6563 
6564 /**
6565  * ice_up_complete - Finish the last steps of bringing up a connection
6566  * @vsi: The VSI being configured
6567  *
6568  * Return 0 on success and negative value on error
6569  */
6570 static int ice_up_complete(struct ice_vsi *vsi)
6571 {
6572 	struct ice_pf *pf = vsi->back;
6573 	int err;
6574 
6575 	ice_vsi_cfg_msix(vsi);
6576 
6577 	/* Enable only Rx rings, Tx rings were enabled by the FW when the
6578 	 * Tx queue group list was configured and the context bits were
6579 	 * programmed using ice_vsi_cfg_txqs
6580 	 */
6581 	err = ice_vsi_start_all_rx_rings(vsi);
6582 	if (err)
6583 		return err;
6584 
6585 	clear_bit(ICE_VSI_DOWN, vsi->state);
6586 	ice_napi_enable_all(vsi);
6587 	ice_vsi_ena_irq(vsi);
6588 
6589 	if (vsi->port_info &&
6590 	    (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) &&
6591 	    vsi->netdev && vsi->type == ICE_VSI_PF) {
6592 		ice_print_link_msg(vsi, true);
6593 		netif_tx_start_all_queues(vsi->netdev);
6594 		netif_carrier_on(vsi->netdev);
6595 		ice_ptp_link_change(pf, pf->hw.pf_id, true);
6596 	}
6597 
6598 	/* Perform an initial read of the statistics registers now to
6599 	 * set the baseline so counters are ready when interface is up
6600 	 */
6601 	ice_update_eth_stats(vsi);
6602 
6603 	if (vsi->type == ICE_VSI_PF)
6604 		ice_service_task_schedule(pf);
6605 
6606 	return 0;
6607 }
6608 
6609 /**
6610  * ice_up - Bring the connection back up after being down
6611  * @vsi: VSI being configured
6612  */
6613 int ice_up(struct ice_vsi *vsi)
6614 {
6615 	int err;
6616 
6617 	err = ice_vsi_cfg_lan(vsi);
6618 	if (!err)
6619 		err = ice_up_complete(vsi);
6620 
6621 	return err;
6622 }
6623 
6624 /**
6625  * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring
6626  * @syncp: pointer to u64_stats_sync
6627  * @stats: stats that pkts and bytes count will be taken from
6628  * @pkts: packets stats counter
6629  * @bytes: bytes stats counter
6630  *
6631  * This function fetches stats from the ring considering the atomic operations
6632  * that needs to be performed to read u64 values in 32 bit machine.
6633  */
6634 void
6635 ice_fetch_u64_stats_per_ring(struct u64_stats_sync *syncp,
6636 			     struct ice_q_stats stats, u64 *pkts, u64 *bytes)
6637 {
6638 	unsigned int start;
6639 
6640 	do {
6641 		start = u64_stats_fetch_begin(syncp);
6642 		*pkts = stats.pkts;
6643 		*bytes = stats.bytes;
6644 	} while (u64_stats_fetch_retry(syncp, start));
6645 }
6646 
6647 /**
6648  * ice_update_vsi_tx_ring_stats - Update VSI Tx ring stats counters
6649  * @vsi: the VSI to be updated
6650  * @vsi_stats: the stats struct to be updated
6651  * @rings: rings to work on
6652  * @count: number of rings
6653  */
6654 static void
6655 ice_update_vsi_tx_ring_stats(struct ice_vsi *vsi,
6656 			     struct rtnl_link_stats64 *vsi_stats,
6657 			     struct ice_tx_ring **rings, u16 count)
6658 {
6659 	u16 i;
6660 
6661 	for (i = 0; i < count; i++) {
6662 		struct ice_tx_ring *ring;
6663 		u64 pkts = 0, bytes = 0;
6664 
6665 		ring = READ_ONCE(rings[i]);
6666 		if (!ring || !ring->ring_stats)
6667 			continue;
6668 		ice_fetch_u64_stats_per_ring(&ring->ring_stats->syncp,
6669 					     ring->ring_stats->stats, &pkts,
6670 					     &bytes);
6671 		vsi_stats->tx_packets += pkts;
6672 		vsi_stats->tx_bytes += bytes;
6673 		vsi->tx_restart += ring->ring_stats->tx_stats.restart_q;
6674 		vsi->tx_busy += ring->ring_stats->tx_stats.tx_busy;
6675 		vsi->tx_linearize += ring->ring_stats->tx_stats.tx_linearize;
6676 	}
6677 }
6678 
6679 /**
6680  * ice_update_vsi_ring_stats - Update VSI stats counters
6681  * @vsi: the VSI to be updated
6682  */
6683 static void ice_update_vsi_ring_stats(struct ice_vsi *vsi)
6684 {
6685 	struct rtnl_link_stats64 *net_stats, *stats_prev;
6686 	struct rtnl_link_stats64 *vsi_stats;
6687 	u64 pkts, bytes;
6688 	int i;
6689 
6690 	vsi_stats = kzalloc(sizeof(*vsi_stats), GFP_ATOMIC);
6691 	if (!vsi_stats)
6692 		return;
6693 
6694 	/* reset non-netdev (extended) stats */
6695 	vsi->tx_restart = 0;
6696 	vsi->tx_busy = 0;
6697 	vsi->tx_linearize = 0;
6698 	vsi->rx_buf_failed = 0;
6699 	vsi->rx_page_failed = 0;
6700 
6701 	rcu_read_lock();
6702 
6703 	/* update Tx rings counters */
6704 	ice_update_vsi_tx_ring_stats(vsi, vsi_stats, vsi->tx_rings,
6705 				     vsi->num_txq);
6706 
6707 	/* update Rx rings counters */
6708 	ice_for_each_rxq(vsi, i) {
6709 		struct ice_rx_ring *ring = READ_ONCE(vsi->rx_rings[i]);
6710 		struct ice_ring_stats *ring_stats;
6711 
6712 		ring_stats = ring->ring_stats;
6713 		ice_fetch_u64_stats_per_ring(&ring_stats->syncp,
6714 					     ring_stats->stats, &pkts,
6715 					     &bytes);
6716 		vsi_stats->rx_packets += pkts;
6717 		vsi_stats->rx_bytes += bytes;
6718 		vsi->rx_buf_failed += ring_stats->rx_stats.alloc_buf_failed;
6719 		vsi->rx_page_failed += ring_stats->rx_stats.alloc_page_failed;
6720 	}
6721 
6722 	/* update XDP Tx rings counters */
6723 	if (ice_is_xdp_ena_vsi(vsi))
6724 		ice_update_vsi_tx_ring_stats(vsi, vsi_stats, vsi->xdp_rings,
6725 					     vsi->num_xdp_txq);
6726 
6727 	rcu_read_unlock();
6728 
6729 	net_stats = &vsi->net_stats;
6730 	stats_prev = &vsi->net_stats_prev;
6731 
6732 	/* clear prev counters after reset */
6733 	if (vsi_stats->tx_packets < stats_prev->tx_packets ||
6734 	    vsi_stats->rx_packets < stats_prev->rx_packets) {
6735 		stats_prev->tx_packets = 0;
6736 		stats_prev->tx_bytes = 0;
6737 		stats_prev->rx_packets = 0;
6738 		stats_prev->rx_bytes = 0;
6739 	}
6740 
6741 	/* update netdev counters */
6742 	net_stats->tx_packets += vsi_stats->tx_packets - stats_prev->tx_packets;
6743 	net_stats->tx_bytes += vsi_stats->tx_bytes - stats_prev->tx_bytes;
6744 	net_stats->rx_packets += vsi_stats->rx_packets - stats_prev->rx_packets;
6745 	net_stats->rx_bytes += vsi_stats->rx_bytes - stats_prev->rx_bytes;
6746 
6747 	stats_prev->tx_packets = vsi_stats->tx_packets;
6748 	stats_prev->tx_bytes = vsi_stats->tx_bytes;
6749 	stats_prev->rx_packets = vsi_stats->rx_packets;
6750 	stats_prev->rx_bytes = vsi_stats->rx_bytes;
6751 
6752 	kfree(vsi_stats);
6753 }
6754 
6755 /**
6756  * ice_update_vsi_stats - Update VSI stats counters
6757  * @vsi: the VSI to be updated
6758  */
6759 void ice_update_vsi_stats(struct ice_vsi *vsi)
6760 {
6761 	struct rtnl_link_stats64 *cur_ns = &vsi->net_stats;
6762 	struct ice_eth_stats *cur_es = &vsi->eth_stats;
6763 	struct ice_pf *pf = vsi->back;
6764 
6765 	if (test_bit(ICE_VSI_DOWN, vsi->state) ||
6766 	    test_bit(ICE_CFG_BUSY, pf->state))
6767 		return;
6768 
6769 	/* get stats as recorded by Tx/Rx rings */
6770 	ice_update_vsi_ring_stats(vsi);
6771 
6772 	/* get VSI stats as recorded by the hardware */
6773 	ice_update_eth_stats(vsi);
6774 
6775 	cur_ns->tx_errors = cur_es->tx_errors;
6776 	cur_ns->rx_dropped = cur_es->rx_discards;
6777 	cur_ns->tx_dropped = cur_es->tx_discards;
6778 	cur_ns->multicast = cur_es->rx_multicast;
6779 
6780 	/* update some more netdev stats if this is main VSI */
6781 	if (vsi->type == ICE_VSI_PF) {
6782 		cur_ns->rx_crc_errors = pf->stats.crc_errors;
6783 		cur_ns->rx_errors = pf->stats.crc_errors +
6784 				    pf->stats.illegal_bytes +
6785 				    pf->stats.rx_len_errors +
6786 				    pf->stats.rx_undersize +
6787 				    pf->hw_csum_rx_error +
6788 				    pf->stats.rx_jabber +
6789 				    pf->stats.rx_fragments +
6790 				    pf->stats.rx_oversize;
6791 		cur_ns->rx_length_errors = pf->stats.rx_len_errors;
6792 		/* record drops from the port level */
6793 		cur_ns->rx_missed_errors = pf->stats.eth.rx_discards;
6794 	}
6795 }
6796 
6797 /**
6798  * ice_update_pf_stats - Update PF port stats counters
6799  * @pf: PF whose stats needs to be updated
6800  */
6801 void ice_update_pf_stats(struct ice_pf *pf)
6802 {
6803 	struct ice_hw_port_stats *prev_ps, *cur_ps;
6804 	struct ice_hw *hw = &pf->hw;
6805 	u16 fd_ctr_base;
6806 	u8 port;
6807 
6808 	port = hw->port_info->lport;
6809 	prev_ps = &pf->stats_prev;
6810 	cur_ps = &pf->stats;
6811 
6812 	if (ice_is_reset_in_progress(pf->state))
6813 		pf->stat_prev_loaded = false;
6814 
6815 	ice_stat_update40(hw, GLPRT_GORCL(port), pf->stat_prev_loaded,
6816 			  &prev_ps->eth.rx_bytes,
6817 			  &cur_ps->eth.rx_bytes);
6818 
6819 	ice_stat_update40(hw, GLPRT_UPRCL(port), pf->stat_prev_loaded,
6820 			  &prev_ps->eth.rx_unicast,
6821 			  &cur_ps->eth.rx_unicast);
6822 
6823 	ice_stat_update40(hw, GLPRT_MPRCL(port), pf->stat_prev_loaded,
6824 			  &prev_ps->eth.rx_multicast,
6825 			  &cur_ps->eth.rx_multicast);
6826 
6827 	ice_stat_update40(hw, GLPRT_BPRCL(port), pf->stat_prev_loaded,
6828 			  &prev_ps->eth.rx_broadcast,
6829 			  &cur_ps->eth.rx_broadcast);
6830 
6831 	ice_stat_update32(hw, PRTRPB_RDPC, pf->stat_prev_loaded,
6832 			  &prev_ps->eth.rx_discards,
6833 			  &cur_ps->eth.rx_discards);
6834 
6835 	ice_stat_update40(hw, GLPRT_GOTCL(port), pf->stat_prev_loaded,
6836 			  &prev_ps->eth.tx_bytes,
6837 			  &cur_ps->eth.tx_bytes);
6838 
6839 	ice_stat_update40(hw, GLPRT_UPTCL(port), pf->stat_prev_loaded,
6840 			  &prev_ps->eth.tx_unicast,
6841 			  &cur_ps->eth.tx_unicast);
6842 
6843 	ice_stat_update40(hw, GLPRT_MPTCL(port), pf->stat_prev_loaded,
6844 			  &prev_ps->eth.tx_multicast,
6845 			  &cur_ps->eth.tx_multicast);
6846 
6847 	ice_stat_update40(hw, GLPRT_BPTCL(port), pf->stat_prev_loaded,
6848 			  &prev_ps->eth.tx_broadcast,
6849 			  &cur_ps->eth.tx_broadcast);
6850 
6851 	ice_stat_update32(hw, GLPRT_TDOLD(port), pf->stat_prev_loaded,
6852 			  &prev_ps->tx_dropped_link_down,
6853 			  &cur_ps->tx_dropped_link_down);
6854 
6855 	ice_stat_update40(hw, GLPRT_PRC64L(port), pf->stat_prev_loaded,
6856 			  &prev_ps->rx_size_64, &cur_ps->rx_size_64);
6857 
6858 	ice_stat_update40(hw, GLPRT_PRC127L(port), pf->stat_prev_loaded,
6859 			  &prev_ps->rx_size_127, &cur_ps->rx_size_127);
6860 
6861 	ice_stat_update40(hw, GLPRT_PRC255L(port), pf->stat_prev_loaded,
6862 			  &prev_ps->rx_size_255, &cur_ps->rx_size_255);
6863 
6864 	ice_stat_update40(hw, GLPRT_PRC511L(port), pf->stat_prev_loaded,
6865 			  &prev_ps->rx_size_511, &cur_ps->rx_size_511);
6866 
6867 	ice_stat_update40(hw, GLPRT_PRC1023L(port), pf->stat_prev_loaded,
6868 			  &prev_ps->rx_size_1023, &cur_ps->rx_size_1023);
6869 
6870 	ice_stat_update40(hw, GLPRT_PRC1522L(port), pf->stat_prev_loaded,
6871 			  &prev_ps->rx_size_1522, &cur_ps->rx_size_1522);
6872 
6873 	ice_stat_update40(hw, GLPRT_PRC9522L(port), pf->stat_prev_loaded,
6874 			  &prev_ps->rx_size_big, &cur_ps->rx_size_big);
6875 
6876 	ice_stat_update40(hw, GLPRT_PTC64L(port), pf->stat_prev_loaded,
6877 			  &prev_ps->tx_size_64, &cur_ps->tx_size_64);
6878 
6879 	ice_stat_update40(hw, GLPRT_PTC127L(port), pf->stat_prev_loaded,
6880 			  &prev_ps->tx_size_127, &cur_ps->tx_size_127);
6881 
6882 	ice_stat_update40(hw, GLPRT_PTC255L(port), pf->stat_prev_loaded,
6883 			  &prev_ps->tx_size_255, &cur_ps->tx_size_255);
6884 
6885 	ice_stat_update40(hw, GLPRT_PTC511L(port), pf->stat_prev_loaded,
6886 			  &prev_ps->tx_size_511, &cur_ps->tx_size_511);
6887 
6888 	ice_stat_update40(hw, GLPRT_PTC1023L(port), pf->stat_prev_loaded,
6889 			  &prev_ps->tx_size_1023, &cur_ps->tx_size_1023);
6890 
6891 	ice_stat_update40(hw, GLPRT_PTC1522L(port), pf->stat_prev_loaded,
6892 			  &prev_ps->tx_size_1522, &cur_ps->tx_size_1522);
6893 
6894 	ice_stat_update40(hw, GLPRT_PTC9522L(port), pf->stat_prev_loaded,
6895 			  &prev_ps->tx_size_big, &cur_ps->tx_size_big);
6896 
6897 	fd_ctr_base = hw->fd_ctr_base;
6898 
6899 	ice_stat_update40(hw,
6900 			  GLSTAT_FD_CNT0L(ICE_FD_SB_STAT_IDX(fd_ctr_base)),
6901 			  pf->stat_prev_loaded, &prev_ps->fd_sb_match,
6902 			  &cur_ps->fd_sb_match);
6903 	ice_stat_update32(hw, GLPRT_LXONRXC(port), pf->stat_prev_loaded,
6904 			  &prev_ps->link_xon_rx, &cur_ps->link_xon_rx);
6905 
6906 	ice_stat_update32(hw, GLPRT_LXOFFRXC(port), pf->stat_prev_loaded,
6907 			  &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx);
6908 
6909 	ice_stat_update32(hw, GLPRT_LXONTXC(port), pf->stat_prev_loaded,
6910 			  &prev_ps->link_xon_tx, &cur_ps->link_xon_tx);
6911 
6912 	ice_stat_update32(hw, GLPRT_LXOFFTXC(port), pf->stat_prev_loaded,
6913 			  &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx);
6914 
6915 	ice_update_dcb_stats(pf);
6916 
6917 	ice_stat_update32(hw, GLPRT_CRCERRS(port), pf->stat_prev_loaded,
6918 			  &prev_ps->crc_errors, &cur_ps->crc_errors);
6919 
6920 	ice_stat_update32(hw, GLPRT_ILLERRC(port), pf->stat_prev_loaded,
6921 			  &prev_ps->illegal_bytes, &cur_ps->illegal_bytes);
6922 
6923 	ice_stat_update32(hw, GLPRT_MLFC(port), pf->stat_prev_loaded,
6924 			  &prev_ps->mac_local_faults,
6925 			  &cur_ps->mac_local_faults);
6926 
6927 	ice_stat_update32(hw, GLPRT_MRFC(port), pf->stat_prev_loaded,
6928 			  &prev_ps->mac_remote_faults,
6929 			  &cur_ps->mac_remote_faults);
6930 
6931 	ice_stat_update32(hw, GLPRT_RLEC(port), pf->stat_prev_loaded,
6932 			  &prev_ps->rx_len_errors, &cur_ps->rx_len_errors);
6933 
6934 	ice_stat_update32(hw, GLPRT_RUC(port), pf->stat_prev_loaded,
6935 			  &prev_ps->rx_undersize, &cur_ps->rx_undersize);
6936 
6937 	ice_stat_update32(hw, GLPRT_RFC(port), pf->stat_prev_loaded,
6938 			  &prev_ps->rx_fragments, &cur_ps->rx_fragments);
6939 
6940 	ice_stat_update32(hw, GLPRT_ROC(port), pf->stat_prev_loaded,
6941 			  &prev_ps->rx_oversize, &cur_ps->rx_oversize);
6942 
6943 	ice_stat_update32(hw, GLPRT_RJC(port), pf->stat_prev_loaded,
6944 			  &prev_ps->rx_jabber, &cur_ps->rx_jabber);
6945 
6946 	cur_ps->fd_sb_status = test_bit(ICE_FLAG_FD_ENA, pf->flags) ? 1 : 0;
6947 
6948 	pf->stat_prev_loaded = true;
6949 }
6950 
6951 /**
6952  * ice_get_stats64 - get statistics for network device structure
6953  * @netdev: network interface device structure
6954  * @stats: main device statistics structure
6955  */
6956 static
6957 void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
6958 {
6959 	struct ice_netdev_priv *np = netdev_priv(netdev);
6960 	struct rtnl_link_stats64 *vsi_stats;
6961 	struct ice_vsi *vsi = np->vsi;
6962 
6963 	vsi_stats = &vsi->net_stats;
6964 
6965 	if (!vsi->num_txq || !vsi->num_rxq)
6966 		return;
6967 
6968 	/* netdev packet/byte stats come from ring counter. These are obtained
6969 	 * by summing up ring counters (done by ice_update_vsi_ring_stats).
6970 	 * But, only call the update routine and read the registers if VSI is
6971 	 * not down.
6972 	 */
6973 	if (!test_bit(ICE_VSI_DOWN, vsi->state))
6974 		ice_update_vsi_ring_stats(vsi);
6975 	stats->tx_packets = vsi_stats->tx_packets;
6976 	stats->tx_bytes = vsi_stats->tx_bytes;
6977 	stats->rx_packets = vsi_stats->rx_packets;
6978 	stats->rx_bytes = vsi_stats->rx_bytes;
6979 
6980 	/* The rest of the stats can be read from the hardware but instead we
6981 	 * just return values that the watchdog task has already obtained from
6982 	 * the hardware.
6983 	 */
6984 	stats->multicast = vsi_stats->multicast;
6985 	stats->tx_errors = vsi_stats->tx_errors;
6986 	stats->tx_dropped = vsi_stats->tx_dropped;
6987 	stats->rx_errors = vsi_stats->rx_errors;
6988 	stats->rx_dropped = vsi_stats->rx_dropped;
6989 	stats->rx_crc_errors = vsi_stats->rx_crc_errors;
6990 	stats->rx_length_errors = vsi_stats->rx_length_errors;
6991 }
6992 
6993 /**
6994  * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI
6995  * @vsi: VSI having NAPI disabled
6996  */
6997 static void ice_napi_disable_all(struct ice_vsi *vsi)
6998 {
6999 	int q_idx;
7000 
7001 	if (!vsi->netdev)
7002 		return;
7003 
7004 	ice_for_each_q_vector(vsi, q_idx) {
7005 		struct ice_q_vector *q_vector = vsi->q_vectors[q_idx];
7006 
7007 		if (q_vector->rx.rx_ring || q_vector->tx.tx_ring)
7008 			napi_disable(&q_vector->napi);
7009 
7010 		cancel_work_sync(&q_vector->tx.dim.work);
7011 		cancel_work_sync(&q_vector->rx.dim.work);
7012 	}
7013 }
7014 
7015 /**
7016  * ice_down - Shutdown the connection
7017  * @vsi: The VSI being stopped
7018  *
7019  * Caller of this function is expected to set the vsi->state ICE_DOWN bit
7020  */
7021 int ice_down(struct ice_vsi *vsi)
7022 {
7023 	int i, tx_err, rx_err, vlan_err = 0;
7024 
7025 	WARN_ON(!test_bit(ICE_VSI_DOWN, vsi->state));
7026 
7027 	if (vsi->netdev && vsi->type == ICE_VSI_PF) {
7028 		vlan_err = ice_vsi_del_vlan_zero(vsi);
7029 		ice_ptp_link_change(vsi->back, vsi->back->hw.pf_id, false);
7030 		netif_carrier_off(vsi->netdev);
7031 		netif_tx_disable(vsi->netdev);
7032 	} else if (vsi->type == ICE_VSI_SWITCHDEV_CTRL) {
7033 		ice_eswitch_stop_all_tx_queues(vsi->back);
7034 	}
7035 
7036 	ice_vsi_dis_irq(vsi);
7037 
7038 	tx_err = ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, 0);
7039 	if (tx_err)
7040 		netdev_err(vsi->netdev, "Failed stop Tx rings, VSI %d error %d\n",
7041 			   vsi->vsi_num, tx_err);
7042 	if (!tx_err && ice_is_xdp_ena_vsi(vsi)) {
7043 		tx_err = ice_vsi_stop_xdp_tx_rings(vsi);
7044 		if (tx_err)
7045 			netdev_err(vsi->netdev, "Failed stop XDP rings, VSI %d error %d\n",
7046 				   vsi->vsi_num, tx_err);
7047 	}
7048 
7049 	rx_err = ice_vsi_stop_all_rx_rings(vsi);
7050 	if (rx_err)
7051 		netdev_err(vsi->netdev, "Failed stop Rx rings, VSI %d error %d\n",
7052 			   vsi->vsi_num, rx_err);
7053 
7054 	ice_napi_disable_all(vsi);
7055 
7056 	ice_for_each_txq(vsi, i)
7057 		ice_clean_tx_ring(vsi->tx_rings[i]);
7058 
7059 	ice_for_each_rxq(vsi, i)
7060 		ice_clean_rx_ring(vsi->rx_rings[i]);
7061 
7062 	if (tx_err || rx_err || vlan_err) {
7063 		netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n",
7064 			   vsi->vsi_num, vsi->vsw->sw_id);
7065 		return -EIO;
7066 	}
7067 
7068 	return 0;
7069 }
7070 
7071 /**
7072  * ice_down_up - shutdown the VSI connection and bring it up
7073  * @vsi: the VSI to be reconnected
7074  */
7075 int ice_down_up(struct ice_vsi *vsi)
7076 {
7077 	int ret;
7078 
7079 	/* if DOWN already set, nothing to do */
7080 	if (test_and_set_bit(ICE_VSI_DOWN, vsi->state))
7081 		return 0;
7082 
7083 	ret = ice_down(vsi);
7084 	if (ret)
7085 		return ret;
7086 
7087 	ret = ice_up(vsi);
7088 	if (ret) {
7089 		netdev_err(vsi->netdev, "reallocating resources failed during netdev features change, may need to reload driver\n");
7090 		return ret;
7091 	}
7092 
7093 	return 0;
7094 }
7095 
7096 /**
7097  * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources
7098  * @vsi: VSI having resources allocated
7099  *
7100  * Return 0 on success, negative on failure
7101  */
7102 int ice_vsi_setup_tx_rings(struct ice_vsi *vsi)
7103 {
7104 	int i, err = 0;
7105 
7106 	if (!vsi->num_txq) {
7107 		dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Tx queues\n",
7108 			vsi->vsi_num);
7109 		return -EINVAL;
7110 	}
7111 
7112 	ice_for_each_txq(vsi, i) {
7113 		struct ice_tx_ring *ring = vsi->tx_rings[i];
7114 
7115 		if (!ring)
7116 			return -EINVAL;
7117 
7118 		if (vsi->netdev)
7119 			ring->netdev = vsi->netdev;
7120 		err = ice_setup_tx_ring(ring);
7121 		if (err)
7122 			break;
7123 	}
7124 
7125 	return err;
7126 }
7127 
7128 /**
7129  * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources
7130  * @vsi: VSI having resources allocated
7131  *
7132  * Return 0 on success, negative on failure
7133  */
7134 int ice_vsi_setup_rx_rings(struct ice_vsi *vsi)
7135 {
7136 	int i, err = 0;
7137 
7138 	if (!vsi->num_rxq) {
7139 		dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Rx queues\n",
7140 			vsi->vsi_num);
7141 		return -EINVAL;
7142 	}
7143 
7144 	ice_for_each_rxq(vsi, i) {
7145 		struct ice_rx_ring *ring = vsi->rx_rings[i];
7146 
7147 		if (!ring)
7148 			return -EINVAL;
7149 
7150 		if (vsi->netdev)
7151 			ring->netdev = vsi->netdev;
7152 		err = ice_setup_rx_ring(ring);
7153 		if (err)
7154 			break;
7155 	}
7156 
7157 	return err;
7158 }
7159 
7160 /**
7161  * ice_vsi_open_ctrl - open control VSI for use
7162  * @vsi: the VSI to open
7163  *
7164  * Initialization of the Control VSI
7165  *
7166  * Returns 0 on success, negative value on error
7167  */
7168 int ice_vsi_open_ctrl(struct ice_vsi *vsi)
7169 {
7170 	char int_name[ICE_INT_NAME_STR_LEN];
7171 	struct ice_pf *pf = vsi->back;
7172 	struct device *dev;
7173 	int err;
7174 
7175 	dev = ice_pf_to_dev(pf);
7176 	/* allocate descriptors */
7177 	err = ice_vsi_setup_tx_rings(vsi);
7178 	if (err)
7179 		goto err_setup_tx;
7180 
7181 	err = ice_vsi_setup_rx_rings(vsi);
7182 	if (err)
7183 		goto err_setup_rx;
7184 
7185 	err = ice_vsi_cfg_lan(vsi);
7186 	if (err)
7187 		goto err_setup_rx;
7188 
7189 	snprintf(int_name, sizeof(int_name) - 1, "%s-%s:ctrl",
7190 		 dev_driver_string(dev), dev_name(dev));
7191 	err = ice_vsi_req_irq_msix(vsi, int_name);
7192 	if (err)
7193 		goto err_setup_rx;
7194 
7195 	ice_vsi_cfg_msix(vsi);
7196 
7197 	err = ice_vsi_start_all_rx_rings(vsi);
7198 	if (err)
7199 		goto err_up_complete;
7200 
7201 	clear_bit(ICE_VSI_DOWN, vsi->state);
7202 	ice_vsi_ena_irq(vsi);
7203 
7204 	return 0;
7205 
7206 err_up_complete:
7207 	ice_down(vsi);
7208 err_setup_rx:
7209 	ice_vsi_free_rx_rings(vsi);
7210 err_setup_tx:
7211 	ice_vsi_free_tx_rings(vsi);
7212 
7213 	return err;
7214 }
7215 
7216 /**
7217  * ice_vsi_open - Called when a network interface is made active
7218  * @vsi: the VSI to open
7219  *
7220  * Initialization of the VSI
7221  *
7222  * Returns 0 on success, negative value on error
7223  */
7224 int ice_vsi_open(struct ice_vsi *vsi)
7225 {
7226 	char int_name[ICE_INT_NAME_STR_LEN];
7227 	struct ice_pf *pf = vsi->back;
7228 	int err;
7229 
7230 	/* allocate descriptors */
7231 	err = ice_vsi_setup_tx_rings(vsi);
7232 	if (err)
7233 		goto err_setup_tx;
7234 
7235 	err = ice_vsi_setup_rx_rings(vsi);
7236 	if (err)
7237 		goto err_setup_rx;
7238 
7239 	err = ice_vsi_cfg_lan(vsi);
7240 	if (err)
7241 		goto err_setup_rx;
7242 
7243 	snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
7244 		 dev_driver_string(ice_pf_to_dev(pf)), vsi->netdev->name);
7245 	err = ice_vsi_req_irq_msix(vsi, int_name);
7246 	if (err)
7247 		goto err_setup_rx;
7248 
7249 	ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc);
7250 
7251 	if (vsi->type == ICE_VSI_PF) {
7252 		/* Notify the stack of the actual queue counts. */
7253 		err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq);
7254 		if (err)
7255 			goto err_set_qs;
7256 
7257 		err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq);
7258 		if (err)
7259 			goto err_set_qs;
7260 	}
7261 
7262 	err = ice_up_complete(vsi);
7263 	if (err)
7264 		goto err_up_complete;
7265 
7266 	return 0;
7267 
7268 err_up_complete:
7269 	ice_down(vsi);
7270 err_set_qs:
7271 	ice_vsi_free_irq(vsi);
7272 err_setup_rx:
7273 	ice_vsi_free_rx_rings(vsi);
7274 err_setup_tx:
7275 	ice_vsi_free_tx_rings(vsi);
7276 
7277 	return err;
7278 }
7279 
7280 /**
7281  * ice_vsi_release_all - Delete all VSIs
7282  * @pf: PF from which all VSIs are being removed
7283  */
7284 static void ice_vsi_release_all(struct ice_pf *pf)
7285 {
7286 	int err, i;
7287 
7288 	if (!pf->vsi)
7289 		return;
7290 
7291 	ice_for_each_vsi(pf, i) {
7292 		if (!pf->vsi[i])
7293 			continue;
7294 
7295 		if (pf->vsi[i]->type == ICE_VSI_CHNL)
7296 			continue;
7297 
7298 		err = ice_vsi_release(pf->vsi[i]);
7299 		if (err)
7300 			dev_dbg(ice_pf_to_dev(pf), "Failed to release pf->vsi[%d], err %d, vsi_num = %d\n",
7301 				i, err, pf->vsi[i]->vsi_num);
7302 	}
7303 }
7304 
7305 /**
7306  * ice_vsi_rebuild_by_type - Rebuild VSI of a given type
7307  * @pf: pointer to the PF instance
7308  * @type: VSI type to rebuild
7309  *
7310  * Iterates through the pf->vsi array and rebuilds VSIs of the requested type
7311  */
7312 static int ice_vsi_rebuild_by_type(struct ice_pf *pf, enum ice_vsi_type type)
7313 {
7314 	struct device *dev = ice_pf_to_dev(pf);
7315 	int i, err;
7316 
7317 	ice_for_each_vsi(pf, i) {
7318 		struct ice_vsi *vsi = pf->vsi[i];
7319 
7320 		if (!vsi || vsi->type != type)
7321 			continue;
7322 
7323 		/* rebuild the VSI */
7324 		err = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_INIT);
7325 		if (err) {
7326 			dev_err(dev, "rebuild VSI failed, err %d, VSI index %d, type %s\n",
7327 				err, vsi->idx, ice_vsi_type_str(type));
7328 			return err;
7329 		}
7330 
7331 		/* replay filters for the VSI */
7332 		err = ice_replay_vsi(&pf->hw, vsi->idx);
7333 		if (err) {
7334 			dev_err(dev, "replay VSI failed, error %d, VSI index %d, type %s\n",
7335 				err, vsi->idx, ice_vsi_type_str(type));
7336 			return err;
7337 		}
7338 
7339 		/* Re-map HW VSI number, using VSI handle that has been
7340 		 * previously validated in ice_replay_vsi() call above
7341 		 */
7342 		vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx);
7343 
7344 		/* enable the VSI */
7345 		err = ice_ena_vsi(vsi, false);
7346 		if (err) {
7347 			dev_err(dev, "enable VSI failed, err %d, VSI index %d, type %s\n",
7348 				err, vsi->idx, ice_vsi_type_str(type));
7349 			return err;
7350 		}
7351 
7352 		dev_info(dev, "VSI rebuilt. VSI index %d, type %s\n", vsi->idx,
7353 			 ice_vsi_type_str(type));
7354 	}
7355 
7356 	return 0;
7357 }
7358 
7359 /**
7360  * ice_update_pf_netdev_link - Update PF netdev link status
7361  * @pf: pointer to the PF instance
7362  */
7363 static void ice_update_pf_netdev_link(struct ice_pf *pf)
7364 {
7365 	bool link_up;
7366 	int i;
7367 
7368 	ice_for_each_vsi(pf, i) {
7369 		struct ice_vsi *vsi = pf->vsi[i];
7370 
7371 		if (!vsi || vsi->type != ICE_VSI_PF)
7372 			return;
7373 
7374 		ice_get_link_status(pf->vsi[i]->port_info, &link_up);
7375 		if (link_up) {
7376 			netif_carrier_on(pf->vsi[i]->netdev);
7377 			netif_tx_wake_all_queues(pf->vsi[i]->netdev);
7378 		} else {
7379 			netif_carrier_off(pf->vsi[i]->netdev);
7380 			netif_tx_stop_all_queues(pf->vsi[i]->netdev);
7381 		}
7382 	}
7383 }
7384 
7385 /**
7386  * ice_rebuild - rebuild after reset
7387  * @pf: PF to rebuild
7388  * @reset_type: type of reset
7389  *
7390  * Do not rebuild VF VSI in this flow because that is already handled via
7391  * ice_reset_all_vfs(). This is because requirements for resetting a VF after a
7392  * PFR/CORER/GLOBER/etc. are different than the normal flow. Also, we don't want
7393  * to reset/rebuild all the VF VSI twice.
7394  */
7395 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type)
7396 {
7397 	struct device *dev = ice_pf_to_dev(pf);
7398 	struct ice_hw *hw = &pf->hw;
7399 	bool dvm;
7400 	int err;
7401 
7402 	if (test_bit(ICE_DOWN, pf->state))
7403 		goto clear_recovery;
7404 
7405 	dev_dbg(dev, "rebuilding PF after reset_type=%d\n", reset_type);
7406 
7407 #define ICE_EMP_RESET_SLEEP_MS 5000
7408 	if (reset_type == ICE_RESET_EMPR) {
7409 		/* If an EMP reset has occurred, any previously pending flash
7410 		 * update will have completed. We no longer know whether or
7411 		 * not the NVM update EMP reset is restricted.
7412 		 */
7413 		pf->fw_emp_reset_disabled = false;
7414 
7415 		msleep(ICE_EMP_RESET_SLEEP_MS);
7416 	}
7417 
7418 	err = ice_init_all_ctrlq(hw);
7419 	if (err) {
7420 		dev_err(dev, "control queues init failed %d\n", err);
7421 		goto err_init_ctrlq;
7422 	}
7423 
7424 	/* if DDP was previously loaded successfully */
7425 	if (!ice_is_safe_mode(pf)) {
7426 		/* reload the SW DB of filter tables */
7427 		if (reset_type == ICE_RESET_PFR)
7428 			ice_fill_blk_tbls(hw);
7429 		else
7430 			/* Reload DDP Package after CORER/GLOBR reset */
7431 			ice_load_pkg(NULL, pf);
7432 	}
7433 
7434 	err = ice_clear_pf_cfg(hw);
7435 	if (err) {
7436 		dev_err(dev, "clear PF configuration failed %d\n", err);
7437 		goto err_init_ctrlq;
7438 	}
7439 
7440 	ice_clear_pxe_mode(hw);
7441 
7442 	err = ice_init_nvm(hw);
7443 	if (err) {
7444 		dev_err(dev, "ice_init_nvm failed %d\n", err);
7445 		goto err_init_ctrlq;
7446 	}
7447 
7448 	err = ice_get_caps(hw);
7449 	if (err) {
7450 		dev_err(dev, "ice_get_caps failed %d\n", err);
7451 		goto err_init_ctrlq;
7452 	}
7453 
7454 	err = ice_aq_set_mac_cfg(hw, ICE_AQ_SET_MAC_FRAME_SIZE_MAX, NULL);
7455 	if (err) {
7456 		dev_err(dev, "set_mac_cfg failed %d\n", err);
7457 		goto err_init_ctrlq;
7458 	}
7459 
7460 	dvm = ice_is_dvm_ena(hw);
7461 
7462 	err = ice_aq_set_port_params(pf->hw.port_info, dvm, NULL);
7463 	if (err)
7464 		goto err_init_ctrlq;
7465 
7466 	err = ice_sched_init_port(hw->port_info);
7467 	if (err)
7468 		goto err_sched_init_port;
7469 
7470 	/* start misc vector */
7471 	err = ice_req_irq_msix_misc(pf);
7472 	if (err) {
7473 		dev_err(dev, "misc vector setup failed: %d\n", err);
7474 		goto err_sched_init_port;
7475 	}
7476 
7477 	if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) {
7478 		wr32(hw, PFQF_FD_ENA, PFQF_FD_ENA_FD_ENA_M);
7479 		if (!rd32(hw, PFQF_FD_SIZE)) {
7480 			u16 unused, guar, b_effort;
7481 
7482 			guar = hw->func_caps.fd_fltr_guar;
7483 			b_effort = hw->func_caps.fd_fltr_best_effort;
7484 
7485 			/* force guaranteed filter pool for PF */
7486 			ice_alloc_fd_guar_item(hw, &unused, guar);
7487 			/* force shared filter pool for PF */
7488 			ice_alloc_fd_shrd_item(hw, &unused, b_effort);
7489 		}
7490 	}
7491 
7492 	if (test_bit(ICE_FLAG_DCB_ENA, pf->flags))
7493 		ice_dcb_rebuild(pf);
7494 
7495 	/* If the PF previously had enabled PTP, PTP init needs to happen before
7496 	 * the VSI rebuild. If not, this causes the PTP link status events to
7497 	 * fail.
7498 	 */
7499 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
7500 		ice_ptp_reset(pf);
7501 
7502 	if (ice_is_feature_supported(pf, ICE_F_GNSS))
7503 		ice_gnss_init(pf);
7504 
7505 	/* rebuild PF VSI */
7506 	err = ice_vsi_rebuild_by_type(pf, ICE_VSI_PF);
7507 	if (err) {
7508 		dev_err(dev, "PF VSI rebuild failed: %d\n", err);
7509 		goto err_vsi_rebuild;
7510 	}
7511 
7512 	/* configure PTP timestamping after VSI rebuild */
7513 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
7514 		ice_ptp_cfg_timestamp(pf, false);
7515 
7516 	err = ice_vsi_rebuild_by_type(pf, ICE_VSI_SWITCHDEV_CTRL);
7517 	if (err) {
7518 		dev_err(dev, "Switchdev CTRL VSI rebuild failed: %d\n", err);
7519 		goto err_vsi_rebuild;
7520 	}
7521 
7522 	if (reset_type == ICE_RESET_PFR) {
7523 		err = ice_rebuild_channels(pf);
7524 		if (err) {
7525 			dev_err(dev, "failed to rebuild and replay ADQ VSIs, err %d\n",
7526 				err);
7527 			goto err_vsi_rebuild;
7528 		}
7529 	}
7530 
7531 	/* If Flow Director is active */
7532 	if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) {
7533 		err = ice_vsi_rebuild_by_type(pf, ICE_VSI_CTRL);
7534 		if (err) {
7535 			dev_err(dev, "control VSI rebuild failed: %d\n", err);
7536 			goto err_vsi_rebuild;
7537 		}
7538 
7539 		/* replay HW Flow Director recipes */
7540 		if (hw->fdir_prof)
7541 			ice_fdir_replay_flows(hw);
7542 
7543 		/* replay Flow Director filters */
7544 		ice_fdir_replay_fltrs(pf);
7545 
7546 		ice_rebuild_arfs(pf);
7547 	}
7548 
7549 	ice_update_pf_netdev_link(pf);
7550 
7551 	/* tell the firmware we are up */
7552 	err = ice_send_version(pf);
7553 	if (err) {
7554 		dev_err(dev, "Rebuild failed due to error sending driver version: %d\n",
7555 			err);
7556 		goto err_vsi_rebuild;
7557 	}
7558 
7559 	ice_replay_post(hw);
7560 
7561 	/* if we get here, reset flow is successful */
7562 	clear_bit(ICE_RESET_FAILED, pf->state);
7563 
7564 	ice_plug_aux_dev(pf);
7565 	return;
7566 
7567 err_vsi_rebuild:
7568 err_sched_init_port:
7569 	ice_sched_cleanup_all(hw);
7570 err_init_ctrlq:
7571 	ice_shutdown_all_ctrlq(hw);
7572 	set_bit(ICE_RESET_FAILED, pf->state);
7573 clear_recovery:
7574 	/* set this bit in PF state to control service task scheduling */
7575 	set_bit(ICE_NEEDS_RESTART, pf->state);
7576 	dev_err(dev, "Rebuild failed, unload and reload driver\n");
7577 }
7578 
7579 /**
7580  * ice_change_mtu - NDO callback to change the MTU
7581  * @netdev: network interface device structure
7582  * @new_mtu: new value for maximum frame size
7583  *
7584  * Returns 0 on success, negative on failure
7585  */
7586 static int ice_change_mtu(struct net_device *netdev, int new_mtu)
7587 {
7588 	struct ice_netdev_priv *np = netdev_priv(netdev);
7589 	struct ice_vsi *vsi = np->vsi;
7590 	struct ice_pf *pf = vsi->back;
7591 	struct bpf_prog *prog;
7592 	u8 count = 0;
7593 	int err = 0;
7594 
7595 	if (new_mtu == (int)netdev->mtu) {
7596 		netdev_warn(netdev, "MTU is already %u\n", netdev->mtu);
7597 		return 0;
7598 	}
7599 
7600 	prog = vsi->xdp_prog;
7601 	if (prog && !prog->aux->xdp_has_frags) {
7602 		int frame_size = ice_max_xdp_frame_size(vsi);
7603 
7604 		if (new_mtu + ICE_ETH_PKT_HDR_PAD > frame_size) {
7605 			netdev_err(netdev, "max MTU for XDP usage is %d\n",
7606 				   frame_size - ICE_ETH_PKT_HDR_PAD);
7607 			return -EINVAL;
7608 		}
7609 	} else if (test_bit(ICE_FLAG_LEGACY_RX, pf->flags)) {
7610 		if (new_mtu + ICE_ETH_PKT_HDR_PAD > ICE_MAX_FRAME_LEGACY_RX) {
7611 			netdev_err(netdev, "Too big MTU for legacy-rx; Max is %d\n",
7612 				   ICE_MAX_FRAME_LEGACY_RX - ICE_ETH_PKT_HDR_PAD);
7613 			return -EINVAL;
7614 		}
7615 	}
7616 
7617 	/* if a reset is in progress, wait for some time for it to complete */
7618 	do {
7619 		if (ice_is_reset_in_progress(pf->state)) {
7620 			count++;
7621 			usleep_range(1000, 2000);
7622 		} else {
7623 			break;
7624 		}
7625 
7626 	} while (count < 100);
7627 
7628 	if (count == 100) {
7629 		netdev_err(netdev, "can't change MTU. Device is busy\n");
7630 		return -EBUSY;
7631 	}
7632 
7633 	netdev->mtu = (unsigned int)new_mtu;
7634 
7635 	/* if VSI is up, bring it down and then back up */
7636 	if (!test_and_set_bit(ICE_VSI_DOWN, vsi->state)) {
7637 		err = ice_down(vsi);
7638 		if (err) {
7639 			netdev_err(netdev, "change MTU if_down err %d\n", err);
7640 			return err;
7641 		}
7642 
7643 		err = ice_up(vsi);
7644 		if (err) {
7645 			netdev_err(netdev, "change MTU if_up err %d\n", err);
7646 			return err;
7647 		}
7648 	}
7649 
7650 	netdev_dbg(netdev, "changed MTU to %d\n", new_mtu);
7651 	set_bit(ICE_FLAG_MTU_CHANGED, pf->flags);
7652 
7653 	return err;
7654 }
7655 
7656 /**
7657  * ice_eth_ioctl - Access the hwtstamp interface
7658  * @netdev: network interface device structure
7659  * @ifr: interface request data
7660  * @cmd: ioctl command
7661  */
7662 static int ice_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7663 {
7664 	struct ice_netdev_priv *np = netdev_priv(netdev);
7665 	struct ice_pf *pf = np->vsi->back;
7666 
7667 	switch (cmd) {
7668 	case SIOCGHWTSTAMP:
7669 		return ice_ptp_get_ts_config(pf, ifr);
7670 	case SIOCSHWTSTAMP:
7671 		return ice_ptp_set_ts_config(pf, ifr);
7672 	default:
7673 		return -EOPNOTSUPP;
7674 	}
7675 }
7676 
7677 /**
7678  * ice_aq_str - convert AQ err code to a string
7679  * @aq_err: the AQ error code to convert
7680  */
7681 const char *ice_aq_str(enum ice_aq_err aq_err)
7682 {
7683 	switch (aq_err) {
7684 	case ICE_AQ_RC_OK:
7685 		return "OK";
7686 	case ICE_AQ_RC_EPERM:
7687 		return "ICE_AQ_RC_EPERM";
7688 	case ICE_AQ_RC_ENOENT:
7689 		return "ICE_AQ_RC_ENOENT";
7690 	case ICE_AQ_RC_ENOMEM:
7691 		return "ICE_AQ_RC_ENOMEM";
7692 	case ICE_AQ_RC_EBUSY:
7693 		return "ICE_AQ_RC_EBUSY";
7694 	case ICE_AQ_RC_EEXIST:
7695 		return "ICE_AQ_RC_EEXIST";
7696 	case ICE_AQ_RC_EINVAL:
7697 		return "ICE_AQ_RC_EINVAL";
7698 	case ICE_AQ_RC_ENOSPC:
7699 		return "ICE_AQ_RC_ENOSPC";
7700 	case ICE_AQ_RC_ENOSYS:
7701 		return "ICE_AQ_RC_ENOSYS";
7702 	case ICE_AQ_RC_EMODE:
7703 		return "ICE_AQ_RC_EMODE";
7704 	case ICE_AQ_RC_ENOSEC:
7705 		return "ICE_AQ_RC_ENOSEC";
7706 	case ICE_AQ_RC_EBADSIG:
7707 		return "ICE_AQ_RC_EBADSIG";
7708 	case ICE_AQ_RC_ESVN:
7709 		return "ICE_AQ_RC_ESVN";
7710 	case ICE_AQ_RC_EBADMAN:
7711 		return "ICE_AQ_RC_EBADMAN";
7712 	case ICE_AQ_RC_EBADBUF:
7713 		return "ICE_AQ_RC_EBADBUF";
7714 	}
7715 
7716 	return "ICE_AQ_RC_UNKNOWN";
7717 }
7718 
7719 /**
7720  * ice_set_rss_lut - Set RSS LUT
7721  * @vsi: Pointer to VSI structure
7722  * @lut: Lookup table
7723  * @lut_size: Lookup table size
7724  *
7725  * Returns 0 on success, negative on failure
7726  */
7727 int ice_set_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size)
7728 {
7729 	struct ice_aq_get_set_rss_lut_params params = {};
7730 	struct ice_hw *hw = &vsi->back->hw;
7731 	int status;
7732 
7733 	if (!lut)
7734 		return -EINVAL;
7735 
7736 	params.vsi_handle = vsi->idx;
7737 	params.lut_size = lut_size;
7738 	params.lut_type = vsi->rss_lut_type;
7739 	params.lut = lut;
7740 
7741 	status = ice_aq_set_rss_lut(hw, &params);
7742 	if (status)
7743 		dev_err(ice_pf_to_dev(vsi->back), "Cannot set RSS lut, err %d aq_err %s\n",
7744 			status, ice_aq_str(hw->adminq.sq_last_status));
7745 
7746 	return status;
7747 }
7748 
7749 /**
7750  * ice_set_rss_key - Set RSS key
7751  * @vsi: Pointer to the VSI structure
7752  * @seed: RSS hash seed
7753  *
7754  * Returns 0 on success, negative on failure
7755  */
7756 int ice_set_rss_key(struct ice_vsi *vsi, u8 *seed)
7757 {
7758 	struct ice_hw *hw = &vsi->back->hw;
7759 	int status;
7760 
7761 	if (!seed)
7762 		return -EINVAL;
7763 
7764 	status = ice_aq_set_rss_key(hw, vsi->idx, (struct ice_aqc_get_set_rss_keys *)seed);
7765 	if (status)
7766 		dev_err(ice_pf_to_dev(vsi->back), "Cannot set RSS key, err %d aq_err %s\n",
7767 			status, ice_aq_str(hw->adminq.sq_last_status));
7768 
7769 	return status;
7770 }
7771 
7772 /**
7773  * ice_get_rss_lut - Get RSS LUT
7774  * @vsi: Pointer to VSI structure
7775  * @lut: Buffer to store the lookup table entries
7776  * @lut_size: Size of buffer to store the lookup table entries
7777  *
7778  * Returns 0 on success, negative on failure
7779  */
7780 int ice_get_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size)
7781 {
7782 	struct ice_aq_get_set_rss_lut_params params = {};
7783 	struct ice_hw *hw = &vsi->back->hw;
7784 	int status;
7785 
7786 	if (!lut)
7787 		return -EINVAL;
7788 
7789 	params.vsi_handle = vsi->idx;
7790 	params.lut_size = lut_size;
7791 	params.lut_type = vsi->rss_lut_type;
7792 	params.lut = lut;
7793 
7794 	status = ice_aq_get_rss_lut(hw, &params);
7795 	if (status)
7796 		dev_err(ice_pf_to_dev(vsi->back), "Cannot get RSS lut, err %d aq_err %s\n",
7797 			status, ice_aq_str(hw->adminq.sq_last_status));
7798 
7799 	return status;
7800 }
7801 
7802 /**
7803  * ice_get_rss_key - Get RSS key
7804  * @vsi: Pointer to VSI structure
7805  * @seed: Buffer to store the key in
7806  *
7807  * Returns 0 on success, negative on failure
7808  */
7809 int ice_get_rss_key(struct ice_vsi *vsi, u8 *seed)
7810 {
7811 	struct ice_hw *hw = &vsi->back->hw;
7812 	int status;
7813 
7814 	if (!seed)
7815 		return -EINVAL;
7816 
7817 	status = ice_aq_get_rss_key(hw, vsi->idx, (struct ice_aqc_get_set_rss_keys *)seed);
7818 	if (status)
7819 		dev_err(ice_pf_to_dev(vsi->back), "Cannot get RSS key, err %d aq_err %s\n",
7820 			status, ice_aq_str(hw->adminq.sq_last_status));
7821 
7822 	return status;
7823 }
7824 
7825 /**
7826  * ice_bridge_getlink - Get the hardware bridge mode
7827  * @skb: skb buff
7828  * @pid: process ID
7829  * @seq: RTNL message seq
7830  * @dev: the netdev being configured
7831  * @filter_mask: filter mask passed in
7832  * @nlflags: netlink flags passed in
7833  *
7834  * Return the bridge mode (VEB/VEPA)
7835  */
7836 static int
7837 ice_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
7838 		   struct net_device *dev, u32 filter_mask, int nlflags)
7839 {
7840 	struct ice_netdev_priv *np = netdev_priv(dev);
7841 	struct ice_vsi *vsi = np->vsi;
7842 	struct ice_pf *pf = vsi->back;
7843 	u16 bmode;
7844 
7845 	bmode = pf->first_sw->bridge_mode;
7846 
7847 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bmode, 0, 0, nlflags,
7848 				       filter_mask, NULL);
7849 }
7850 
7851 /**
7852  * ice_vsi_update_bridge_mode - Update VSI for switching bridge mode (VEB/VEPA)
7853  * @vsi: Pointer to VSI structure
7854  * @bmode: Hardware bridge mode (VEB/VEPA)
7855  *
7856  * Returns 0 on success, negative on failure
7857  */
7858 static int ice_vsi_update_bridge_mode(struct ice_vsi *vsi, u16 bmode)
7859 {
7860 	struct ice_aqc_vsi_props *vsi_props;
7861 	struct ice_hw *hw = &vsi->back->hw;
7862 	struct ice_vsi_ctx *ctxt;
7863 	int ret;
7864 
7865 	vsi_props = &vsi->info;
7866 
7867 	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
7868 	if (!ctxt)
7869 		return -ENOMEM;
7870 
7871 	ctxt->info = vsi->info;
7872 
7873 	if (bmode == BRIDGE_MODE_VEB)
7874 		/* change from VEPA to VEB mode */
7875 		ctxt->info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB;
7876 	else
7877 		/* change from VEB to VEPA mode */
7878 		ctxt->info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_ALLOW_LB;
7879 	ctxt->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID);
7880 
7881 	ret = ice_update_vsi(hw, vsi->idx, ctxt, NULL);
7882 	if (ret) {
7883 		dev_err(ice_pf_to_dev(vsi->back), "update VSI for bridge mode failed, bmode = %d err %d aq_err %s\n",
7884 			bmode, ret, ice_aq_str(hw->adminq.sq_last_status));
7885 		goto out;
7886 	}
7887 	/* Update sw flags for book keeping */
7888 	vsi_props->sw_flags = ctxt->info.sw_flags;
7889 
7890 out:
7891 	kfree(ctxt);
7892 	return ret;
7893 }
7894 
7895 /**
7896  * ice_bridge_setlink - Set the hardware bridge mode
7897  * @dev: the netdev being configured
7898  * @nlh: RTNL message
7899  * @flags: bridge setlink flags
7900  * @extack: netlink extended ack
7901  *
7902  * Sets the bridge mode (VEB/VEPA) of the switch to which the netdev (VSI) is
7903  * hooked up to. Iterates through the PF VSI list and sets the loopback mode (if
7904  * not already set for all VSIs connected to this switch. And also update the
7905  * unicast switch filter rules for the corresponding switch of the netdev.
7906  */
7907 static int
7908 ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
7909 		   u16 __always_unused flags,
7910 		   struct netlink_ext_ack __always_unused *extack)
7911 {
7912 	struct ice_netdev_priv *np = netdev_priv(dev);
7913 	struct ice_pf *pf = np->vsi->back;
7914 	struct nlattr *attr, *br_spec;
7915 	struct ice_hw *hw = &pf->hw;
7916 	struct ice_sw *pf_sw;
7917 	int rem, v, err = 0;
7918 
7919 	pf_sw = pf->first_sw;
7920 	/* find the attribute in the netlink message */
7921 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
7922 
7923 	nla_for_each_nested(attr, br_spec, rem) {
7924 		__u16 mode;
7925 
7926 		if (nla_type(attr) != IFLA_BRIDGE_MODE)
7927 			continue;
7928 		mode = nla_get_u16(attr);
7929 		if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
7930 			return -EINVAL;
7931 		/* Continue  if bridge mode is not being flipped */
7932 		if (mode == pf_sw->bridge_mode)
7933 			continue;
7934 		/* Iterates through the PF VSI list and update the loopback
7935 		 * mode of the VSI
7936 		 */
7937 		ice_for_each_vsi(pf, v) {
7938 			if (!pf->vsi[v])
7939 				continue;
7940 			err = ice_vsi_update_bridge_mode(pf->vsi[v], mode);
7941 			if (err)
7942 				return err;
7943 		}
7944 
7945 		hw->evb_veb = (mode == BRIDGE_MODE_VEB);
7946 		/* Update the unicast switch filter rules for the corresponding
7947 		 * switch of the netdev
7948 		 */
7949 		err = ice_update_sw_rule_bridge_mode(hw);
7950 		if (err) {
7951 			netdev_err(dev, "switch rule update failed, mode = %d err %d aq_err %s\n",
7952 				   mode, err,
7953 				   ice_aq_str(hw->adminq.sq_last_status));
7954 			/* revert hw->evb_veb */
7955 			hw->evb_veb = (pf_sw->bridge_mode == BRIDGE_MODE_VEB);
7956 			return err;
7957 		}
7958 
7959 		pf_sw->bridge_mode = mode;
7960 	}
7961 
7962 	return 0;
7963 }
7964 
7965 /**
7966  * ice_tx_timeout - Respond to a Tx Hang
7967  * @netdev: network interface device structure
7968  * @txqueue: Tx queue
7969  */
7970 static void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue)
7971 {
7972 	struct ice_netdev_priv *np = netdev_priv(netdev);
7973 	struct ice_tx_ring *tx_ring = NULL;
7974 	struct ice_vsi *vsi = np->vsi;
7975 	struct ice_pf *pf = vsi->back;
7976 	u32 i;
7977 
7978 	pf->tx_timeout_count++;
7979 
7980 	/* Check if PFC is enabled for the TC to which the queue belongs
7981 	 * to. If yes then Tx timeout is not caused by a hung queue, no
7982 	 * need to reset and rebuild
7983 	 */
7984 	if (ice_is_pfc_causing_hung_q(pf, txqueue)) {
7985 		dev_info(ice_pf_to_dev(pf), "Fake Tx hang detected on queue %u, timeout caused by PFC storm\n",
7986 			 txqueue);
7987 		return;
7988 	}
7989 
7990 	/* now that we have an index, find the tx_ring struct */
7991 	ice_for_each_txq(vsi, i)
7992 		if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
7993 			if (txqueue == vsi->tx_rings[i]->q_index) {
7994 				tx_ring = vsi->tx_rings[i];
7995 				break;
7996 			}
7997 
7998 	/* Reset recovery level if enough time has elapsed after last timeout.
7999 	 * Also ensure no new reset action happens before next timeout period.
8000 	 */
8001 	if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ * 20)))
8002 		pf->tx_timeout_recovery_level = 1;
8003 	else if (time_before(jiffies, (pf->tx_timeout_last_recovery +
8004 				       netdev->watchdog_timeo)))
8005 		return;
8006 
8007 	if (tx_ring) {
8008 		struct ice_hw *hw = &pf->hw;
8009 		u32 head, val = 0;
8010 
8011 		head = (rd32(hw, QTX_COMM_HEAD(vsi->txq_map[txqueue])) &
8012 			QTX_COMM_HEAD_HEAD_M) >> QTX_COMM_HEAD_HEAD_S;
8013 		/* Read interrupt register */
8014 		val = rd32(hw, GLINT_DYN_CTL(tx_ring->q_vector->reg_idx));
8015 
8016 		netdev_info(netdev, "tx_timeout: VSI_num: %d, Q %u, NTC: 0x%x, HW_HEAD: 0x%x, NTU: 0x%x, INT: 0x%x\n",
8017 			    vsi->vsi_num, txqueue, tx_ring->next_to_clean,
8018 			    head, tx_ring->next_to_use, val);
8019 	}
8020 
8021 	pf->tx_timeout_last_recovery = jiffies;
8022 	netdev_info(netdev, "tx_timeout recovery level %d, txqueue %u\n",
8023 		    pf->tx_timeout_recovery_level, txqueue);
8024 
8025 	switch (pf->tx_timeout_recovery_level) {
8026 	case 1:
8027 		set_bit(ICE_PFR_REQ, pf->state);
8028 		break;
8029 	case 2:
8030 		set_bit(ICE_CORER_REQ, pf->state);
8031 		break;
8032 	case 3:
8033 		set_bit(ICE_GLOBR_REQ, pf->state);
8034 		break;
8035 	default:
8036 		netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in unrecoverable state.\n");
8037 		set_bit(ICE_DOWN, pf->state);
8038 		set_bit(ICE_VSI_NEEDS_RESTART, vsi->state);
8039 		set_bit(ICE_SERVICE_DIS, pf->state);
8040 		break;
8041 	}
8042 
8043 	ice_service_task_schedule(pf);
8044 	pf->tx_timeout_recovery_level++;
8045 }
8046 
8047 /**
8048  * ice_setup_tc_cls_flower - flower classifier offloads
8049  * @np: net device to configure
8050  * @filter_dev: device on which filter is added
8051  * @cls_flower: offload data
8052  */
8053 static int
8054 ice_setup_tc_cls_flower(struct ice_netdev_priv *np,
8055 			struct net_device *filter_dev,
8056 			struct flow_cls_offload *cls_flower)
8057 {
8058 	struct ice_vsi *vsi = np->vsi;
8059 
8060 	if (cls_flower->common.chain_index)
8061 		return -EOPNOTSUPP;
8062 
8063 	switch (cls_flower->command) {
8064 	case FLOW_CLS_REPLACE:
8065 		return ice_add_cls_flower(filter_dev, vsi, cls_flower);
8066 	case FLOW_CLS_DESTROY:
8067 		return ice_del_cls_flower(vsi, cls_flower);
8068 	default:
8069 		return -EINVAL;
8070 	}
8071 }
8072 
8073 /**
8074  * ice_setup_tc_block_cb - callback handler registered for TC block
8075  * @type: TC SETUP type
8076  * @type_data: TC flower offload data that contains user input
8077  * @cb_priv: netdev private data
8078  */
8079 static int
8080 ice_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
8081 {
8082 	struct ice_netdev_priv *np = cb_priv;
8083 
8084 	switch (type) {
8085 	case TC_SETUP_CLSFLOWER:
8086 		return ice_setup_tc_cls_flower(np, np->vsi->netdev,
8087 					       type_data);
8088 	default:
8089 		return -EOPNOTSUPP;
8090 	}
8091 }
8092 
8093 /**
8094  * ice_validate_mqprio_qopt - Validate TCF input parameters
8095  * @vsi: Pointer to VSI
8096  * @mqprio_qopt: input parameters for mqprio queue configuration
8097  *
8098  * This function validates MQPRIO params, such as qcount (power of 2 wherever
8099  * needed), and make sure user doesn't specify qcount and BW rate limit
8100  * for TCs, which are more than "num_tc"
8101  */
8102 static int
8103 ice_validate_mqprio_qopt(struct ice_vsi *vsi,
8104 			 struct tc_mqprio_qopt_offload *mqprio_qopt)
8105 {
8106 	u64 sum_max_rate = 0, sum_min_rate = 0;
8107 	int non_power_of_2_qcount = 0;
8108 	struct ice_pf *pf = vsi->back;
8109 	int max_rss_q_cnt = 0;
8110 	struct device *dev;
8111 	int i, speed;
8112 	u8 num_tc;
8113 
8114 	if (vsi->type != ICE_VSI_PF)
8115 		return -EINVAL;
8116 
8117 	if (mqprio_qopt->qopt.offset[0] != 0 ||
8118 	    mqprio_qopt->qopt.num_tc < 1 ||
8119 	    mqprio_qopt->qopt.num_tc > ICE_CHNL_MAX_TC)
8120 		return -EINVAL;
8121 
8122 	dev = ice_pf_to_dev(pf);
8123 	vsi->ch_rss_size = 0;
8124 	num_tc = mqprio_qopt->qopt.num_tc;
8125 
8126 	for (i = 0; num_tc; i++) {
8127 		int qcount = mqprio_qopt->qopt.count[i];
8128 		u64 max_rate, min_rate, rem;
8129 
8130 		if (!qcount)
8131 			return -EINVAL;
8132 
8133 		if (is_power_of_2(qcount)) {
8134 			if (non_power_of_2_qcount &&
8135 			    qcount > non_power_of_2_qcount) {
8136 				dev_err(dev, "qcount[%d] cannot be greater than non power of 2 qcount[%d]\n",
8137 					qcount, non_power_of_2_qcount);
8138 				return -EINVAL;
8139 			}
8140 			if (qcount > max_rss_q_cnt)
8141 				max_rss_q_cnt = qcount;
8142 		} else {
8143 			if (non_power_of_2_qcount &&
8144 			    qcount != non_power_of_2_qcount) {
8145 				dev_err(dev, "Only one non power of 2 qcount allowed[%d,%d]\n",
8146 					qcount, non_power_of_2_qcount);
8147 				return -EINVAL;
8148 			}
8149 			if (qcount < max_rss_q_cnt) {
8150 				dev_err(dev, "non power of 2 qcount[%d] cannot be less than other qcount[%d]\n",
8151 					qcount, max_rss_q_cnt);
8152 				return -EINVAL;
8153 			}
8154 			max_rss_q_cnt = qcount;
8155 			non_power_of_2_qcount = qcount;
8156 		}
8157 
8158 		/* TC command takes input in K/N/Gbps or K/M/Gbit etc but
8159 		 * converts the bandwidth rate limit into Bytes/s when
8160 		 * passing it down to the driver. So convert input bandwidth
8161 		 * from Bytes/s to Kbps
8162 		 */
8163 		max_rate = mqprio_qopt->max_rate[i];
8164 		max_rate = div_u64(max_rate, ICE_BW_KBPS_DIVISOR);
8165 		sum_max_rate += max_rate;
8166 
8167 		/* min_rate is minimum guaranteed rate and it can't be zero */
8168 		min_rate = mqprio_qopt->min_rate[i];
8169 		min_rate = div_u64(min_rate, ICE_BW_KBPS_DIVISOR);
8170 		sum_min_rate += min_rate;
8171 
8172 		if (min_rate && min_rate < ICE_MIN_BW_LIMIT) {
8173 			dev_err(dev, "TC%d: min_rate(%llu Kbps) < %u Kbps\n", i,
8174 				min_rate, ICE_MIN_BW_LIMIT);
8175 			return -EINVAL;
8176 		}
8177 
8178 		iter_div_u64_rem(min_rate, ICE_MIN_BW_LIMIT, &rem);
8179 		if (rem) {
8180 			dev_err(dev, "TC%d: Min Rate not multiple of %u Kbps",
8181 				i, ICE_MIN_BW_LIMIT);
8182 			return -EINVAL;
8183 		}
8184 
8185 		iter_div_u64_rem(max_rate, ICE_MIN_BW_LIMIT, &rem);
8186 		if (rem) {
8187 			dev_err(dev, "TC%d: Max Rate not multiple of %u Kbps",
8188 				i, ICE_MIN_BW_LIMIT);
8189 			return -EINVAL;
8190 		}
8191 
8192 		/* min_rate can't be more than max_rate, except when max_rate
8193 		 * is zero (implies max_rate sought is max line rate). In such
8194 		 * a case min_rate can be more than max.
8195 		 */
8196 		if (max_rate && min_rate > max_rate) {
8197 			dev_err(dev, "min_rate %llu Kbps can't be more than max_rate %llu Kbps\n",
8198 				min_rate, max_rate);
8199 			return -EINVAL;
8200 		}
8201 
8202 		if (i >= mqprio_qopt->qopt.num_tc - 1)
8203 			break;
8204 		if (mqprio_qopt->qopt.offset[i + 1] !=
8205 		    (mqprio_qopt->qopt.offset[i] + qcount))
8206 			return -EINVAL;
8207 	}
8208 	if (vsi->num_rxq <
8209 	    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
8210 		return -EINVAL;
8211 	if (vsi->num_txq <
8212 	    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
8213 		return -EINVAL;
8214 
8215 	speed = ice_get_link_speed_kbps(vsi);
8216 	if (sum_max_rate && sum_max_rate > (u64)speed) {
8217 		dev_err(dev, "Invalid max Tx rate(%llu) Kbps > speed(%u) Kbps specified\n",
8218 			sum_max_rate, speed);
8219 		return -EINVAL;
8220 	}
8221 	if (sum_min_rate && sum_min_rate > (u64)speed) {
8222 		dev_err(dev, "Invalid min Tx rate(%llu) Kbps > speed (%u) Kbps specified\n",
8223 			sum_min_rate, speed);
8224 		return -EINVAL;
8225 	}
8226 
8227 	/* make sure vsi->ch_rss_size is set correctly based on TC's qcount */
8228 	vsi->ch_rss_size = max_rss_q_cnt;
8229 
8230 	return 0;
8231 }
8232 
8233 /**
8234  * ice_add_vsi_to_fdir - add a VSI to the flow director group for PF
8235  * @pf: ptr to PF device
8236  * @vsi: ptr to VSI
8237  */
8238 static int ice_add_vsi_to_fdir(struct ice_pf *pf, struct ice_vsi *vsi)
8239 {
8240 	struct device *dev = ice_pf_to_dev(pf);
8241 	bool added = false;
8242 	struct ice_hw *hw;
8243 	int flow;
8244 
8245 	if (!(vsi->num_gfltr || vsi->num_bfltr))
8246 		return -EINVAL;
8247 
8248 	hw = &pf->hw;
8249 	for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++) {
8250 		struct ice_fd_hw_prof *prof;
8251 		int tun, status;
8252 		u64 entry_h;
8253 
8254 		if (!(hw->fdir_prof && hw->fdir_prof[flow] &&
8255 		      hw->fdir_prof[flow]->cnt))
8256 			continue;
8257 
8258 		for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
8259 			enum ice_flow_priority prio;
8260 			u64 prof_id;
8261 
8262 			/* add this VSI to FDir profile for this flow */
8263 			prio = ICE_FLOW_PRIO_NORMAL;
8264 			prof = hw->fdir_prof[flow];
8265 			prof_id = flow + tun * ICE_FLTR_PTYPE_MAX;
8266 			status = ice_flow_add_entry(hw, ICE_BLK_FD, prof_id,
8267 						    prof->vsi_h[0], vsi->idx,
8268 						    prio, prof->fdir_seg[tun],
8269 						    &entry_h);
8270 			if (status) {
8271 				dev_err(dev, "channel VSI idx %d, not able to add to group %d\n",
8272 					vsi->idx, flow);
8273 				continue;
8274 			}
8275 
8276 			prof->entry_h[prof->cnt][tun] = entry_h;
8277 		}
8278 
8279 		/* store VSI for filter replay and delete */
8280 		prof->vsi_h[prof->cnt] = vsi->idx;
8281 		prof->cnt++;
8282 
8283 		added = true;
8284 		dev_dbg(dev, "VSI idx %d added to fdir group %d\n", vsi->idx,
8285 			flow);
8286 	}
8287 
8288 	if (!added)
8289 		dev_dbg(dev, "VSI idx %d not added to fdir groups\n", vsi->idx);
8290 
8291 	return 0;
8292 }
8293 
8294 /**
8295  * ice_add_channel - add a channel by adding VSI
8296  * @pf: ptr to PF device
8297  * @sw_id: underlying HW switching element ID
8298  * @ch: ptr to channel structure
8299  *
8300  * Add a channel (VSI) using add_vsi and queue_map
8301  */
8302 static int ice_add_channel(struct ice_pf *pf, u16 sw_id, struct ice_channel *ch)
8303 {
8304 	struct device *dev = ice_pf_to_dev(pf);
8305 	struct ice_vsi *vsi;
8306 
8307 	if (ch->type != ICE_VSI_CHNL) {
8308 		dev_err(dev, "add new VSI failed, ch->type %d\n", ch->type);
8309 		return -EINVAL;
8310 	}
8311 
8312 	vsi = ice_chnl_vsi_setup(pf, pf->hw.port_info, ch);
8313 	if (!vsi || vsi->type != ICE_VSI_CHNL) {
8314 		dev_err(dev, "create chnl VSI failure\n");
8315 		return -EINVAL;
8316 	}
8317 
8318 	ice_add_vsi_to_fdir(pf, vsi);
8319 
8320 	ch->sw_id = sw_id;
8321 	ch->vsi_num = vsi->vsi_num;
8322 	ch->info.mapping_flags = vsi->info.mapping_flags;
8323 	ch->ch_vsi = vsi;
8324 	/* set the back pointer of channel for newly created VSI */
8325 	vsi->ch = ch;
8326 
8327 	memcpy(&ch->info.q_mapping, &vsi->info.q_mapping,
8328 	       sizeof(vsi->info.q_mapping));
8329 	memcpy(&ch->info.tc_mapping, vsi->info.tc_mapping,
8330 	       sizeof(vsi->info.tc_mapping));
8331 
8332 	return 0;
8333 }
8334 
8335 /**
8336  * ice_chnl_cfg_res
8337  * @vsi: the VSI being setup
8338  * @ch: ptr to channel structure
8339  *
8340  * Configure channel specific resources such as rings, vector.
8341  */
8342 static void ice_chnl_cfg_res(struct ice_vsi *vsi, struct ice_channel *ch)
8343 {
8344 	int i;
8345 
8346 	for (i = 0; i < ch->num_txq; i++) {
8347 		struct ice_q_vector *tx_q_vector, *rx_q_vector;
8348 		struct ice_ring_container *rc;
8349 		struct ice_tx_ring *tx_ring;
8350 		struct ice_rx_ring *rx_ring;
8351 
8352 		tx_ring = vsi->tx_rings[ch->base_q + i];
8353 		rx_ring = vsi->rx_rings[ch->base_q + i];
8354 		if (!tx_ring || !rx_ring)
8355 			continue;
8356 
8357 		/* setup ring being channel enabled */
8358 		tx_ring->ch = ch;
8359 		rx_ring->ch = ch;
8360 
8361 		/* following code block sets up vector specific attributes */
8362 		tx_q_vector = tx_ring->q_vector;
8363 		rx_q_vector = rx_ring->q_vector;
8364 		if (!tx_q_vector && !rx_q_vector)
8365 			continue;
8366 
8367 		if (tx_q_vector) {
8368 			tx_q_vector->ch = ch;
8369 			/* setup Tx and Rx ITR setting if DIM is off */
8370 			rc = &tx_q_vector->tx;
8371 			if (!ITR_IS_DYNAMIC(rc))
8372 				ice_write_itr(rc, rc->itr_setting);
8373 		}
8374 		if (rx_q_vector) {
8375 			rx_q_vector->ch = ch;
8376 			/* setup Tx and Rx ITR setting if DIM is off */
8377 			rc = &rx_q_vector->rx;
8378 			if (!ITR_IS_DYNAMIC(rc))
8379 				ice_write_itr(rc, rc->itr_setting);
8380 		}
8381 	}
8382 
8383 	/* it is safe to assume that, if channel has non-zero num_t[r]xq, then
8384 	 * GLINT_ITR register would have written to perform in-context
8385 	 * update, hence perform flush
8386 	 */
8387 	if (ch->num_txq || ch->num_rxq)
8388 		ice_flush(&vsi->back->hw);
8389 }
8390 
8391 /**
8392  * ice_cfg_chnl_all_res - configure channel resources
8393  * @vsi: pte to main_vsi
8394  * @ch: ptr to channel structure
8395  *
8396  * This function configures channel specific resources such as flow-director
8397  * counter index, and other resources such as queues, vectors, ITR settings
8398  */
8399 static void
8400 ice_cfg_chnl_all_res(struct ice_vsi *vsi, struct ice_channel *ch)
8401 {
8402 	/* configure channel (aka ADQ) resources such as queues, vectors,
8403 	 * ITR settings for channel specific vectors and anything else
8404 	 */
8405 	ice_chnl_cfg_res(vsi, ch);
8406 }
8407 
8408 /**
8409  * ice_setup_hw_channel - setup new channel
8410  * @pf: ptr to PF device
8411  * @vsi: the VSI being setup
8412  * @ch: ptr to channel structure
8413  * @sw_id: underlying HW switching element ID
8414  * @type: type of channel to be created (VMDq2/VF)
8415  *
8416  * Setup new channel (VSI) based on specified type (VMDq2/VF)
8417  * and configures Tx rings accordingly
8418  */
8419 static int
8420 ice_setup_hw_channel(struct ice_pf *pf, struct ice_vsi *vsi,
8421 		     struct ice_channel *ch, u16 sw_id, u8 type)
8422 {
8423 	struct device *dev = ice_pf_to_dev(pf);
8424 	int ret;
8425 
8426 	ch->base_q = vsi->next_base_q;
8427 	ch->type = type;
8428 
8429 	ret = ice_add_channel(pf, sw_id, ch);
8430 	if (ret) {
8431 		dev_err(dev, "failed to add_channel using sw_id %u\n", sw_id);
8432 		return ret;
8433 	}
8434 
8435 	/* configure/setup ADQ specific resources */
8436 	ice_cfg_chnl_all_res(vsi, ch);
8437 
8438 	/* make sure to update the next_base_q so that subsequent channel's
8439 	 * (aka ADQ) VSI queue map is correct
8440 	 */
8441 	vsi->next_base_q = vsi->next_base_q + ch->num_rxq;
8442 	dev_dbg(dev, "added channel: vsi_num %u, num_rxq %u\n", ch->vsi_num,
8443 		ch->num_rxq);
8444 
8445 	return 0;
8446 }
8447 
8448 /**
8449  * ice_setup_channel - setup new channel using uplink element
8450  * @pf: ptr to PF device
8451  * @vsi: the VSI being setup
8452  * @ch: ptr to channel structure
8453  *
8454  * Setup new channel (VSI) based on specified type (VMDq2/VF)
8455  * and uplink switching element
8456  */
8457 static bool
8458 ice_setup_channel(struct ice_pf *pf, struct ice_vsi *vsi,
8459 		  struct ice_channel *ch)
8460 {
8461 	struct device *dev = ice_pf_to_dev(pf);
8462 	u16 sw_id;
8463 	int ret;
8464 
8465 	if (vsi->type != ICE_VSI_PF) {
8466 		dev_err(dev, "unsupported parent VSI type(%d)\n", vsi->type);
8467 		return false;
8468 	}
8469 
8470 	sw_id = pf->first_sw->sw_id;
8471 
8472 	/* create channel (VSI) */
8473 	ret = ice_setup_hw_channel(pf, vsi, ch, sw_id, ICE_VSI_CHNL);
8474 	if (ret) {
8475 		dev_err(dev, "failed to setup hw_channel\n");
8476 		return false;
8477 	}
8478 	dev_dbg(dev, "successfully created channel()\n");
8479 
8480 	return ch->ch_vsi ? true : false;
8481 }
8482 
8483 /**
8484  * ice_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
8485  * @vsi: VSI to be configured
8486  * @max_tx_rate: max Tx rate in Kbps to be configured as maximum BW limit
8487  * @min_tx_rate: min Tx rate in Kbps to be configured as minimum BW limit
8488  */
8489 static int
8490 ice_set_bw_limit(struct ice_vsi *vsi, u64 max_tx_rate, u64 min_tx_rate)
8491 {
8492 	int err;
8493 
8494 	err = ice_set_min_bw_limit(vsi, min_tx_rate);
8495 	if (err)
8496 		return err;
8497 
8498 	return ice_set_max_bw_limit(vsi, max_tx_rate);
8499 }
8500 
8501 /**
8502  * ice_create_q_channel - function to create channel
8503  * @vsi: VSI to be configured
8504  * @ch: ptr to channel (it contains channel specific params)
8505  *
8506  * This function creates channel (VSI) using num_queues specified by user,
8507  * reconfigs RSS if needed.
8508  */
8509 static int ice_create_q_channel(struct ice_vsi *vsi, struct ice_channel *ch)
8510 {
8511 	struct ice_pf *pf = vsi->back;
8512 	struct device *dev;
8513 
8514 	if (!ch)
8515 		return -EINVAL;
8516 
8517 	dev = ice_pf_to_dev(pf);
8518 	if (!ch->num_txq || !ch->num_rxq) {
8519 		dev_err(dev, "Invalid num_queues requested: %d\n", ch->num_rxq);
8520 		return -EINVAL;
8521 	}
8522 
8523 	if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_txq) {
8524 		dev_err(dev, "cnt_q_avail (%u) less than num_queues %d\n",
8525 			vsi->cnt_q_avail, ch->num_txq);
8526 		return -EINVAL;
8527 	}
8528 
8529 	if (!ice_setup_channel(pf, vsi, ch)) {
8530 		dev_info(dev, "Failed to setup channel\n");
8531 		return -EINVAL;
8532 	}
8533 	/* configure BW rate limit */
8534 	if (ch->ch_vsi && (ch->max_tx_rate || ch->min_tx_rate)) {
8535 		int ret;
8536 
8537 		ret = ice_set_bw_limit(ch->ch_vsi, ch->max_tx_rate,
8538 				       ch->min_tx_rate);
8539 		if (ret)
8540 			dev_err(dev, "failed to set Tx rate of %llu Kbps for VSI(%u)\n",
8541 				ch->max_tx_rate, ch->ch_vsi->vsi_num);
8542 		else
8543 			dev_dbg(dev, "set Tx rate of %llu Kbps for VSI(%u)\n",
8544 				ch->max_tx_rate, ch->ch_vsi->vsi_num);
8545 	}
8546 
8547 	vsi->cnt_q_avail -= ch->num_txq;
8548 
8549 	return 0;
8550 }
8551 
8552 /**
8553  * ice_rem_all_chnl_fltrs - removes all channel filters
8554  * @pf: ptr to PF, TC-flower based filter are tracked at PF level
8555  *
8556  * Remove all advanced switch filters only if they are channel specific
8557  * tc-flower based filter
8558  */
8559 static void ice_rem_all_chnl_fltrs(struct ice_pf *pf)
8560 {
8561 	struct ice_tc_flower_fltr *fltr;
8562 	struct hlist_node *node;
8563 
8564 	/* to remove all channel filters, iterate an ordered list of filters */
8565 	hlist_for_each_entry_safe(fltr, node,
8566 				  &pf->tc_flower_fltr_list,
8567 				  tc_flower_node) {
8568 		struct ice_rule_query_data rule;
8569 		int status;
8570 
8571 		/* for now process only channel specific filters */
8572 		if (!ice_is_chnl_fltr(fltr))
8573 			continue;
8574 
8575 		rule.rid = fltr->rid;
8576 		rule.rule_id = fltr->rule_id;
8577 		rule.vsi_handle = fltr->dest_vsi_handle;
8578 		status = ice_rem_adv_rule_by_id(&pf->hw, &rule);
8579 		if (status) {
8580 			if (status == -ENOENT)
8581 				dev_dbg(ice_pf_to_dev(pf), "TC flower filter (rule_id %u) does not exist\n",
8582 					rule.rule_id);
8583 			else
8584 				dev_err(ice_pf_to_dev(pf), "failed to delete TC flower filter, status %d\n",
8585 					status);
8586 		} else if (fltr->dest_vsi) {
8587 			/* update advanced switch filter count */
8588 			if (fltr->dest_vsi->type == ICE_VSI_CHNL) {
8589 				u32 flags = fltr->flags;
8590 
8591 				fltr->dest_vsi->num_chnl_fltr--;
8592 				if (flags & (ICE_TC_FLWR_FIELD_DST_MAC |
8593 					     ICE_TC_FLWR_FIELD_ENC_DST_MAC))
8594 					pf->num_dmac_chnl_fltrs--;
8595 			}
8596 		}
8597 
8598 		hlist_del(&fltr->tc_flower_node);
8599 		kfree(fltr);
8600 	}
8601 }
8602 
8603 /**
8604  * ice_remove_q_channels - Remove queue channels for the TCs
8605  * @vsi: VSI to be configured
8606  * @rem_fltr: delete advanced switch filter or not
8607  *
8608  * Remove queue channels for the TCs
8609  */
8610 static void ice_remove_q_channels(struct ice_vsi *vsi, bool rem_fltr)
8611 {
8612 	struct ice_channel *ch, *ch_tmp;
8613 	struct ice_pf *pf = vsi->back;
8614 	int i;
8615 
8616 	/* remove all tc-flower based filter if they are channel filters only */
8617 	if (rem_fltr)
8618 		ice_rem_all_chnl_fltrs(pf);
8619 
8620 	/* remove ntuple filters since queue configuration is being changed */
8621 	if  (vsi->netdev->features & NETIF_F_NTUPLE) {
8622 		struct ice_hw *hw = &pf->hw;
8623 
8624 		mutex_lock(&hw->fdir_fltr_lock);
8625 		ice_fdir_del_all_fltrs(vsi);
8626 		mutex_unlock(&hw->fdir_fltr_lock);
8627 	}
8628 
8629 	/* perform cleanup for channels if they exist */
8630 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
8631 		struct ice_vsi *ch_vsi;
8632 
8633 		list_del(&ch->list);
8634 		ch_vsi = ch->ch_vsi;
8635 		if (!ch_vsi) {
8636 			kfree(ch);
8637 			continue;
8638 		}
8639 
8640 		/* Reset queue contexts */
8641 		for (i = 0; i < ch->num_rxq; i++) {
8642 			struct ice_tx_ring *tx_ring;
8643 			struct ice_rx_ring *rx_ring;
8644 
8645 			tx_ring = vsi->tx_rings[ch->base_q + i];
8646 			rx_ring = vsi->rx_rings[ch->base_q + i];
8647 			if (tx_ring) {
8648 				tx_ring->ch = NULL;
8649 				if (tx_ring->q_vector)
8650 					tx_ring->q_vector->ch = NULL;
8651 			}
8652 			if (rx_ring) {
8653 				rx_ring->ch = NULL;
8654 				if (rx_ring->q_vector)
8655 					rx_ring->q_vector->ch = NULL;
8656 			}
8657 		}
8658 
8659 		/* Release FD resources for the channel VSI */
8660 		ice_fdir_rem_adq_chnl(&pf->hw, ch->ch_vsi->idx);
8661 
8662 		/* clear the VSI from scheduler tree */
8663 		ice_rm_vsi_lan_cfg(ch->ch_vsi->port_info, ch->ch_vsi->idx);
8664 
8665 		/* Delete VSI from FW, PF and HW VSI arrays */
8666 		ice_vsi_delete(ch->ch_vsi);
8667 
8668 		/* free the channel */
8669 		kfree(ch);
8670 	}
8671 
8672 	/* clear the channel VSI map which is stored in main VSI */
8673 	ice_for_each_chnl_tc(i)
8674 		vsi->tc_map_vsi[i] = NULL;
8675 
8676 	/* reset main VSI's all TC information */
8677 	vsi->all_enatc = 0;
8678 	vsi->all_numtc = 0;
8679 }
8680 
8681 /**
8682  * ice_rebuild_channels - rebuild channel
8683  * @pf: ptr to PF
8684  *
8685  * Recreate channel VSIs and replay filters
8686  */
8687 static int ice_rebuild_channels(struct ice_pf *pf)
8688 {
8689 	struct device *dev = ice_pf_to_dev(pf);
8690 	struct ice_vsi *main_vsi;
8691 	bool rem_adv_fltr = true;
8692 	struct ice_channel *ch;
8693 	struct ice_vsi *vsi;
8694 	int tc_idx = 1;
8695 	int i, err;
8696 
8697 	main_vsi = ice_get_main_vsi(pf);
8698 	if (!main_vsi)
8699 		return 0;
8700 
8701 	if (!test_bit(ICE_FLAG_TC_MQPRIO, pf->flags) ||
8702 	    main_vsi->old_numtc == 1)
8703 		return 0; /* nothing to be done */
8704 
8705 	/* reconfigure main VSI based on old value of TC and cached values
8706 	 * for MQPRIO opts
8707 	 */
8708 	err = ice_vsi_cfg_tc(main_vsi, main_vsi->old_ena_tc);
8709 	if (err) {
8710 		dev_err(dev, "failed configuring TC(ena_tc:0x%02x) for HW VSI=%u\n",
8711 			main_vsi->old_ena_tc, main_vsi->vsi_num);
8712 		return err;
8713 	}
8714 
8715 	/* rebuild ADQ VSIs */
8716 	ice_for_each_vsi(pf, i) {
8717 		enum ice_vsi_type type;
8718 
8719 		vsi = pf->vsi[i];
8720 		if (!vsi || vsi->type != ICE_VSI_CHNL)
8721 			continue;
8722 
8723 		type = vsi->type;
8724 
8725 		/* rebuild ADQ VSI */
8726 		err = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_INIT);
8727 		if (err) {
8728 			dev_err(dev, "VSI (type:%s) at index %d rebuild failed, err %d\n",
8729 				ice_vsi_type_str(type), vsi->idx, err);
8730 			goto cleanup;
8731 		}
8732 
8733 		/* Re-map HW VSI number, using VSI handle that has been
8734 		 * previously validated in ice_replay_vsi() call above
8735 		 */
8736 		vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx);
8737 
8738 		/* replay filters for the VSI */
8739 		err = ice_replay_vsi(&pf->hw, vsi->idx);
8740 		if (err) {
8741 			dev_err(dev, "VSI (type:%s) replay failed, err %d, VSI index %d\n",
8742 				ice_vsi_type_str(type), err, vsi->idx);
8743 			rem_adv_fltr = false;
8744 			goto cleanup;
8745 		}
8746 		dev_info(dev, "VSI (type:%s) at index %d rebuilt successfully\n",
8747 			 ice_vsi_type_str(type), vsi->idx);
8748 
8749 		/* store ADQ VSI at correct TC index in main VSI's
8750 		 * map of TC to VSI
8751 		 */
8752 		main_vsi->tc_map_vsi[tc_idx++] = vsi;
8753 	}
8754 
8755 	/* ADQ VSI(s) has been rebuilt successfully, so setup
8756 	 * channel for main VSI's Tx and Rx rings
8757 	 */
8758 	list_for_each_entry(ch, &main_vsi->ch_list, list) {
8759 		struct ice_vsi *ch_vsi;
8760 
8761 		ch_vsi = ch->ch_vsi;
8762 		if (!ch_vsi)
8763 			continue;
8764 
8765 		/* reconfig channel resources */
8766 		ice_cfg_chnl_all_res(main_vsi, ch);
8767 
8768 		/* replay BW rate limit if it is non-zero */
8769 		if (!ch->max_tx_rate && !ch->min_tx_rate)
8770 			continue;
8771 
8772 		err = ice_set_bw_limit(ch_vsi, ch->max_tx_rate,
8773 				       ch->min_tx_rate);
8774 		if (err)
8775 			dev_err(dev, "failed (err:%d) to rebuild BW rate limit, max_tx_rate: %llu Kbps, min_tx_rate: %llu Kbps for VSI(%u)\n",
8776 				err, ch->max_tx_rate, ch->min_tx_rate,
8777 				ch_vsi->vsi_num);
8778 		else
8779 			dev_dbg(dev, "successfully rebuild BW rate limit, max_tx_rate: %llu Kbps, min_tx_rate: %llu Kbps for VSI(%u)\n",
8780 				ch->max_tx_rate, ch->min_tx_rate,
8781 				ch_vsi->vsi_num);
8782 	}
8783 
8784 	/* reconfig RSS for main VSI */
8785 	if (main_vsi->ch_rss_size)
8786 		ice_vsi_cfg_rss_lut_key(main_vsi);
8787 
8788 	return 0;
8789 
8790 cleanup:
8791 	ice_remove_q_channels(main_vsi, rem_adv_fltr);
8792 	return err;
8793 }
8794 
8795 /**
8796  * ice_create_q_channels - Add queue channel for the given TCs
8797  * @vsi: VSI to be configured
8798  *
8799  * Configures queue channel mapping to the given TCs
8800  */
8801 static int ice_create_q_channels(struct ice_vsi *vsi)
8802 {
8803 	struct ice_pf *pf = vsi->back;
8804 	struct ice_channel *ch;
8805 	int ret = 0, i;
8806 
8807 	ice_for_each_chnl_tc(i) {
8808 		if (!(vsi->all_enatc & BIT(i)))
8809 			continue;
8810 
8811 		ch = kzalloc(sizeof(*ch), GFP_KERNEL);
8812 		if (!ch) {
8813 			ret = -ENOMEM;
8814 			goto err_free;
8815 		}
8816 		INIT_LIST_HEAD(&ch->list);
8817 		ch->num_rxq = vsi->mqprio_qopt.qopt.count[i];
8818 		ch->num_txq = vsi->mqprio_qopt.qopt.count[i];
8819 		ch->base_q = vsi->mqprio_qopt.qopt.offset[i];
8820 		ch->max_tx_rate = vsi->mqprio_qopt.max_rate[i];
8821 		ch->min_tx_rate = vsi->mqprio_qopt.min_rate[i];
8822 
8823 		/* convert to Kbits/s */
8824 		if (ch->max_tx_rate)
8825 			ch->max_tx_rate = div_u64(ch->max_tx_rate,
8826 						  ICE_BW_KBPS_DIVISOR);
8827 		if (ch->min_tx_rate)
8828 			ch->min_tx_rate = div_u64(ch->min_tx_rate,
8829 						  ICE_BW_KBPS_DIVISOR);
8830 
8831 		ret = ice_create_q_channel(vsi, ch);
8832 		if (ret) {
8833 			dev_err(ice_pf_to_dev(pf),
8834 				"failed creating channel TC:%d\n", i);
8835 			kfree(ch);
8836 			goto err_free;
8837 		}
8838 		list_add_tail(&ch->list, &vsi->ch_list);
8839 		vsi->tc_map_vsi[i] = ch->ch_vsi;
8840 		dev_dbg(ice_pf_to_dev(pf),
8841 			"successfully created channel: VSI %pK\n", ch->ch_vsi);
8842 	}
8843 	return 0;
8844 
8845 err_free:
8846 	ice_remove_q_channels(vsi, false);
8847 
8848 	return ret;
8849 }
8850 
8851 /**
8852  * ice_setup_tc_mqprio_qdisc - configure multiple traffic classes
8853  * @netdev: net device to configure
8854  * @type_data: TC offload data
8855  */
8856 static int ice_setup_tc_mqprio_qdisc(struct net_device *netdev, void *type_data)
8857 {
8858 	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
8859 	struct ice_netdev_priv *np = netdev_priv(netdev);
8860 	struct ice_vsi *vsi = np->vsi;
8861 	struct ice_pf *pf = vsi->back;
8862 	u16 mode, ena_tc_qdisc = 0;
8863 	int cur_txq, cur_rxq;
8864 	u8 hw = 0, num_tcf;
8865 	struct device *dev;
8866 	int ret, i;
8867 
8868 	dev = ice_pf_to_dev(pf);
8869 	num_tcf = mqprio_qopt->qopt.num_tc;
8870 	hw = mqprio_qopt->qopt.hw;
8871 	mode = mqprio_qopt->mode;
8872 	if (!hw) {
8873 		clear_bit(ICE_FLAG_TC_MQPRIO, pf->flags);
8874 		vsi->ch_rss_size = 0;
8875 		memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8876 		goto config_tcf;
8877 	}
8878 
8879 	/* Generate queue region map for number of TCF requested */
8880 	for (i = 0; i < num_tcf; i++)
8881 		ena_tc_qdisc |= BIT(i);
8882 
8883 	switch (mode) {
8884 	case TC_MQPRIO_MODE_CHANNEL:
8885 
8886 		if (pf->hw.port_info->is_custom_tx_enabled) {
8887 			dev_err(dev, "Custom Tx scheduler feature enabled, can't configure ADQ\n");
8888 			return -EBUSY;
8889 		}
8890 		ice_tear_down_devlink_rate_tree(pf);
8891 
8892 		ret = ice_validate_mqprio_qopt(vsi, mqprio_qopt);
8893 		if (ret) {
8894 			netdev_err(netdev, "failed to validate_mqprio_qopt(), ret %d\n",
8895 				   ret);
8896 			return ret;
8897 		}
8898 		memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8899 		set_bit(ICE_FLAG_TC_MQPRIO, pf->flags);
8900 		/* don't assume state of hw_tc_offload during driver load
8901 		 * and set the flag for TC flower filter if hw_tc_offload
8902 		 * already ON
8903 		 */
8904 		if (vsi->netdev->features & NETIF_F_HW_TC)
8905 			set_bit(ICE_FLAG_CLS_FLOWER, pf->flags);
8906 		break;
8907 	default:
8908 		return -EINVAL;
8909 	}
8910 
8911 config_tcf:
8912 
8913 	/* Requesting same TCF configuration as already enabled */
8914 	if (ena_tc_qdisc == vsi->tc_cfg.ena_tc &&
8915 	    mode != TC_MQPRIO_MODE_CHANNEL)
8916 		return 0;
8917 
8918 	/* Pause VSI queues */
8919 	ice_dis_vsi(vsi, true);
8920 
8921 	if (!hw && !test_bit(ICE_FLAG_TC_MQPRIO, pf->flags))
8922 		ice_remove_q_channels(vsi, true);
8923 
8924 	if (!hw && !test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) {
8925 		vsi->req_txq = min_t(int, ice_get_avail_txq_count(pf),
8926 				     num_online_cpus());
8927 		vsi->req_rxq = min_t(int, ice_get_avail_rxq_count(pf),
8928 				     num_online_cpus());
8929 	} else {
8930 		/* logic to rebuild VSI, same like ethtool -L */
8931 		u16 offset = 0, qcount_tx = 0, qcount_rx = 0;
8932 
8933 		for (i = 0; i < num_tcf; i++) {
8934 			if (!(ena_tc_qdisc & BIT(i)))
8935 				continue;
8936 
8937 			offset = vsi->mqprio_qopt.qopt.offset[i];
8938 			qcount_rx = vsi->mqprio_qopt.qopt.count[i];
8939 			qcount_tx = vsi->mqprio_qopt.qopt.count[i];
8940 		}
8941 		vsi->req_txq = offset + qcount_tx;
8942 		vsi->req_rxq = offset + qcount_rx;
8943 
8944 		/* store away original rss_size info, so that it gets reused
8945 		 * form ice_vsi_rebuild during tc-qdisc delete stage - to
8946 		 * determine, what should be the rss_sizefor main VSI
8947 		 */
8948 		vsi->orig_rss_size = vsi->rss_size;
8949 	}
8950 
8951 	/* save current values of Tx and Rx queues before calling VSI rebuild
8952 	 * for fallback option
8953 	 */
8954 	cur_txq = vsi->num_txq;
8955 	cur_rxq = vsi->num_rxq;
8956 
8957 	/* proceed with rebuild main VSI using correct number of queues */
8958 	ret = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT);
8959 	if (ret) {
8960 		/* fallback to current number of queues */
8961 		dev_info(dev, "Rebuild failed with new queues, try with current number of queues\n");
8962 		vsi->req_txq = cur_txq;
8963 		vsi->req_rxq = cur_rxq;
8964 		clear_bit(ICE_RESET_FAILED, pf->state);
8965 		if (ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT)) {
8966 			dev_err(dev, "Rebuild of main VSI failed again\n");
8967 			return ret;
8968 		}
8969 	}
8970 
8971 	vsi->all_numtc = num_tcf;
8972 	vsi->all_enatc = ena_tc_qdisc;
8973 	ret = ice_vsi_cfg_tc(vsi, ena_tc_qdisc);
8974 	if (ret) {
8975 		netdev_err(netdev, "failed configuring TC for VSI id=%d\n",
8976 			   vsi->vsi_num);
8977 		goto exit;
8978 	}
8979 
8980 	if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) {
8981 		u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
8982 		u64 min_tx_rate = vsi->mqprio_qopt.min_rate[0];
8983 
8984 		/* set TC0 rate limit if specified */
8985 		if (max_tx_rate || min_tx_rate) {
8986 			/* convert to Kbits/s */
8987 			if (max_tx_rate)
8988 				max_tx_rate = div_u64(max_tx_rate, ICE_BW_KBPS_DIVISOR);
8989 			if (min_tx_rate)
8990 				min_tx_rate = div_u64(min_tx_rate, ICE_BW_KBPS_DIVISOR);
8991 
8992 			ret = ice_set_bw_limit(vsi, max_tx_rate, min_tx_rate);
8993 			if (!ret) {
8994 				dev_dbg(dev, "set Tx rate max %llu min %llu for VSI(%u)\n",
8995 					max_tx_rate, min_tx_rate, vsi->vsi_num);
8996 			} else {
8997 				dev_err(dev, "failed to set Tx rate max %llu min %llu for VSI(%u)\n",
8998 					max_tx_rate, min_tx_rate, vsi->vsi_num);
8999 				goto exit;
9000 			}
9001 		}
9002 		ret = ice_create_q_channels(vsi);
9003 		if (ret) {
9004 			netdev_err(netdev, "failed configuring queue channels\n");
9005 			goto exit;
9006 		} else {
9007 			netdev_dbg(netdev, "successfully configured channels\n");
9008 		}
9009 	}
9010 
9011 	if (vsi->ch_rss_size)
9012 		ice_vsi_cfg_rss_lut_key(vsi);
9013 
9014 exit:
9015 	/* if error, reset the all_numtc and all_enatc */
9016 	if (ret) {
9017 		vsi->all_numtc = 0;
9018 		vsi->all_enatc = 0;
9019 	}
9020 	/* resume VSI */
9021 	ice_ena_vsi(vsi, true);
9022 
9023 	return ret;
9024 }
9025 
9026 static LIST_HEAD(ice_block_cb_list);
9027 
9028 static int
9029 ice_setup_tc(struct net_device *netdev, enum tc_setup_type type,
9030 	     void *type_data)
9031 {
9032 	struct ice_netdev_priv *np = netdev_priv(netdev);
9033 	struct ice_pf *pf = np->vsi->back;
9034 	int err;
9035 
9036 	switch (type) {
9037 	case TC_SETUP_BLOCK:
9038 		return flow_block_cb_setup_simple(type_data,
9039 						  &ice_block_cb_list,
9040 						  ice_setup_tc_block_cb,
9041 						  np, np, true);
9042 	case TC_SETUP_QDISC_MQPRIO:
9043 		/* setup traffic classifier for receive side */
9044 		mutex_lock(&pf->tc_mutex);
9045 		err = ice_setup_tc_mqprio_qdisc(netdev, type_data);
9046 		mutex_unlock(&pf->tc_mutex);
9047 		return err;
9048 	default:
9049 		return -EOPNOTSUPP;
9050 	}
9051 	return -EOPNOTSUPP;
9052 }
9053 
9054 static struct ice_indr_block_priv *
9055 ice_indr_block_priv_lookup(struct ice_netdev_priv *np,
9056 			   struct net_device *netdev)
9057 {
9058 	struct ice_indr_block_priv *cb_priv;
9059 
9060 	list_for_each_entry(cb_priv, &np->tc_indr_block_priv_list, list) {
9061 		if (!cb_priv->netdev)
9062 			return NULL;
9063 		if (cb_priv->netdev == netdev)
9064 			return cb_priv;
9065 	}
9066 	return NULL;
9067 }
9068 
9069 static int
9070 ice_indr_setup_block_cb(enum tc_setup_type type, void *type_data,
9071 			void *indr_priv)
9072 {
9073 	struct ice_indr_block_priv *priv = indr_priv;
9074 	struct ice_netdev_priv *np = priv->np;
9075 
9076 	switch (type) {
9077 	case TC_SETUP_CLSFLOWER:
9078 		return ice_setup_tc_cls_flower(np, priv->netdev,
9079 					       (struct flow_cls_offload *)
9080 					       type_data);
9081 	default:
9082 		return -EOPNOTSUPP;
9083 	}
9084 }
9085 
9086 static int
9087 ice_indr_setup_tc_block(struct net_device *netdev, struct Qdisc *sch,
9088 			struct ice_netdev_priv *np,
9089 			struct flow_block_offload *f, void *data,
9090 			void (*cleanup)(struct flow_block_cb *block_cb))
9091 {
9092 	struct ice_indr_block_priv *indr_priv;
9093 	struct flow_block_cb *block_cb;
9094 
9095 	if (!ice_is_tunnel_supported(netdev) &&
9096 	    !(is_vlan_dev(netdev) &&
9097 	      vlan_dev_real_dev(netdev) == np->vsi->netdev))
9098 		return -EOPNOTSUPP;
9099 
9100 	if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
9101 		return -EOPNOTSUPP;
9102 
9103 	switch (f->command) {
9104 	case FLOW_BLOCK_BIND:
9105 		indr_priv = ice_indr_block_priv_lookup(np, netdev);
9106 		if (indr_priv)
9107 			return -EEXIST;
9108 
9109 		indr_priv = kzalloc(sizeof(*indr_priv), GFP_KERNEL);
9110 		if (!indr_priv)
9111 			return -ENOMEM;
9112 
9113 		indr_priv->netdev = netdev;
9114 		indr_priv->np = np;
9115 		list_add(&indr_priv->list, &np->tc_indr_block_priv_list);
9116 
9117 		block_cb =
9118 			flow_indr_block_cb_alloc(ice_indr_setup_block_cb,
9119 						 indr_priv, indr_priv,
9120 						 ice_rep_indr_tc_block_unbind,
9121 						 f, netdev, sch, data, np,
9122 						 cleanup);
9123 
9124 		if (IS_ERR(block_cb)) {
9125 			list_del(&indr_priv->list);
9126 			kfree(indr_priv);
9127 			return PTR_ERR(block_cb);
9128 		}
9129 		flow_block_cb_add(block_cb, f);
9130 		list_add_tail(&block_cb->driver_list, &ice_block_cb_list);
9131 		break;
9132 	case FLOW_BLOCK_UNBIND:
9133 		indr_priv = ice_indr_block_priv_lookup(np, netdev);
9134 		if (!indr_priv)
9135 			return -ENOENT;
9136 
9137 		block_cb = flow_block_cb_lookup(f->block,
9138 						ice_indr_setup_block_cb,
9139 						indr_priv);
9140 		if (!block_cb)
9141 			return -ENOENT;
9142 
9143 		flow_indr_block_cb_remove(block_cb, f);
9144 
9145 		list_del(&block_cb->driver_list);
9146 		break;
9147 	default:
9148 		return -EOPNOTSUPP;
9149 	}
9150 	return 0;
9151 }
9152 
9153 static int
9154 ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch,
9155 		     void *cb_priv, enum tc_setup_type type, void *type_data,
9156 		     void *data,
9157 		     void (*cleanup)(struct flow_block_cb *block_cb))
9158 {
9159 	switch (type) {
9160 	case TC_SETUP_BLOCK:
9161 		return ice_indr_setup_tc_block(netdev, sch, cb_priv, type_data,
9162 					       data, cleanup);
9163 
9164 	default:
9165 		return -EOPNOTSUPP;
9166 	}
9167 }
9168 
9169 /**
9170  * ice_open - Called when a network interface becomes active
9171  * @netdev: network interface device structure
9172  *
9173  * The open entry point is called when a network interface is made
9174  * active by the system (IFF_UP). At this point all resources needed
9175  * for transmit and receive operations are allocated, the interrupt
9176  * handler is registered with the OS, the netdev watchdog is enabled,
9177  * and the stack is notified that the interface is ready.
9178  *
9179  * Returns 0 on success, negative value on failure
9180  */
9181 int ice_open(struct net_device *netdev)
9182 {
9183 	struct ice_netdev_priv *np = netdev_priv(netdev);
9184 	struct ice_pf *pf = np->vsi->back;
9185 
9186 	if (ice_is_reset_in_progress(pf->state)) {
9187 		netdev_err(netdev, "can't open net device while reset is in progress");
9188 		return -EBUSY;
9189 	}
9190 
9191 	return ice_open_internal(netdev);
9192 }
9193 
9194 /**
9195  * ice_open_internal - Called when a network interface becomes active
9196  * @netdev: network interface device structure
9197  *
9198  * Internal ice_open implementation. Should not be used directly except for ice_open and reset
9199  * handling routine
9200  *
9201  * Returns 0 on success, negative value on failure
9202  */
9203 int ice_open_internal(struct net_device *netdev)
9204 {
9205 	struct ice_netdev_priv *np = netdev_priv(netdev);
9206 	struct ice_vsi *vsi = np->vsi;
9207 	struct ice_pf *pf = vsi->back;
9208 	struct ice_port_info *pi;
9209 	int err;
9210 
9211 	if (test_bit(ICE_NEEDS_RESTART, pf->state)) {
9212 		netdev_err(netdev, "driver needs to be unloaded and reloaded\n");
9213 		return -EIO;
9214 	}
9215 
9216 	netif_carrier_off(netdev);
9217 
9218 	pi = vsi->port_info;
9219 	err = ice_update_link_info(pi);
9220 	if (err) {
9221 		netdev_err(netdev, "Failed to get link info, error %d\n", err);
9222 		return err;
9223 	}
9224 
9225 	ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err);
9226 
9227 	/* Set PHY if there is media, otherwise, turn off PHY */
9228 	if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
9229 		clear_bit(ICE_FLAG_NO_MEDIA, pf->flags);
9230 		if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state)) {
9231 			err = ice_init_phy_user_cfg(pi);
9232 			if (err) {
9233 				netdev_err(netdev, "Failed to initialize PHY settings, error %d\n",
9234 					   err);
9235 				return err;
9236 			}
9237 		}
9238 
9239 		err = ice_configure_phy(vsi);
9240 		if (err) {
9241 			netdev_err(netdev, "Failed to set physical link up, error %d\n",
9242 				   err);
9243 			return err;
9244 		}
9245 	} else {
9246 		set_bit(ICE_FLAG_NO_MEDIA, pf->flags);
9247 		ice_set_link(vsi, false);
9248 	}
9249 
9250 	err = ice_vsi_open(vsi);
9251 	if (err)
9252 		netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n",
9253 			   vsi->vsi_num, vsi->vsw->sw_id);
9254 
9255 	/* Update existing tunnels information */
9256 	udp_tunnel_get_rx_info(netdev);
9257 
9258 	return err;
9259 }
9260 
9261 /**
9262  * ice_stop - Disables a network interface
9263  * @netdev: network interface device structure
9264  *
9265  * The stop entry point is called when an interface is de-activated by the OS,
9266  * and the netdevice enters the DOWN state. The hardware is still under the
9267  * driver's control, but the netdev interface is disabled.
9268  *
9269  * Returns success only - not allowed to fail
9270  */
9271 int ice_stop(struct net_device *netdev)
9272 {
9273 	struct ice_netdev_priv *np = netdev_priv(netdev);
9274 	struct ice_vsi *vsi = np->vsi;
9275 	struct ice_pf *pf = vsi->back;
9276 
9277 	if (ice_is_reset_in_progress(pf->state)) {
9278 		netdev_err(netdev, "can't stop net device while reset is in progress");
9279 		return -EBUSY;
9280 	}
9281 
9282 	if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) {
9283 		int link_err = ice_force_phys_link_state(vsi, false);
9284 
9285 		if (link_err) {
9286 			netdev_err(vsi->netdev, "Failed to set physical link down, VSI %d error %d\n",
9287 				   vsi->vsi_num, link_err);
9288 			return -EIO;
9289 		}
9290 	}
9291 
9292 	ice_vsi_close(vsi);
9293 
9294 	return 0;
9295 }
9296 
9297 /**
9298  * ice_features_check - Validate encapsulated packet conforms to limits
9299  * @skb: skb buffer
9300  * @netdev: This port's netdev
9301  * @features: Offload features that the stack believes apply
9302  */
9303 static netdev_features_t
9304 ice_features_check(struct sk_buff *skb,
9305 		   struct net_device __always_unused *netdev,
9306 		   netdev_features_t features)
9307 {
9308 	bool gso = skb_is_gso(skb);
9309 	size_t len;
9310 
9311 	/* No point in doing any of this if neither checksum nor GSO are
9312 	 * being requested for this frame. We can rule out both by just
9313 	 * checking for CHECKSUM_PARTIAL
9314 	 */
9315 	if (skb->ip_summed != CHECKSUM_PARTIAL)
9316 		return features;
9317 
9318 	/* We cannot support GSO if the MSS is going to be less than
9319 	 * 64 bytes. If it is then we need to drop support for GSO.
9320 	 */
9321 	if (gso && (skb_shinfo(skb)->gso_size < ICE_TXD_CTX_MIN_MSS))
9322 		features &= ~NETIF_F_GSO_MASK;
9323 
9324 	len = skb_network_offset(skb);
9325 	if (len > ICE_TXD_MACLEN_MAX || len & 0x1)
9326 		goto out_rm_features;
9327 
9328 	len = skb_network_header_len(skb);
9329 	if (len > ICE_TXD_IPLEN_MAX || len & 0x1)
9330 		goto out_rm_features;
9331 
9332 	if (skb->encapsulation) {
9333 		/* this must work for VXLAN frames AND IPIP/SIT frames, and in
9334 		 * the case of IPIP frames, the transport header pointer is
9335 		 * after the inner header! So check to make sure that this
9336 		 * is a GRE or UDP_TUNNEL frame before doing that math.
9337 		 */
9338 		if (gso && (skb_shinfo(skb)->gso_type &
9339 			    (SKB_GSO_GRE | SKB_GSO_UDP_TUNNEL))) {
9340 			len = skb_inner_network_header(skb) -
9341 			      skb_transport_header(skb);
9342 			if (len > ICE_TXD_L4LEN_MAX || len & 0x1)
9343 				goto out_rm_features;
9344 		}
9345 
9346 		len = skb_inner_network_header_len(skb);
9347 		if (len > ICE_TXD_IPLEN_MAX || len & 0x1)
9348 			goto out_rm_features;
9349 	}
9350 
9351 	return features;
9352 out_rm_features:
9353 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
9354 }
9355 
9356 static const struct net_device_ops ice_netdev_safe_mode_ops = {
9357 	.ndo_open = ice_open,
9358 	.ndo_stop = ice_stop,
9359 	.ndo_start_xmit = ice_start_xmit,
9360 	.ndo_set_mac_address = ice_set_mac_address,
9361 	.ndo_validate_addr = eth_validate_addr,
9362 	.ndo_change_mtu = ice_change_mtu,
9363 	.ndo_get_stats64 = ice_get_stats64,
9364 	.ndo_tx_timeout = ice_tx_timeout,
9365 	.ndo_bpf = ice_xdp_safe_mode,
9366 };
9367 
9368 static const struct net_device_ops ice_netdev_ops = {
9369 	.ndo_open = ice_open,
9370 	.ndo_stop = ice_stop,
9371 	.ndo_start_xmit = ice_start_xmit,
9372 	.ndo_select_queue = ice_select_queue,
9373 	.ndo_features_check = ice_features_check,
9374 	.ndo_fix_features = ice_fix_features,
9375 	.ndo_set_rx_mode = ice_set_rx_mode,
9376 	.ndo_set_mac_address = ice_set_mac_address,
9377 	.ndo_validate_addr = eth_validate_addr,
9378 	.ndo_change_mtu = ice_change_mtu,
9379 	.ndo_get_stats64 = ice_get_stats64,
9380 	.ndo_set_tx_maxrate = ice_set_tx_maxrate,
9381 	.ndo_eth_ioctl = ice_eth_ioctl,
9382 	.ndo_set_vf_spoofchk = ice_set_vf_spoofchk,
9383 	.ndo_set_vf_mac = ice_set_vf_mac,
9384 	.ndo_get_vf_config = ice_get_vf_cfg,
9385 	.ndo_set_vf_trust = ice_set_vf_trust,
9386 	.ndo_set_vf_vlan = ice_set_vf_port_vlan,
9387 	.ndo_set_vf_link_state = ice_set_vf_link_state,
9388 	.ndo_get_vf_stats = ice_get_vf_stats,
9389 	.ndo_set_vf_rate = ice_set_vf_bw,
9390 	.ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid,
9391 	.ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid,
9392 	.ndo_setup_tc = ice_setup_tc,
9393 	.ndo_set_features = ice_set_features,
9394 	.ndo_bridge_getlink = ice_bridge_getlink,
9395 	.ndo_bridge_setlink = ice_bridge_setlink,
9396 	.ndo_fdb_add = ice_fdb_add,
9397 	.ndo_fdb_del = ice_fdb_del,
9398 #ifdef CONFIG_RFS_ACCEL
9399 	.ndo_rx_flow_steer = ice_rx_flow_steer,
9400 #endif
9401 	.ndo_tx_timeout = ice_tx_timeout,
9402 	.ndo_bpf = ice_xdp,
9403 	.ndo_xdp_xmit = ice_xdp_xmit,
9404 	.ndo_xsk_wakeup = ice_xsk_wakeup,
9405 };
9406