1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2019 Intel Corporation. */
3 
4 #include <linux/module.h>
5 #include <linux/interrupt.h>
6 #include <linux/aer.h>
7 
8 #include "fm10k.h"
9 
10 static const struct fm10k_info *fm10k_info_tbl[] = {
11 	[fm10k_device_pf] = &fm10k_pf_info,
12 	[fm10k_device_vf] = &fm10k_vf_info,
13 };
14 
15 /*
16  * fm10k_pci_tbl - PCI Device ID Table
17  *
18  * Wildcard entries (PCI_ANY_ID) should come last
19  * Last entry must be all 0s
20  *
21  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
22  *   Class, Class Mask, private data (not used) }
23  */
24 static const struct pci_device_id fm10k_pci_tbl[] = {
25 	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_PF), fm10k_device_pf },
26 	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_SDI_FM10420_QDA2), fm10k_device_pf },
27 	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_SDI_FM10420_DA2), fm10k_device_pf },
28 	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_VF), fm10k_device_vf },
29 	/* required last entry */
30 	{ 0, }
31 };
32 MODULE_DEVICE_TABLE(pci, fm10k_pci_tbl);
33 
34 u16 fm10k_read_pci_cfg_word(struct fm10k_hw *hw, u32 reg)
35 {
36 	struct fm10k_intfc *interface = hw->back;
37 	u16 value = 0;
38 
39 	if (FM10K_REMOVED(hw->hw_addr))
40 		return ~value;
41 
42 	pci_read_config_word(interface->pdev, reg, &value);
43 	if (value == 0xFFFF)
44 		fm10k_write_flush(hw);
45 
46 	return value;
47 }
48 
49 u32 fm10k_read_reg(struct fm10k_hw *hw, int reg)
50 {
51 	u32 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
52 	u32 value = 0;
53 
54 	if (FM10K_REMOVED(hw_addr))
55 		return ~value;
56 
57 	value = readl(&hw_addr[reg]);
58 	if (!(~value) && (!reg || !(~readl(hw_addr)))) {
59 		struct fm10k_intfc *interface = hw->back;
60 		struct net_device *netdev = interface->netdev;
61 
62 		hw->hw_addr = NULL;
63 		netif_device_detach(netdev);
64 		netdev_err(netdev, "PCIe link lost, device now detached\n");
65 	}
66 
67 	return value;
68 }
69 
70 static int fm10k_hw_ready(struct fm10k_intfc *interface)
71 {
72 	struct fm10k_hw *hw = &interface->hw;
73 
74 	fm10k_write_flush(hw);
75 
76 	return FM10K_REMOVED(hw->hw_addr) ? -ENODEV : 0;
77 }
78 
79 /**
80  * fm10k_macvlan_schedule - Schedule MAC/VLAN queue task
81  * @interface: fm10k private interface structure
82  *
83  * Schedule the MAC/VLAN queue monitor task. If the MAC/VLAN task cannot be
84  * started immediately, request that it be restarted when possible.
85  */
86 void fm10k_macvlan_schedule(struct fm10k_intfc *interface)
87 {
88 	/* Avoid processing the MAC/VLAN queue when the service task is
89 	 * disabled, or when we're resetting the device.
90 	 */
91 	if (!test_bit(__FM10K_MACVLAN_DISABLE, interface->state) &&
92 	    !test_and_set_bit(__FM10K_MACVLAN_SCHED, interface->state)) {
93 		clear_bit(__FM10K_MACVLAN_REQUEST, interface->state);
94 		/* We delay the actual start of execution in order to allow
95 		 * multiple MAC/VLAN updates to accumulate before handling
96 		 * them, and to allow some time to let the mailbox drain
97 		 * between runs.
98 		 */
99 		queue_delayed_work(fm10k_workqueue,
100 				   &interface->macvlan_task, 10);
101 	} else {
102 		set_bit(__FM10K_MACVLAN_REQUEST, interface->state);
103 	}
104 }
105 
106 /**
107  * fm10k_stop_macvlan_task - Stop the MAC/VLAN queue monitor
108  * @interface: fm10k private interface structure
109  *
110  * Wait until the MAC/VLAN queue task has stopped, and cancel any future
111  * requests.
112  */
113 static void fm10k_stop_macvlan_task(struct fm10k_intfc *interface)
114 {
115 	/* Disable the MAC/VLAN work item */
116 	set_bit(__FM10K_MACVLAN_DISABLE, interface->state);
117 
118 	/* Make sure we waited until any current invocations have stopped */
119 	cancel_delayed_work_sync(&interface->macvlan_task);
120 
121 	/* We set the __FM10K_MACVLAN_SCHED bit when we schedule the task.
122 	 * However, it may not be unset of the MAC/VLAN task never actually
123 	 * got a chance to run. Since we've canceled the task here, and it
124 	 * cannot be rescheuled right now, we need to ensure the scheduled bit
125 	 * gets unset.
126 	 */
127 	clear_bit(__FM10K_MACVLAN_SCHED, interface->state);
128 }
129 
130 /**
131  * fm10k_resume_macvlan_task - Restart the MAC/VLAN queue monitor
132  * @interface: fm10k private interface structure
133  *
134  * Clear the __FM10K_MACVLAN_DISABLE bit and, if a request occurred, schedule
135  * the MAC/VLAN work monitor.
136  */
137 static void fm10k_resume_macvlan_task(struct fm10k_intfc *interface)
138 {
139 	/* Re-enable the MAC/VLAN work item */
140 	clear_bit(__FM10K_MACVLAN_DISABLE, interface->state);
141 
142 	/* We might have received a MAC/VLAN request while disabled. If so,
143 	 * kick off the queue now.
144 	 */
145 	if (test_bit(__FM10K_MACVLAN_REQUEST, interface->state))
146 		fm10k_macvlan_schedule(interface);
147 }
148 
149 void fm10k_service_event_schedule(struct fm10k_intfc *interface)
150 {
151 	if (!test_bit(__FM10K_SERVICE_DISABLE, interface->state) &&
152 	    !test_and_set_bit(__FM10K_SERVICE_SCHED, interface->state)) {
153 		clear_bit(__FM10K_SERVICE_REQUEST, interface->state);
154 		queue_work(fm10k_workqueue, &interface->service_task);
155 	} else {
156 		set_bit(__FM10K_SERVICE_REQUEST, interface->state);
157 	}
158 }
159 
160 static void fm10k_service_event_complete(struct fm10k_intfc *interface)
161 {
162 	WARN_ON(!test_bit(__FM10K_SERVICE_SCHED, interface->state));
163 
164 	/* flush memory to make sure state is correct before next watchog */
165 	smp_mb__before_atomic();
166 	clear_bit(__FM10K_SERVICE_SCHED, interface->state);
167 
168 	/* If a service event was requested since we started, immediately
169 	 * re-schedule now. This ensures we don't drop a request until the
170 	 * next timer event.
171 	 */
172 	if (test_bit(__FM10K_SERVICE_REQUEST, interface->state))
173 		fm10k_service_event_schedule(interface);
174 }
175 
176 static void fm10k_stop_service_event(struct fm10k_intfc *interface)
177 {
178 	set_bit(__FM10K_SERVICE_DISABLE, interface->state);
179 	cancel_work_sync(&interface->service_task);
180 
181 	/* It's possible that cancel_work_sync stopped the service task from
182 	 * running before it could actually start. In this case the
183 	 * __FM10K_SERVICE_SCHED bit will never be cleared. Since we know that
184 	 * the service task cannot be running at this point, we need to clear
185 	 * the scheduled bit, as otherwise the service task may never be
186 	 * restarted.
187 	 */
188 	clear_bit(__FM10K_SERVICE_SCHED, interface->state);
189 }
190 
191 static void fm10k_start_service_event(struct fm10k_intfc *interface)
192 {
193 	clear_bit(__FM10K_SERVICE_DISABLE, interface->state);
194 	fm10k_service_event_schedule(interface);
195 }
196 
197 /**
198  * fm10k_service_timer - Timer Call-back
199  * @t: pointer to timer data
200  **/
201 static void fm10k_service_timer(struct timer_list *t)
202 {
203 	struct fm10k_intfc *interface = from_timer(interface, t,
204 						   service_timer);
205 
206 	/* Reset the timer */
207 	mod_timer(&interface->service_timer, (HZ * 2) + jiffies);
208 
209 	fm10k_service_event_schedule(interface);
210 }
211 
212 /**
213  * fm10k_prepare_for_reset - Prepare the driver and device for a pending reset
214  * @interface: fm10k private data structure
215  *
216  * This function prepares for a device reset by shutting as much down as we
217  * can. It does nothing and returns false if __FM10K_RESETTING was already set
218  * prior to calling this function. It returns true if it actually did work.
219  */
220 static bool fm10k_prepare_for_reset(struct fm10k_intfc *interface)
221 {
222 	struct net_device *netdev = interface->netdev;
223 
224 	/* put off any impending NetWatchDogTimeout */
225 	netif_trans_update(netdev);
226 
227 	/* Nothing to do if a reset is already in progress */
228 	if (test_and_set_bit(__FM10K_RESETTING, interface->state))
229 		return false;
230 
231 	/* As the MAC/VLAN task will be accessing registers it must not be
232 	 * running while we reset. Although the task will not be scheduled
233 	 * once we start resetting it may already be running
234 	 */
235 	fm10k_stop_macvlan_task(interface);
236 
237 	rtnl_lock();
238 
239 	fm10k_iov_suspend(interface->pdev);
240 
241 	if (netif_running(netdev))
242 		fm10k_close(netdev);
243 
244 	fm10k_mbx_free_irq(interface);
245 
246 	/* free interrupts */
247 	fm10k_clear_queueing_scheme(interface);
248 
249 	/* delay any future reset requests */
250 	interface->last_reset = jiffies + (10 * HZ);
251 
252 	rtnl_unlock();
253 
254 	return true;
255 }
256 
257 static int fm10k_handle_reset(struct fm10k_intfc *interface)
258 {
259 	struct net_device *netdev = interface->netdev;
260 	struct fm10k_hw *hw = &interface->hw;
261 	int err;
262 
263 	WARN_ON(!test_bit(__FM10K_RESETTING, interface->state));
264 
265 	rtnl_lock();
266 
267 	pci_set_master(interface->pdev);
268 
269 	/* reset and initialize the hardware so it is in a known state */
270 	err = hw->mac.ops.reset_hw(hw);
271 	if (err) {
272 		dev_err(&interface->pdev->dev, "reset_hw failed: %d\n", err);
273 		goto reinit_err;
274 	}
275 
276 	err = hw->mac.ops.init_hw(hw);
277 	if (err) {
278 		dev_err(&interface->pdev->dev, "init_hw failed: %d\n", err);
279 		goto reinit_err;
280 	}
281 
282 	err = fm10k_init_queueing_scheme(interface);
283 	if (err) {
284 		dev_err(&interface->pdev->dev,
285 			"init_queueing_scheme failed: %d\n", err);
286 		goto reinit_err;
287 	}
288 
289 	/* re-associate interrupts */
290 	err = fm10k_mbx_request_irq(interface);
291 	if (err)
292 		goto err_mbx_irq;
293 
294 	err = fm10k_hw_ready(interface);
295 	if (err)
296 		goto err_open;
297 
298 	/* update hardware address for VFs if perm_addr has changed */
299 	if (hw->mac.type == fm10k_mac_vf) {
300 		if (is_valid_ether_addr(hw->mac.perm_addr)) {
301 			ether_addr_copy(hw->mac.addr, hw->mac.perm_addr);
302 			ether_addr_copy(netdev->perm_addr, hw->mac.perm_addr);
303 			ether_addr_copy(netdev->dev_addr, hw->mac.perm_addr);
304 			netdev->addr_assign_type &= ~NET_ADDR_RANDOM;
305 		}
306 
307 		if (hw->mac.vlan_override)
308 			netdev->features &= ~NETIF_F_HW_VLAN_CTAG_RX;
309 		else
310 			netdev->features |= NETIF_F_HW_VLAN_CTAG_RX;
311 	}
312 
313 	err = netif_running(netdev) ? fm10k_open(netdev) : 0;
314 	if (err)
315 		goto err_open;
316 
317 	fm10k_iov_resume(interface->pdev);
318 
319 	rtnl_unlock();
320 
321 	fm10k_resume_macvlan_task(interface);
322 
323 	clear_bit(__FM10K_RESETTING, interface->state);
324 
325 	return err;
326 err_open:
327 	fm10k_mbx_free_irq(interface);
328 err_mbx_irq:
329 	fm10k_clear_queueing_scheme(interface);
330 reinit_err:
331 	netif_device_detach(netdev);
332 
333 	rtnl_unlock();
334 
335 	clear_bit(__FM10K_RESETTING, interface->state);
336 
337 	return err;
338 }
339 
340 static void fm10k_detach_subtask(struct fm10k_intfc *interface)
341 {
342 	struct net_device *netdev = interface->netdev;
343 	u32 __iomem *hw_addr;
344 	u32 value;
345 
346 	/* do nothing if netdev is still present or hw_addr is set */
347 	if (netif_device_present(netdev) || interface->hw.hw_addr)
348 		return;
349 
350 	/* We've lost the PCIe register space, and can no longer access the
351 	 * device. Shut everything except the detach subtask down and prepare
352 	 * to reset the device in case we recover. If we actually prepare for
353 	 * reset, indicate that we're detached.
354 	 */
355 	if (fm10k_prepare_for_reset(interface))
356 		set_bit(__FM10K_RESET_DETACHED, interface->state);
357 
358 	/* check the real address space to see if we've recovered */
359 	hw_addr = READ_ONCE(interface->uc_addr);
360 	value = readl(hw_addr);
361 	if (~value) {
362 		int err;
363 
364 		/* Make sure the reset was initiated because we detached,
365 		 * otherwise we might race with a different reset flow.
366 		 */
367 		if (!test_and_clear_bit(__FM10K_RESET_DETACHED,
368 					interface->state))
369 			return;
370 
371 		/* Restore the hardware address */
372 		interface->hw.hw_addr = interface->uc_addr;
373 
374 		/* PCIe link has been restored, and the device is active
375 		 * again. Restore everything and reset the device.
376 		 */
377 		err = fm10k_handle_reset(interface);
378 		if (err) {
379 			netdev_err(netdev, "Unable to reset device: %d\n", err);
380 			interface->hw.hw_addr = NULL;
381 			return;
382 		}
383 
384 		/* Re-attach the netdev */
385 		netif_device_attach(netdev);
386 		netdev_warn(netdev, "PCIe link restored, device now attached\n");
387 		return;
388 	}
389 }
390 
391 static void fm10k_reset_subtask(struct fm10k_intfc *interface)
392 {
393 	int err;
394 
395 	if (!test_and_clear_bit(FM10K_FLAG_RESET_REQUESTED,
396 				interface->flags))
397 		return;
398 
399 	/* If another thread has already prepared to reset the device, we
400 	 * should not attempt to handle a reset here, since we'd race with
401 	 * that thread. This may happen if we suspend the device or if the
402 	 * PCIe link is lost. In this case, we'll just ignore the RESET
403 	 * request, as it will (eventually) be taken care of when the thread
404 	 * which actually started the reset is finished.
405 	 */
406 	if (!fm10k_prepare_for_reset(interface))
407 		return;
408 
409 	netdev_err(interface->netdev, "Reset interface\n");
410 
411 	err = fm10k_handle_reset(interface);
412 	if (err)
413 		dev_err(&interface->pdev->dev,
414 			"fm10k_handle_reset failed: %d\n", err);
415 }
416 
417 /**
418  * fm10k_configure_swpri_map - Configure Receive SWPRI to PC mapping
419  * @interface: board private structure
420  *
421  * Configure the SWPRI to PC mapping for the port.
422  **/
423 static void fm10k_configure_swpri_map(struct fm10k_intfc *interface)
424 {
425 	struct net_device *netdev = interface->netdev;
426 	struct fm10k_hw *hw = &interface->hw;
427 	int i;
428 
429 	/* clear flag indicating update is needed */
430 	clear_bit(FM10K_FLAG_SWPRI_CONFIG, interface->flags);
431 
432 	/* these registers are only available on the PF */
433 	if (hw->mac.type != fm10k_mac_pf)
434 		return;
435 
436 	/* configure SWPRI to PC map */
437 	for (i = 0; i < FM10K_SWPRI_MAX; i++)
438 		fm10k_write_reg(hw, FM10K_SWPRI_MAP(i),
439 				netdev_get_prio_tc_map(netdev, i));
440 }
441 
442 /**
443  * fm10k_watchdog_update_host_state - Update the link status based on host.
444  * @interface: board private structure
445  **/
446 static void fm10k_watchdog_update_host_state(struct fm10k_intfc *interface)
447 {
448 	struct fm10k_hw *hw = &interface->hw;
449 	s32 err;
450 
451 	if (test_bit(__FM10K_LINK_DOWN, interface->state)) {
452 		interface->host_ready = false;
453 		if (time_is_after_jiffies(interface->link_down_event))
454 			return;
455 		clear_bit(__FM10K_LINK_DOWN, interface->state);
456 	}
457 
458 	if (test_bit(FM10K_FLAG_SWPRI_CONFIG, interface->flags)) {
459 		if (rtnl_trylock()) {
460 			fm10k_configure_swpri_map(interface);
461 			rtnl_unlock();
462 		}
463 	}
464 
465 	/* lock the mailbox for transmit and receive */
466 	fm10k_mbx_lock(interface);
467 
468 	err = hw->mac.ops.get_host_state(hw, &interface->host_ready);
469 	if (err && time_is_before_jiffies(interface->last_reset))
470 		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
471 
472 	/* free the lock */
473 	fm10k_mbx_unlock(interface);
474 }
475 
476 /**
477  * fm10k_mbx_subtask - Process upstream and downstream mailboxes
478  * @interface: board private structure
479  *
480  * This function will process both the upstream and downstream mailboxes.
481  **/
482 static void fm10k_mbx_subtask(struct fm10k_intfc *interface)
483 {
484 	/* If we're resetting, bail out */
485 	if (test_bit(__FM10K_RESETTING, interface->state))
486 		return;
487 
488 	/* process upstream mailbox and update device state */
489 	fm10k_watchdog_update_host_state(interface);
490 
491 	/* process downstream mailboxes */
492 	fm10k_iov_mbx(interface);
493 }
494 
495 /**
496  * fm10k_watchdog_host_is_ready - Update netdev status based on host ready
497  * @interface: board private structure
498  **/
499 static void fm10k_watchdog_host_is_ready(struct fm10k_intfc *interface)
500 {
501 	struct net_device *netdev = interface->netdev;
502 
503 	/* only continue if link state is currently down */
504 	if (netif_carrier_ok(netdev))
505 		return;
506 
507 	netif_info(interface, drv, netdev, "NIC Link is up\n");
508 
509 	netif_carrier_on(netdev);
510 	netif_tx_wake_all_queues(netdev);
511 }
512 
513 /**
514  * fm10k_watchdog_host_not_ready - Update netdev status based on host not ready
515  * @interface: board private structure
516  **/
517 static void fm10k_watchdog_host_not_ready(struct fm10k_intfc *interface)
518 {
519 	struct net_device *netdev = interface->netdev;
520 
521 	/* only continue if link state is currently up */
522 	if (!netif_carrier_ok(netdev))
523 		return;
524 
525 	netif_info(interface, drv, netdev, "NIC Link is down\n");
526 
527 	netif_carrier_off(netdev);
528 	netif_tx_stop_all_queues(netdev);
529 }
530 
531 /**
532  * fm10k_update_stats - Update the board statistics counters.
533  * @interface: board private structure
534  **/
535 void fm10k_update_stats(struct fm10k_intfc *interface)
536 {
537 	struct net_device_stats *net_stats = &interface->netdev->stats;
538 	struct fm10k_hw *hw = &interface->hw;
539 	u64 hw_csum_tx_good = 0, hw_csum_rx_good = 0, rx_length_errors = 0;
540 	u64 rx_switch_errors = 0, rx_drops = 0, rx_pp_errors = 0;
541 	u64 rx_link_errors = 0;
542 	u64 rx_errors = 0, rx_csum_errors = 0, tx_csum_errors = 0;
543 	u64 restart_queue = 0, tx_busy = 0, alloc_failed = 0;
544 	u64 rx_bytes_nic = 0, rx_pkts_nic = 0, rx_drops_nic = 0;
545 	u64 tx_bytes_nic = 0, tx_pkts_nic = 0;
546 	u64 bytes, pkts;
547 	int i;
548 
549 	/* ensure only one thread updates stats at a time */
550 	if (test_and_set_bit(__FM10K_UPDATING_STATS, interface->state))
551 		return;
552 
553 	/* do not allow stats update via service task for next second */
554 	interface->next_stats_update = jiffies + HZ;
555 
556 	/* gather some stats to the interface struct that are per queue */
557 	for (bytes = 0, pkts = 0, i = 0; i < interface->num_tx_queues; i++) {
558 		struct fm10k_ring *tx_ring = READ_ONCE(interface->tx_ring[i]);
559 
560 		if (!tx_ring)
561 			continue;
562 
563 		restart_queue += tx_ring->tx_stats.restart_queue;
564 		tx_busy += tx_ring->tx_stats.tx_busy;
565 		tx_csum_errors += tx_ring->tx_stats.csum_err;
566 		bytes += tx_ring->stats.bytes;
567 		pkts += tx_ring->stats.packets;
568 		hw_csum_tx_good += tx_ring->tx_stats.csum_good;
569 	}
570 
571 	interface->restart_queue = restart_queue;
572 	interface->tx_busy = tx_busy;
573 	net_stats->tx_bytes = bytes;
574 	net_stats->tx_packets = pkts;
575 	interface->tx_csum_errors = tx_csum_errors;
576 	interface->hw_csum_tx_good = hw_csum_tx_good;
577 
578 	/* gather some stats to the interface struct that are per queue */
579 	for (bytes = 0, pkts = 0, i = 0; i < interface->num_rx_queues; i++) {
580 		struct fm10k_ring *rx_ring = READ_ONCE(interface->rx_ring[i]);
581 
582 		if (!rx_ring)
583 			continue;
584 
585 		bytes += rx_ring->stats.bytes;
586 		pkts += rx_ring->stats.packets;
587 		alloc_failed += rx_ring->rx_stats.alloc_failed;
588 		rx_csum_errors += rx_ring->rx_stats.csum_err;
589 		rx_errors += rx_ring->rx_stats.errors;
590 		hw_csum_rx_good += rx_ring->rx_stats.csum_good;
591 		rx_switch_errors += rx_ring->rx_stats.switch_errors;
592 		rx_drops += rx_ring->rx_stats.drops;
593 		rx_pp_errors += rx_ring->rx_stats.pp_errors;
594 		rx_link_errors += rx_ring->rx_stats.link_errors;
595 		rx_length_errors += rx_ring->rx_stats.length_errors;
596 	}
597 
598 	net_stats->rx_bytes = bytes;
599 	net_stats->rx_packets = pkts;
600 	interface->alloc_failed = alloc_failed;
601 	interface->rx_csum_errors = rx_csum_errors;
602 	interface->hw_csum_rx_good = hw_csum_rx_good;
603 	interface->rx_switch_errors = rx_switch_errors;
604 	interface->rx_drops = rx_drops;
605 	interface->rx_pp_errors = rx_pp_errors;
606 	interface->rx_link_errors = rx_link_errors;
607 	interface->rx_length_errors = rx_length_errors;
608 
609 	hw->mac.ops.update_hw_stats(hw, &interface->stats);
610 
611 	for (i = 0; i < hw->mac.max_queues; i++) {
612 		struct fm10k_hw_stats_q *q = &interface->stats.q[i];
613 
614 		tx_bytes_nic += q->tx_bytes.count;
615 		tx_pkts_nic += q->tx_packets.count;
616 		rx_bytes_nic += q->rx_bytes.count;
617 		rx_pkts_nic += q->rx_packets.count;
618 		rx_drops_nic += q->rx_drops.count;
619 	}
620 
621 	interface->tx_bytes_nic = tx_bytes_nic;
622 	interface->tx_packets_nic = tx_pkts_nic;
623 	interface->rx_bytes_nic = rx_bytes_nic;
624 	interface->rx_packets_nic = rx_pkts_nic;
625 	interface->rx_drops_nic = rx_drops_nic;
626 
627 	/* Fill out the OS statistics structure */
628 	net_stats->rx_errors = rx_errors;
629 	net_stats->rx_dropped = interface->stats.nodesc_drop.count;
630 
631 	/* Update VF statistics */
632 	fm10k_iov_update_stats(interface);
633 
634 	clear_bit(__FM10K_UPDATING_STATS, interface->state);
635 }
636 
637 /**
638  * fm10k_watchdog_flush_tx - flush queues on host not ready
639  * @interface: pointer to the device interface structure
640  **/
641 static void fm10k_watchdog_flush_tx(struct fm10k_intfc *interface)
642 {
643 	int some_tx_pending = 0;
644 	int i;
645 
646 	/* nothing to do if carrier is up */
647 	if (netif_carrier_ok(interface->netdev))
648 		return;
649 
650 	for (i = 0; i < interface->num_tx_queues; i++) {
651 		struct fm10k_ring *tx_ring = interface->tx_ring[i];
652 
653 		if (tx_ring->next_to_use != tx_ring->next_to_clean) {
654 			some_tx_pending = 1;
655 			break;
656 		}
657 	}
658 
659 	/* We've lost link, so the controller stops DMA, but we've got
660 	 * queued Tx work that's never going to get done, so reset
661 	 * controller to flush Tx.
662 	 */
663 	if (some_tx_pending)
664 		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
665 }
666 
667 /**
668  * fm10k_watchdog_subtask - check and bring link up
669  * @interface: pointer to the device interface structure
670  **/
671 static void fm10k_watchdog_subtask(struct fm10k_intfc *interface)
672 {
673 	/* if interface is down do nothing */
674 	if (test_bit(__FM10K_DOWN, interface->state) ||
675 	    test_bit(__FM10K_RESETTING, interface->state))
676 		return;
677 
678 	if (interface->host_ready)
679 		fm10k_watchdog_host_is_ready(interface);
680 	else
681 		fm10k_watchdog_host_not_ready(interface);
682 
683 	/* update stats only once every second */
684 	if (time_is_before_jiffies(interface->next_stats_update))
685 		fm10k_update_stats(interface);
686 
687 	/* flush any uncompleted work */
688 	fm10k_watchdog_flush_tx(interface);
689 }
690 
691 /**
692  * fm10k_check_hang_subtask - check for hung queues and dropped interrupts
693  * @interface: pointer to the device interface structure
694  *
695  * This function serves two purposes.  First it strobes the interrupt lines
696  * in order to make certain interrupts are occurring.  Secondly it sets the
697  * bits needed to check for TX hangs.  As a result we should immediately
698  * determine if a hang has occurred.
699  */
700 static void fm10k_check_hang_subtask(struct fm10k_intfc *interface)
701 {
702 	/* If we're down or resetting, just bail */
703 	if (test_bit(__FM10K_DOWN, interface->state) ||
704 	    test_bit(__FM10K_RESETTING, interface->state))
705 		return;
706 
707 	/* rate limit tx hang checks to only once every 2 seconds */
708 	if (time_is_after_eq_jiffies(interface->next_tx_hang_check))
709 		return;
710 	interface->next_tx_hang_check = jiffies + (2 * HZ);
711 
712 	if (netif_carrier_ok(interface->netdev)) {
713 		int i;
714 
715 		/* Force detection of hung controller */
716 		for (i = 0; i < interface->num_tx_queues; i++)
717 			set_check_for_tx_hang(interface->tx_ring[i]);
718 
719 		/* Rearm all in-use q_vectors for immediate firing */
720 		for (i = 0; i < interface->num_q_vectors; i++) {
721 			struct fm10k_q_vector *qv = interface->q_vector[i];
722 
723 			if (!qv->tx.count && !qv->rx.count)
724 				continue;
725 			writel(FM10K_ITR_ENABLE | FM10K_ITR_PENDING2, qv->itr);
726 		}
727 	}
728 }
729 
730 /**
731  * fm10k_service_task - manages and runs subtasks
732  * @work: pointer to work_struct containing our data
733  **/
734 static void fm10k_service_task(struct work_struct *work)
735 {
736 	struct fm10k_intfc *interface;
737 
738 	interface = container_of(work, struct fm10k_intfc, service_task);
739 
740 	/* Check whether we're detached first */
741 	fm10k_detach_subtask(interface);
742 
743 	/* tasks run even when interface is down */
744 	fm10k_mbx_subtask(interface);
745 	fm10k_reset_subtask(interface);
746 
747 	/* tasks only run when interface is up */
748 	fm10k_watchdog_subtask(interface);
749 	fm10k_check_hang_subtask(interface);
750 
751 	/* release lock on service events to allow scheduling next event */
752 	fm10k_service_event_complete(interface);
753 }
754 
755 /**
756  * fm10k_macvlan_task - send queued MAC/VLAN requests to switch manager
757  * @work: pointer to work_struct containing our data
758  *
759  * This work item handles sending MAC/VLAN updates to the switch manager. When
760  * the interface is up, it will attempt to queue mailbox messages to the
761  * switch manager requesting updates for MAC/VLAN pairs. If the Tx fifo of the
762  * mailbox is full, it will reschedule itself to try again in a short while.
763  * This ensures that the driver does not overload the switch mailbox with too
764  * many simultaneous requests, causing an unnecessary reset.
765  **/
766 static void fm10k_macvlan_task(struct work_struct *work)
767 {
768 	struct fm10k_macvlan_request *item;
769 	struct fm10k_intfc *interface;
770 	struct delayed_work *dwork;
771 	struct list_head *requests;
772 	struct fm10k_hw *hw;
773 	unsigned long flags;
774 
775 	dwork = to_delayed_work(work);
776 	interface = container_of(dwork, struct fm10k_intfc, macvlan_task);
777 	hw = &interface->hw;
778 	requests = &interface->macvlan_requests;
779 
780 	do {
781 		/* Pop the first item off the list */
782 		spin_lock_irqsave(&interface->macvlan_lock, flags);
783 		item = list_first_entry_or_null(requests,
784 						struct fm10k_macvlan_request,
785 						list);
786 		if (item)
787 			list_del_init(&item->list);
788 
789 		spin_unlock_irqrestore(&interface->macvlan_lock, flags);
790 
791 		/* We have no more items to process */
792 		if (!item)
793 			goto done;
794 
795 		fm10k_mbx_lock(interface);
796 
797 		/* Check that we have plenty of space to send the message. We
798 		 * want to ensure that the mailbox stays low enough to avoid a
799 		 * change in the host state, otherwise we may see spurious
800 		 * link up / link down notifications.
801 		 */
802 		if (!hw->mbx.ops.tx_ready(&hw->mbx, FM10K_VFMBX_MSG_MTU + 5)) {
803 			hw->mbx.ops.process(hw, &hw->mbx);
804 			set_bit(__FM10K_MACVLAN_REQUEST, interface->state);
805 			fm10k_mbx_unlock(interface);
806 
807 			/* Put the request back on the list */
808 			spin_lock_irqsave(&interface->macvlan_lock, flags);
809 			list_add(&item->list, requests);
810 			spin_unlock_irqrestore(&interface->macvlan_lock, flags);
811 			break;
812 		}
813 
814 		switch (item->type) {
815 		case FM10K_MC_MAC_REQUEST:
816 			hw->mac.ops.update_mc_addr(hw,
817 						   item->mac.glort,
818 						   item->mac.addr,
819 						   item->mac.vid,
820 						   item->set);
821 			break;
822 		case FM10K_UC_MAC_REQUEST:
823 			hw->mac.ops.update_uc_addr(hw,
824 						   item->mac.glort,
825 						   item->mac.addr,
826 						   item->mac.vid,
827 						   item->set,
828 						   0);
829 			break;
830 		case FM10K_VLAN_REQUEST:
831 			hw->mac.ops.update_vlan(hw,
832 						item->vlan.vid,
833 						item->vlan.vsi,
834 						item->set);
835 			break;
836 		default:
837 			break;
838 		}
839 
840 		fm10k_mbx_unlock(interface);
841 
842 		/* Free the item now that we've sent the update */
843 		kfree(item);
844 	} while (true);
845 
846 done:
847 	WARN_ON(!test_bit(__FM10K_MACVLAN_SCHED, interface->state));
848 
849 	/* flush memory to make sure state is correct */
850 	smp_mb__before_atomic();
851 	clear_bit(__FM10K_MACVLAN_SCHED, interface->state);
852 
853 	/* If a MAC/VLAN request was scheduled since we started, we should
854 	 * re-schedule. However, there is no reason to re-schedule if there is
855 	 * no work to do.
856 	 */
857 	if (test_bit(__FM10K_MACVLAN_REQUEST, interface->state))
858 		fm10k_macvlan_schedule(interface);
859 }
860 
861 /**
862  * fm10k_configure_tx_ring - Configure Tx ring after Reset
863  * @interface: board private structure
864  * @ring: structure containing ring specific data
865  *
866  * Configure the Tx descriptor ring after a reset.
867  **/
868 static void fm10k_configure_tx_ring(struct fm10k_intfc *interface,
869 				    struct fm10k_ring *ring)
870 {
871 	struct fm10k_hw *hw = &interface->hw;
872 	u64 tdba = ring->dma;
873 	u32 size = ring->count * sizeof(struct fm10k_tx_desc);
874 	u32 txint = FM10K_INT_MAP_DISABLE;
875 	u32 txdctl = BIT(FM10K_TXDCTL_MAX_TIME_SHIFT) | FM10K_TXDCTL_ENABLE;
876 	u8 reg_idx = ring->reg_idx;
877 
878 	/* disable queue to avoid issues while updating state */
879 	fm10k_write_reg(hw, FM10K_TXDCTL(reg_idx), 0);
880 	fm10k_write_flush(hw);
881 
882 	/* possible poll here to verify ring resources have been cleaned */
883 
884 	/* set location and size for descriptor ring */
885 	fm10k_write_reg(hw, FM10K_TDBAL(reg_idx), tdba & DMA_BIT_MASK(32));
886 	fm10k_write_reg(hw, FM10K_TDBAH(reg_idx), tdba >> 32);
887 	fm10k_write_reg(hw, FM10K_TDLEN(reg_idx), size);
888 
889 	/* reset head and tail pointers */
890 	fm10k_write_reg(hw, FM10K_TDH(reg_idx), 0);
891 	fm10k_write_reg(hw, FM10K_TDT(reg_idx), 0);
892 
893 	/* store tail pointer */
894 	ring->tail = &interface->uc_addr[FM10K_TDT(reg_idx)];
895 
896 	/* reset ntu and ntc to place SW in sync with hardware */
897 	ring->next_to_clean = 0;
898 	ring->next_to_use = 0;
899 
900 	/* Map interrupt */
901 	if (ring->q_vector) {
902 		txint = ring->q_vector->v_idx + NON_Q_VECTORS;
903 		txint |= FM10K_INT_MAP_TIMER0;
904 	}
905 
906 	fm10k_write_reg(hw, FM10K_TXINT(reg_idx), txint);
907 
908 	/* enable use of FTAG bit in Tx descriptor, register is RO for VF */
909 	fm10k_write_reg(hw, FM10K_PFVTCTL(reg_idx),
910 			FM10K_PFVTCTL_FTAG_DESC_ENABLE);
911 
912 	/* Initialize XPS */
913 	if (!test_and_set_bit(__FM10K_TX_XPS_INIT_DONE, ring->state) &&
914 	    ring->q_vector)
915 		netif_set_xps_queue(ring->netdev,
916 				    &ring->q_vector->affinity_mask,
917 				    ring->queue_index);
918 
919 	/* enable queue */
920 	fm10k_write_reg(hw, FM10K_TXDCTL(reg_idx), txdctl);
921 }
922 
923 /**
924  * fm10k_enable_tx_ring - Verify Tx ring is enabled after configuration
925  * @interface: board private structure
926  * @ring: structure containing ring specific data
927  *
928  * Verify the Tx descriptor ring is ready for transmit.
929  **/
930 static void fm10k_enable_tx_ring(struct fm10k_intfc *interface,
931 				 struct fm10k_ring *ring)
932 {
933 	struct fm10k_hw *hw = &interface->hw;
934 	int wait_loop = 10;
935 	u32 txdctl;
936 	u8 reg_idx = ring->reg_idx;
937 
938 	/* if we are already enabled just exit */
939 	if (fm10k_read_reg(hw, FM10K_TXDCTL(reg_idx)) & FM10K_TXDCTL_ENABLE)
940 		return;
941 
942 	/* poll to verify queue is enabled */
943 	do {
944 		usleep_range(1000, 2000);
945 		txdctl = fm10k_read_reg(hw, FM10K_TXDCTL(reg_idx));
946 	} while (!(txdctl & FM10K_TXDCTL_ENABLE) && --wait_loop);
947 	if (!wait_loop)
948 		netif_err(interface, drv, interface->netdev,
949 			  "Could not enable Tx Queue %d\n", reg_idx);
950 }
951 
952 /**
953  * fm10k_configure_tx - Configure Transmit Unit after Reset
954  * @interface: board private structure
955  *
956  * Configure the Tx unit of the MAC after a reset.
957  **/
958 static void fm10k_configure_tx(struct fm10k_intfc *interface)
959 {
960 	int i;
961 
962 	/* Setup the HW Tx Head and Tail descriptor pointers */
963 	for (i = 0; i < interface->num_tx_queues; i++)
964 		fm10k_configure_tx_ring(interface, interface->tx_ring[i]);
965 
966 	/* poll here to verify that Tx rings are now enabled */
967 	for (i = 0; i < interface->num_tx_queues; i++)
968 		fm10k_enable_tx_ring(interface, interface->tx_ring[i]);
969 }
970 
971 /**
972  * fm10k_configure_rx_ring - Configure Rx ring after Reset
973  * @interface: board private structure
974  * @ring: structure containing ring specific data
975  *
976  * Configure the Rx descriptor ring after a reset.
977  **/
978 static void fm10k_configure_rx_ring(struct fm10k_intfc *interface,
979 				    struct fm10k_ring *ring)
980 {
981 	u64 rdba = ring->dma;
982 	struct fm10k_hw *hw = &interface->hw;
983 	u32 size = ring->count * sizeof(union fm10k_rx_desc);
984 	u32 rxqctl, rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
985 	u32 srrctl = FM10K_SRRCTL_BUFFER_CHAINING_EN;
986 	u32 rxint = FM10K_INT_MAP_DISABLE;
987 	u8 rx_pause = interface->rx_pause;
988 	u8 reg_idx = ring->reg_idx;
989 
990 	/* disable queue to avoid issues while updating state */
991 	rxqctl = fm10k_read_reg(hw, FM10K_RXQCTL(reg_idx));
992 	rxqctl &= ~FM10K_RXQCTL_ENABLE;
993 	fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), rxqctl);
994 	fm10k_write_flush(hw);
995 
996 	/* possible poll here to verify ring resources have been cleaned */
997 
998 	/* set location and size for descriptor ring */
999 	fm10k_write_reg(hw, FM10K_RDBAL(reg_idx), rdba & DMA_BIT_MASK(32));
1000 	fm10k_write_reg(hw, FM10K_RDBAH(reg_idx), rdba >> 32);
1001 	fm10k_write_reg(hw, FM10K_RDLEN(reg_idx), size);
1002 
1003 	/* reset head and tail pointers */
1004 	fm10k_write_reg(hw, FM10K_RDH(reg_idx), 0);
1005 	fm10k_write_reg(hw, FM10K_RDT(reg_idx), 0);
1006 
1007 	/* store tail pointer */
1008 	ring->tail = &interface->uc_addr[FM10K_RDT(reg_idx)];
1009 
1010 	/* reset ntu and ntc to place SW in sync with hardware */
1011 	ring->next_to_clean = 0;
1012 	ring->next_to_use = 0;
1013 	ring->next_to_alloc = 0;
1014 
1015 	/* Configure the Rx buffer size for one buff without split */
1016 	srrctl |= FM10K_RX_BUFSZ >> FM10K_SRRCTL_BSIZEPKT_SHIFT;
1017 
1018 	/* Configure the Rx ring to suppress loopback packets */
1019 	srrctl |= FM10K_SRRCTL_LOOPBACK_SUPPRESS;
1020 	fm10k_write_reg(hw, FM10K_SRRCTL(reg_idx), srrctl);
1021 
1022 	/* Enable drop on empty */
1023 #ifdef CONFIG_DCB
1024 	if (interface->pfc_en)
1025 		rx_pause = interface->pfc_en;
1026 #endif
1027 	if (!(rx_pause & BIT(ring->qos_pc)))
1028 		rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
1029 
1030 	fm10k_write_reg(hw, FM10K_RXDCTL(reg_idx), rxdctl);
1031 
1032 	/* assign default VLAN to queue */
1033 	ring->vid = hw->mac.default_vid;
1034 
1035 	/* if we have an active VLAN, disable default VLAN ID */
1036 	if (test_bit(hw->mac.default_vid, interface->active_vlans))
1037 		ring->vid |= FM10K_VLAN_CLEAR;
1038 
1039 	/* Map interrupt */
1040 	if (ring->q_vector) {
1041 		rxint = ring->q_vector->v_idx + NON_Q_VECTORS;
1042 		rxint |= FM10K_INT_MAP_TIMER1;
1043 	}
1044 
1045 	fm10k_write_reg(hw, FM10K_RXINT(reg_idx), rxint);
1046 
1047 	/* enable queue */
1048 	rxqctl = fm10k_read_reg(hw, FM10K_RXQCTL(reg_idx));
1049 	rxqctl |= FM10K_RXQCTL_ENABLE;
1050 	fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), rxqctl);
1051 
1052 	/* place buffers on ring for receive data */
1053 	fm10k_alloc_rx_buffers(ring, fm10k_desc_unused(ring));
1054 }
1055 
1056 /**
1057  * fm10k_update_rx_drop_en - Configures the drop enable bits for Rx rings
1058  * @interface: board private structure
1059  *
1060  * Configure the drop enable bits for the Rx rings.
1061  **/
1062 void fm10k_update_rx_drop_en(struct fm10k_intfc *interface)
1063 {
1064 	struct fm10k_hw *hw = &interface->hw;
1065 	u8 rx_pause = interface->rx_pause;
1066 	int i;
1067 
1068 #ifdef CONFIG_DCB
1069 	if (interface->pfc_en)
1070 		rx_pause = interface->pfc_en;
1071 
1072 #endif
1073 	for (i = 0; i < interface->num_rx_queues; i++) {
1074 		struct fm10k_ring *ring = interface->rx_ring[i];
1075 		u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
1076 		u8 reg_idx = ring->reg_idx;
1077 
1078 		if (!(rx_pause & BIT(ring->qos_pc)))
1079 			rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
1080 
1081 		fm10k_write_reg(hw, FM10K_RXDCTL(reg_idx), rxdctl);
1082 	}
1083 }
1084 
1085 /**
1086  * fm10k_configure_dglort - Configure Receive DGLORT after reset
1087  * @interface: board private structure
1088  *
1089  * Configure the DGLORT description and RSS tables.
1090  **/
1091 static void fm10k_configure_dglort(struct fm10k_intfc *interface)
1092 {
1093 	struct fm10k_dglort_cfg dglort = { 0 };
1094 	struct fm10k_hw *hw = &interface->hw;
1095 	int i;
1096 	u32 mrqc;
1097 
1098 	/* Fill out hash function seeds */
1099 	for (i = 0; i < FM10K_RSSRK_SIZE; i++)
1100 		fm10k_write_reg(hw, FM10K_RSSRK(0, i), interface->rssrk[i]);
1101 
1102 	/* Write RETA table to hardware */
1103 	for (i = 0; i < FM10K_RETA_SIZE; i++)
1104 		fm10k_write_reg(hw, FM10K_RETA(0, i), interface->reta[i]);
1105 
1106 	/* Generate RSS hash based on packet types, TCP/UDP
1107 	 * port numbers and/or IPv4/v6 src and dst addresses
1108 	 */
1109 	mrqc = FM10K_MRQC_IPV4 |
1110 	       FM10K_MRQC_TCP_IPV4 |
1111 	       FM10K_MRQC_IPV6 |
1112 	       FM10K_MRQC_TCP_IPV6;
1113 
1114 	if (test_bit(FM10K_FLAG_RSS_FIELD_IPV4_UDP, interface->flags))
1115 		mrqc |= FM10K_MRQC_UDP_IPV4;
1116 	if (test_bit(FM10K_FLAG_RSS_FIELD_IPV6_UDP, interface->flags))
1117 		mrqc |= FM10K_MRQC_UDP_IPV6;
1118 
1119 	fm10k_write_reg(hw, FM10K_MRQC(0), mrqc);
1120 
1121 	/* configure default DGLORT mapping for RSS/DCB */
1122 	dglort.inner_rss = 1;
1123 	dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask);
1124 	dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask);
1125 	hw->mac.ops.configure_dglort_map(hw, &dglort);
1126 
1127 	/* assign GLORT per queue for queue mapped testing */
1128 	if (interface->glort_count > 64) {
1129 		memset(&dglort, 0, sizeof(dglort));
1130 		dglort.inner_rss = 1;
1131 		dglort.glort = interface->glort + 64;
1132 		dglort.idx = fm10k_dglort_pf_queue;
1133 		dglort.queue_l = fls(interface->num_rx_queues - 1);
1134 		hw->mac.ops.configure_dglort_map(hw, &dglort);
1135 	}
1136 
1137 	/* assign glort value for RSS/DCB specific to this interface */
1138 	memset(&dglort, 0, sizeof(dglort));
1139 	dglort.inner_rss = 1;
1140 	dglort.glort = interface->glort;
1141 	dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask);
1142 	dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask);
1143 	/* configure DGLORT mapping for RSS/DCB */
1144 	dglort.idx = fm10k_dglort_pf_rss;
1145 	if (interface->l2_accel)
1146 		dglort.shared_l = fls(interface->l2_accel->size);
1147 	hw->mac.ops.configure_dglort_map(hw, &dglort);
1148 }
1149 
1150 /**
1151  * fm10k_configure_rx - Configure Receive Unit after Reset
1152  * @interface: board private structure
1153  *
1154  * Configure the Rx unit of the MAC after a reset.
1155  **/
1156 static void fm10k_configure_rx(struct fm10k_intfc *interface)
1157 {
1158 	int i;
1159 
1160 	/* Configure SWPRI to PC map */
1161 	fm10k_configure_swpri_map(interface);
1162 
1163 	/* Configure RSS and DGLORT map */
1164 	fm10k_configure_dglort(interface);
1165 
1166 	/* Setup the HW Rx Head and Tail descriptor pointers */
1167 	for (i = 0; i < interface->num_rx_queues; i++)
1168 		fm10k_configure_rx_ring(interface, interface->rx_ring[i]);
1169 
1170 	/* possible poll here to verify that Rx rings are now enabled */
1171 }
1172 
1173 static void fm10k_napi_enable_all(struct fm10k_intfc *interface)
1174 {
1175 	struct fm10k_q_vector *q_vector;
1176 	int q_idx;
1177 
1178 	for (q_idx = 0; q_idx < interface->num_q_vectors; q_idx++) {
1179 		q_vector = interface->q_vector[q_idx];
1180 		napi_enable(&q_vector->napi);
1181 	}
1182 }
1183 
1184 static irqreturn_t fm10k_msix_clean_rings(int __always_unused irq, void *data)
1185 {
1186 	struct fm10k_q_vector *q_vector = data;
1187 
1188 	if (q_vector->rx.count || q_vector->tx.count)
1189 		napi_schedule_irqoff(&q_vector->napi);
1190 
1191 	return IRQ_HANDLED;
1192 }
1193 
1194 static irqreturn_t fm10k_msix_mbx_vf(int __always_unused irq, void *data)
1195 {
1196 	struct fm10k_intfc *interface = data;
1197 	struct fm10k_hw *hw = &interface->hw;
1198 	struct fm10k_mbx_info *mbx = &hw->mbx;
1199 
1200 	/* re-enable mailbox interrupt and indicate 20us delay */
1201 	fm10k_write_reg(hw, FM10K_VFITR(FM10K_MBX_VECTOR),
1202 			(FM10K_MBX_INT_DELAY >> hw->mac.itr_scale) |
1203 			FM10K_ITR_ENABLE);
1204 
1205 	/* service upstream mailbox */
1206 	if (fm10k_mbx_trylock(interface)) {
1207 		mbx->ops.process(hw, mbx);
1208 		fm10k_mbx_unlock(interface);
1209 	}
1210 
1211 	hw->mac.get_host_state = true;
1212 	fm10k_service_event_schedule(interface);
1213 
1214 	return IRQ_HANDLED;
1215 }
1216 
1217 #define FM10K_ERR_MSG(type) case (type): error = #type; break
1218 static void fm10k_handle_fault(struct fm10k_intfc *interface, int type,
1219 			       struct fm10k_fault *fault)
1220 {
1221 	struct pci_dev *pdev = interface->pdev;
1222 	struct fm10k_hw *hw = &interface->hw;
1223 	struct fm10k_iov_data *iov_data = interface->iov_data;
1224 	char *error;
1225 
1226 	switch (type) {
1227 	case FM10K_PCA_FAULT:
1228 		switch (fault->type) {
1229 		default:
1230 			error = "Unknown PCA error";
1231 			break;
1232 		FM10K_ERR_MSG(PCA_NO_FAULT);
1233 		FM10K_ERR_MSG(PCA_UNMAPPED_ADDR);
1234 		FM10K_ERR_MSG(PCA_BAD_QACCESS_PF);
1235 		FM10K_ERR_MSG(PCA_BAD_QACCESS_VF);
1236 		FM10K_ERR_MSG(PCA_MALICIOUS_REQ);
1237 		FM10K_ERR_MSG(PCA_POISONED_TLP);
1238 		FM10K_ERR_MSG(PCA_TLP_ABORT);
1239 		}
1240 		break;
1241 	case FM10K_THI_FAULT:
1242 		switch (fault->type) {
1243 		default:
1244 			error = "Unknown THI error";
1245 			break;
1246 		FM10K_ERR_MSG(THI_NO_FAULT);
1247 		FM10K_ERR_MSG(THI_MAL_DIS_Q_FAULT);
1248 		}
1249 		break;
1250 	case FM10K_FUM_FAULT:
1251 		switch (fault->type) {
1252 		default:
1253 			error = "Unknown FUM error";
1254 			break;
1255 		FM10K_ERR_MSG(FUM_NO_FAULT);
1256 		FM10K_ERR_MSG(FUM_UNMAPPED_ADDR);
1257 		FM10K_ERR_MSG(FUM_BAD_VF_QACCESS);
1258 		FM10K_ERR_MSG(FUM_ADD_DECODE_ERR);
1259 		FM10K_ERR_MSG(FUM_RO_ERROR);
1260 		FM10K_ERR_MSG(FUM_QPRC_CRC_ERROR);
1261 		FM10K_ERR_MSG(FUM_CSR_TIMEOUT);
1262 		FM10K_ERR_MSG(FUM_INVALID_TYPE);
1263 		FM10K_ERR_MSG(FUM_INVALID_LENGTH);
1264 		FM10K_ERR_MSG(FUM_INVALID_BE);
1265 		FM10K_ERR_MSG(FUM_INVALID_ALIGN);
1266 		}
1267 		break;
1268 	default:
1269 		error = "Undocumented fault";
1270 		break;
1271 	}
1272 
1273 	dev_warn(&pdev->dev,
1274 		 "%s Address: 0x%llx SpecInfo: 0x%x Func: %02x.%0x\n",
1275 		 error, fault->address, fault->specinfo,
1276 		 PCI_SLOT(fault->func), PCI_FUNC(fault->func));
1277 
1278 	/* For VF faults, clear out the respective LPORT, reset the queue
1279 	 * resources, and then reconnect to the mailbox. This allows the
1280 	 * VF in question to resume behavior. For transient faults that are
1281 	 * the result of non-malicious behavior this will log the fault and
1282 	 * allow the VF to resume functionality. Obviously for malicious VFs
1283 	 * they will be able to attempt malicious behavior again. In this
1284 	 * case, the system administrator will need to step in and manually
1285 	 * remove or disable the VF in question.
1286 	 */
1287 	if (fault->func && iov_data) {
1288 		int vf = fault->func - 1;
1289 		struct fm10k_vf_info *vf_info = &iov_data->vf_info[vf];
1290 
1291 		hw->iov.ops.reset_lport(hw, vf_info);
1292 		hw->iov.ops.reset_resources(hw, vf_info);
1293 
1294 		/* reset_lport disables the VF, so re-enable it */
1295 		hw->iov.ops.set_lport(hw, vf_info, vf,
1296 				      FM10K_VF_FLAG_MULTI_CAPABLE);
1297 
1298 		/* reset_resources will disconnect from the mbx  */
1299 		vf_info->mbx.ops.connect(hw, &vf_info->mbx);
1300 	}
1301 }
1302 
1303 static void fm10k_report_fault(struct fm10k_intfc *interface, u32 eicr)
1304 {
1305 	struct fm10k_hw *hw = &interface->hw;
1306 	struct fm10k_fault fault = { 0 };
1307 	int type, err;
1308 
1309 	for (eicr &= FM10K_EICR_FAULT_MASK, type = FM10K_PCA_FAULT;
1310 	     eicr;
1311 	     eicr >>= 1, type += FM10K_FAULT_SIZE) {
1312 		/* only check if there is an error reported */
1313 		if (!(eicr & 0x1))
1314 			continue;
1315 
1316 		/* retrieve fault info */
1317 		err = hw->mac.ops.get_fault(hw, type, &fault);
1318 		if (err) {
1319 			dev_err(&interface->pdev->dev,
1320 				"error reading fault\n");
1321 			continue;
1322 		}
1323 
1324 		fm10k_handle_fault(interface, type, &fault);
1325 	}
1326 }
1327 
1328 static void fm10k_reset_drop_on_empty(struct fm10k_intfc *interface, u32 eicr)
1329 {
1330 	struct fm10k_hw *hw = &interface->hw;
1331 	const u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
1332 	u32 maxholdq;
1333 	int q;
1334 
1335 	if (!(eicr & FM10K_EICR_MAXHOLDTIME))
1336 		return;
1337 
1338 	maxholdq = fm10k_read_reg(hw, FM10K_MAXHOLDQ(7));
1339 	if (maxholdq)
1340 		fm10k_write_reg(hw, FM10K_MAXHOLDQ(7), maxholdq);
1341 	for (q = 255;;) {
1342 		if (maxholdq & BIT(31)) {
1343 			if (q < FM10K_MAX_QUEUES_PF) {
1344 				interface->rx_overrun_pf++;
1345 				fm10k_write_reg(hw, FM10K_RXDCTL(q), rxdctl);
1346 			} else {
1347 				interface->rx_overrun_vf++;
1348 			}
1349 		}
1350 
1351 		maxholdq *= 2;
1352 		if (!maxholdq)
1353 			q &= ~(32 - 1);
1354 
1355 		if (!q)
1356 			break;
1357 
1358 		if (q-- % 32)
1359 			continue;
1360 
1361 		maxholdq = fm10k_read_reg(hw, FM10K_MAXHOLDQ(q / 32));
1362 		if (maxholdq)
1363 			fm10k_write_reg(hw, FM10K_MAXHOLDQ(q / 32), maxholdq);
1364 	}
1365 }
1366 
1367 static irqreturn_t fm10k_msix_mbx_pf(int __always_unused irq, void *data)
1368 {
1369 	struct fm10k_intfc *interface = data;
1370 	struct fm10k_hw *hw = &interface->hw;
1371 	struct fm10k_mbx_info *mbx = &hw->mbx;
1372 	u32 eicr;
1373 	s32 err = 0;
1374 
1375 	/* unmask any set bits related to this interrupt */
1376 	eicr = fm10k_read_reg(hw, FM10K_EICR);
1377 	fm10k_write_reg(hw, FM10K_EICR, eicr & (FM10K_EICR_MAILBOX |
1378 						FM10K_EICR_SWITCHREADY |
1379 						FM10K_EICR_SWITCHNOTREADY));
1380 
1381 	/* report any faults found to the message log */
1382 	fm10k_report_fault(interface, eicr);
1383 
1384 	/* reset any queues disabled due to receiver overrun */
1385 	fm10k_reset_drop_on_empty(interface, eicr);
1386 
1387 	/* service mailboxes */
1388 	if (fm10k_mbx_trylock(interface)) {
1389 		err = mbx->ops.process(hw, mbx);
1390 		/* handle VFLRE events */
1391 		fm10k_iov_event(interface);
1392 		fm10k_mbx_unlock(interface);
1393 	}
1394 
1395 	if (err == FM10K_ERR_RESET_REQUESTED)
1396 		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
1397 
1398 	/* if switch toggled state we should reset GLORTs */
1399 	if (eicr & FM10K_EICR_SWITCHNOTREADY) {
1400 		/* force link down for at least 4 seconds */
1401 		interface->link_down_event = jiffies + (4 * HZ);
1402 		set_bit(__FM10K_LINK_DOWN, interface->state);
1403 
1404 		/* reset dglort_map back to no config */
1405 		hw->mac.dglort_map = FM10K_DGLORTMAP_NONE;
1406 	}
1407 
1408 	/* we should validate host state after interrupt event */
1409 	hw->mac.get_host_state = true;
1410 
1411 	/* validate host state, and handle VF mailboxes in the service task */
1412 	fm10k_service_event_schedule(interface);
1413 
1414 	/* re-enable mailbox interrupt and indicate 20us delay */
1415 	fm10k_write_reg(hw, FM10K_ITR(FM10K_MBX_VECTOR),
1416 			(FM10K_MBX_INT_DELAY >> hw->mac.itr_scale) |
1417 			FM10K_ITR_ENABLE);
1418 
1419 	return IRQ_HANDLED;
1420 }
1421 
1422 void fm10k_mbx_free_irq(struct fm10k_intfc *interface)
1423 {
1424 	struct fm10k_hw *hw = &interface->hw;
1425 	struct msix_entry *entry;
1426 	int itr_reg;
1427 
1428 	/* no mailbox IRQ to free if MSI-X is not enabled */
1429 	if (!interface->msix_entries)
1430 		return;
1431 
1432 	entry = &interface->msix_entries[FM10K_MBX_VECTOR];
1433 
1434 	/* disconnect the mailbox */
1435 	hw->mbx.ops.disconnect(hw, &hw->mbx);
1436 
1437 	/* disable Mailbox cause */
1438 	if (hw->mac.type == fm10k_mac_pf) {
1439 		fm10k_write_reg(hw, FM10K_EIMR,
1440 				FM10K_EIMR_DISABLE(PCA_FAULT) |
1441 				FM10K_EIMR_DISABLE(FUM_FAULT) |
1442 				FM10K_EIMR_DISABLE(MAILBOX) |
1443 				FM10K_EIMR_DISABLE(SWITCHREADY) |
1444 				FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1445 				FM10K_EIMR_DISABLE(SRAMERROR) |
1446 				FM10K_EIMR_DISABLE(VFLR) |
1447 				FM10K_EIMR_DISABLE(MAXHOLDTIME));
1448 		itr_reg = FM10K_ITR(FM10K_MBX_VECTOR);
1449 	} else {
1450 		itr_reg = FM10K_VFITR(FM10K_MBX_VECTOR);
1451 	}
1452 
1453 	fm10k_write_reg(hw, itr_reg, FM10K_ITR_MASK_SET);
1454 
1455 	free_irq(entry->vector, interface);
1456 }
1457 
1458 static s32 fm10k_mbx_mac_addr(struct fm10k_hw *hw, u32 **results,
1459 			      struct fm10k_mbx_info *mbx)
1460 {
1461 	bool vlan_override = hw->mac.vlan_override;
1462 	u16 default_vid = hw->mac.default_vid;
1463 	struct fm10k_intfc *interface;
1464 	s32 err;
1465 
1466 	err = fm10k_msg_mac_vlan_vf(hw, results, mbx);
1467 	if (err)
1468 		return err;
1469 
1470 	interface = container_of(hw, struct fm10k_intfc, hw);
1471 
1472 	/* MAC was changed so we need reset */
1473 	if (is_valid_ether_addr(hw->mac.perm_addr) &&
1474 	    !ether_addr_equal(hw->mac.perm_addr, hw->mac.addr))
1475 		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
1476 
1477 	/* VLAN override was changed, or default VLAN changed */
1478 	if ((vlan_override != hw->mac.vlan_override) ||
1479 	    (default_vid != hw->mac.default_vid))
1480 		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
1481 
1482 	return 0;
1483 }
1484 
1485 /* generic error handler for mailbox issues */
1486 static s32 fm10k_mbx_error(struct fm10k_hw *hw, u32 **results,
1487 			   struct fm10k_mbx_info __always_unused *mbx)
1488 {
1489 	struct fm10k_intfc *interface;
1490 	struct pci_dev *pdev;
1491 
1492 	interface = container_of(hw, struct fm10k_intfc, hw);
1493 	pdev = interface->pdev;
1494 
1495 	dev_err(&pdev->dev, "Unknown message ID %u\n",
1496 		**results & FM10K_TLV_ID_MASK);
1497 
1498 	return 0;
1499 }
1500 
1501 static const struct fm10k_msg_data vf_mbx_data[] = {
1502 	FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1503 	FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_mbx_mac_addr),
1504 	FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1505 	FM10K_TLV_MSG_ERROR_HANDLER(fm10k_mbx_error),
1506 };
1507 
1508 static int fm10k_mbx_request_irq_vf(struct fm10k_intfc *interface)
1509 {
1510 	struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR];
1511 	struct net_device *dev = interface->netdev;
1512 	struct fm10k_hw *hw = &interface->hw;
1513 	int err;
1514 
1515 	/* Use timer0 for interrupt moderation on the mailbox */
1516 	u32 itr = entry->entry | FM10K_INT_MAP_TIMER0;
1517 
1518 	/* register mailbox handlers */
1519 	err = hw->mbx.ops.register_handlers(&hw->mbx, vf_mbx_data);
1520 	if (err)
1521 		return err;
1522 
1523 	/* request the IRQ */
1524 	err = request_irq(entry->vector, fm10k_msix_mbx_vf, 0,
1525 			  dev->name, interface);
1526 	if (err) {
1527 		netif_err(interface, probe, dev,
1528 			  "request_irq for msix_mbx failed: %d\n", err);
1529 		return err;
1530 	}
1531 
1532 	/* map all of the interrupt sources */
1533 	fm10k_write_reg(hw, FM10K_VFINT_MAP, itr);
1534 
1535 	/* enable interrupt */
1536 	fm10k_write_reg(hw, FM10K_VFITR(entry->entry), FM10K_ITR_ENABLE);
1537 
1538 	return 0;
1539 }
1540 
1541 static s32 fm10k_lport_map(struct fm10k_hw *hw, u32 **results,
1542 			   struct fm10k_mbx_info *mbx)
1543 {
1544 	struct fm10k_intfc *interface;
1545 	u32 dglort_map = hw->mac.dglort_map;
1546 	s32 err;
1547 
1548 	interface = container_of(hw, struct fm10k_intfc, hw);
1549 
1550 	err = fm10k_msg_err_pf(hw, results, mbx);
1551 	if (!err && hw->swapi.status) {
1552 		/* force link down for a reasonable delay */
1553 		interface->link_down_event = jiffies + (2 * HZ);
1554 		set_bit(__FM10K_LINK_DOWN, interface->state);
1555 
1556 		/* reset dglort_map back to no config */
1557 		hw->mac.dglort_map = FM10K_DGLORTMAP_NONE;
1558 
1559 		fm10k_service_event_schedule(interface);
1560 
1561 		/* prevent overloading kernel message buffer */
1562 		if (interface->lport_map_failed)
1563 			return 0;
1564 
1565 		interface->lport_map_failed = true;
1566 
1567 		if (hw->swapi.status == FM10K_MSG_ERR_PEP_NOT_SCHEDULED)
1568 			dev_warn(&interface->pdev->dev,
1569 				 "cannot obtain link because the host interface is configured for a PCIe host interface bandwidth of zero\n");
1570 		dev_warn(&interface->pdev->dev,
1571 			 "request logical port map failed: %d\n",
1572 			 hw->swapi.status);
1573 
1574 		return 0;
1575 	}
1576 
1577 	err = fm10k_msg_lport_map_pf(hw, results, mbx);
1578 	if (err)
1579 		return err;
1580 
1581 	interface->lport_map_failed = false;
1582 
1583 	/* we need to reset if port count was just updated */
1584 	if (dglort_map != hw->mac.dglort_map)
1585 		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
1586 
1587 	return 0;
1588 }
1589 
1590 static s32 fm10k_update_pvid(struct fm10k_hw *hw, u32 **results,
1591 			     struct fm10k_mbx_info __always_unused *mbx)
1592 {
1593 	struct fm10k_intfc *interface;
1594 	u16 glort, pvid;
1595 	u32 pvid_update;
1596 	s32 err;
1597 
1598 	err = fm10k_tlv_attr_get_u32(results[FM10K_PF_ATTR_ID_UPDATE_PVID],
1599 				     &pvid_update);
1600 	if (err)
1601 		return err;
1602 
1603 	/* extract values from the pvid update */
1604 	glort = FM10K_MSG_HDR_FIELD_GET(pvid_update, UPDATE_PVID_GLORT);
1605 	pvid = FM10K_MSG_HDR_FIELD_GET(pvid_update, UPDATE_PVID_PVID);
1606 
1607 	/* if glort is not valid return error */
1608 	if (!fm10k_glort_valid_pf(hw, glort))
1609 		return FM10K_ERR_PARAM;
1610 
1611 	/* verify VLAN ID is valid */
1612 	if (pvid >= FM10K_VLAN_TABLE_VID_MAX)
1613 		return FM10K_ERR_PARAM;
1614 
1615 	interface = container_of(hw, struct fm10k_intfc, hw);
1616 
1617 	/* check to see if this belongs to one of the VFs */
1618 	err = fm10k_iov_update_pvid(interface, glort, pvid);
1619 	if (!err)
1620 		return 0;
1621 
1622 	/* we need to reset if default VLAN was just updated */
1623 	if (pvid != hw->mac.default_vid)
1624 		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
1625 
1626 	hw->mac.default_vid = pvid;
1627 
1628 	return 0;
1629 }
1630 
1631 static const struct fm10k_msg_data pf_mbx_data[] = {
1632 	FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1633 	FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1634 	FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_lport_map),
1635 	FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1636 	FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1637 	FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_update_pvid),
1638 	FM10K_TLV_MSG_ERROR_HANDLER(fm10k_mbx_error),
1639 };
1640 
1641 static int fm10k_mbx_request_irq_pf(struct fm10k_intfc *interface)
1642 {
1643 	struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR];
1644 	struct net_device *dev = interface->netdev;
1645 	struct fm10k_hw *hw = &interface->hw;
1646 	int err;
1647 
1648 	/* Use timer0 for interrupt moderation on the mailbox */
1649 	u32 mbx_itr = entry->entry | FM10K_INT_MAP_TIMER0;
1650 	u32 other_itr = entry->entry | FM10K_INT_MAP_IMMEDIATE;
1651 
1652 	/* register mailbox handlers */
1653 	err = hw->mbx.ops.register_handlers(&hw->mbx, pf_mbx_data);
1654 	if (err)
1655 		return err;
1656 
1657 	/* request the IRQ */
1658 	err = request_irq(entry->vector, fm10k_msix_mbx_pf, 0,
1659 			  dev->name, interface);
1660 	if (err) {
1661 		netif_err(interface, probe, dev,
1662 			  "request_irq for msix_mbx failed: %d\n", err);
1663 		return err;
1664 	}
1665 
1666 	/* Enable interrupts w/ no moderation for "other" interrupts */
1667 	fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), other_itr);
1668 	fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), other_itr);
1669 	fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_sram), other_itr);
1670 	fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_max_hold_time), other_itr);
1671 	fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_vflr), other_itr);
1672 
1673 	/* Enable interrupts w/ moderation for mailbox */
1674 	fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_mailbox), mbx_itr);
1675 
1676 	/* Enable individual interrupt causes */
1677 	fm10k_write_reg(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1678 					FM10K_EIMR_ENABLE(FUM_FAULT) |
1679 					FM10K_EIMR_ENABLE(MAILBOX) |
1680 					FM10K_EIMR_ENABLE(SWITCHREADY) |
1681 					FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1682 					FM10K_EIMR_ENABLE(SRAMERROR) |
1683 					FM10K_EIMR_ENABLE(VFLR) |
1684 					FM10K_EIMR_ENABLE(MAXHOLDTIME));
1685 
1686 	/* enable interrupt */
1687 	fm10k_write_reg(hw, FM10K_ITR(entry->entry), FM10K_ITR_ENABLE);
1688 
1689 	return 0;
1690 }
1691 
1692 int fm10k_mbx_request_irq(struct fm10k_intfc *interface)
1693 {
1694 	struct fm10k_hw *hw = &interface->hw;
1695 	int err;
1696 
1697 	/* enable Mailbox cause */
1698 	if (hw->mac.type == fm10k_mac_pf)
1699 		err = fm10k_mbx_request_irq_pf(interface);
1700 	else
1701 		err = fm10k_mbx_request_irq_vf(interface);
1702 	if (err)
1703 		return err;
1704 
1705 	/* connect mailbox */
1706 	err = hw->mbx.ops.connect(hw, &hw->mbx);
1707 
1708 	/* if the mailbox failed to connect, then free IRQ */
1709 	if (err)
1710 		fm10k_mbx_free_irq(interface);
1711 
1712 	return err;
1713 }
1714 
1715 /**
1716  * fm10k_qv_free_irq - release interrupts associated with queue vectors
1717  * @interface: board private structure
1718  *
1719  * Release all interrupts associated with this interface
1720  **/
1721 void fm10k_qv_free_irq(struct fm10k_intfc *interface)
1722 {
1723 	int vector = interface->num_q_vectors;
1724 	struct msix_entry *entry;
1725 
1726 	entry = &interface->msix_entries[NON_Q_VECTORS + vector];
1727 
1728 	while (vector) {
1729 		struct fm10k_q_vector *q_vector;
1730 
1731 		vector--;
1732 		entry--;
1733 		q_vector = interface->q_vector[vector];
1734 
1735 		if (!q_vector->tx.count && !q_vector->rx.count)
1736 			continue;
1737 
1738 		/* clear the affinity_mask in the IRQ descriptor */
1739 		irq_set_affinity_hint(entry->vector, NULL);
1740 
1741 		/* disable interrupts */
1742 		writel(FM10K_ITR_MASK_SET, q_vector->itr);
1743 
1744 		free_irq(entry->vector, q_vector);
1745 	}
1746 }
1747 
1748 /**
1749  * fm10k_qv_request_irq - initialize interrupts for queue vectors
1750  * @interface: board private structure
1751  *
1752  * Attempts to configure interrupts using the best available
1753  * capabilities of the hardware and kernel.
1754  **/
1755 int fm10k_qv_request_irq(struct fm10k_intfc *interface)
1756 {
1757 	struct net_device *dev = interface->netdev;
1758 	struct fm10k_hw *hw = &interface->hw;
1759 	struct msix_entry *entry;
1760 	unsigned int ri = 0, ti = 0;
1761 	int vector, err;
1762 
1763 	entry = &interface->msix_entries[NON_Q_VECTORS];
1764 
1765 	for (vector = 0; vector < interface->num_q_vectors; vector++) {
1766 		struct fm10k_q_vector *q_vector = interface->q_vector[vector];
1767 
1768 		/* name the vector */
1769 		if (q_vector->tx.count && q_vector->rx.count) {
1770 			snprintf(q_vector->name, sizeof(q_vector->name),
1771 				 "%s-TxRx-%u", dev->name, ri++);
1772 			ti++;
1773 		} else if (q_vector->rx.count) {
1774 			snprintf(q_vector->name, sizeof(q_vector->name),
1775 				 "%s-rx-%u", dev->name, ri++);
1776 		} else if (q_vector->tx.count) {
1777 			snprintf(q_vector->name, sizeof(q_vector->name),
1778 				 "%s-tx-%u", dev->name, ti++);
1779 		} else {
1780 			/* skip this unused q_vector */
1781 			continue;
1782 		}
1783 
1784 		/* Assign ITR register to q_vector */
1785 		q_vector->itr = (hw->mac.type == fm10k_mac_pf) ?
1786 				&interface->uc_addr[FM10K_ITR(entry->entry)] :
1787 				&interface->uc_addr[FM10K_VFITR(entry->entry)];
1788 
1789 		/* request the IRQ */
1790 		err = request_irq(entry->vector, &fm10k_msix_clean_rings, 0,
1791 				  q_vector->name, q_vector);
1792 		if (err) {
1793 			netif_err(interface, probe, dev,
1794 				  "request_irq failed for MSIX interrupt Error: %d\n",
1795 				  err);
1796 			goto err_out;
1797 		}
1798 
1799 		/* assign the mask for this irq */
1800 		irq_set_affinity_hint(entry->vector, &q_vector->affinity_mask);
1801 
1802 		/* Enable q_vector */
1803 		writel(FM10K_ITR_ENABLE, q_vector->itr);
1804 
1805 		entry++;
1806 	}
1807 
1808 	return 0;
1809 
1810 err_out:
1811 	/* wind through the ring freeing all entries and vectors */
1812 	while (vector) {
1813 		struct fm10k_q_vector *q_vector;
1814 
1815 		entry--;
1816 		vector--;
1817 		q_vector = interface->q_vector[vector];
1818 
1819 		if (!q_vector->tx.count && !q_vector->rx.count)
1820 			continue;
1821 
1822 		/* clear the affinity_mask in the IRQ descriptor */
1823 		irq_set_affinity_hint(entry->vector, NULL);
1824 
1825 		/* disable interrupts */
1826 		writel(FM10K_ITR_MASK_SET, q_vector->itr);
1827 
1828 		free_irq(entry->vector, q_vector);
1829 	}
1830 
1831 	return err;
1832 }
1833 
1834 void fm10k_up(struct fm10k_intfc *interface)
1835 {
1836 	struct fm10k_hw *hw = &interface->hw;
1837 
1838 	/* Enable Tx/Rx DMA */
1839 	hw->mac.ops.start_hw(hw);
1840 
1841 	/* configure Tx descriptor rings */
1842 	fm10k_configure_tx(interface);
1843 
1844 	/* configure Rx descriptor rings */
1845 	fm10k_configure_rx(interface);
1846 
1847 	/* configure interrupts */
1848 	hw->mac.ops.update_int_moderator(hw);
1849 
1850 	/* enable statistics capture again */
1851 	clear_bit(__FM10K_UPDATING_STATS, interface->state);
1852 
1853 	/* clear down bit to indicate we are ready to go */
1854 	clear_bit(__FM10K_DOWN, interface->state);
1855 
1856 	/* enable polling cleanups */
1857 	fm10k_napi_enable_all(interface);
1858 
1859 	/* re-establish Rx filters */
1860 	fm10k_restore_rx_state(interface);
1861 
1862 	/* enable transmits */
1863 	netif_tx_start_all_queues(interface->netdev);
1864 
1865 	/* kick off the service timer now */
1866 	hw->mac.get_host_state = true;
1867 	mod_timer(&interface->service_timer, jiffies);
1868 }
1869 
1870 static void fm10k_napi_disable_all(struct fm10k_intfc *interface)
1871 {
1872 	struct fm10k_q_vector *q_vector;
1873 	int q_idx;
1874 
1875 	for (q_idx = 0; q_idx < interface->num_q_vectors; q_idx++) {
1876 		q_vector = interface->q_vector[q_idx];
1877 		napi_disable(&q_vector->napi);
1878 	}
1879 }
1880 
1881 void fm10k_down(struct fm10k_intfc *interface)
1882 {
1883 	struct net_device *netdev = interface->netdev;
1884 	struct fm10k_hw *hw = &interface->hw;
1885 	int err, i = 0, count = 0;
1886 
1887 	/* signal that we are down to the interrupt handler and service task */
1888 	if (test_and_set_bit(__FM10K_DOWN, interface->state))
1889 		return;
1890 
1891 	/* call carrier off first to avoid false dev_watchdog timeouts */
1892 	netif_carrier_off(netdev);
1893 
1894 	/* disable transmits */
1895 	netif_tx_stop_all_queues(netdev);
1896 	netif_tx_disable(netdev);
1897 
1898 	/* reset Rx filters */
1899 	fm10k_reset_rx_state(interface);
1900 
1901 	/* disable polling routines */
1902 	fm10k_napi_disable_all(interface);
1903 
1904 	/* capture stats one last time before stopping interface */
1905 	fm10k_update_stats(interface);
1906 
1907 	/* prevent updating statistics while we're down */
1908 	while (test_and_set_bit(__FM10K_UPDATING_STATS, interface->state))
1909 		usleep_range(1000, 2000);
1910 
1911 	/* skip waiting for TX DMA if we lost PCIe link */
1912 	if (FM10K_REMOVED(hw->hw_addr))
1913 		goto skip_tx_dma_drain;
1914 
1915 	/* In some rare circumstances it can take a while for Tx queues to
1916 	 * quiesce and be fully disabled. Attempt to .stop_hw() first, and
1917 	 * then if we get ERR_REQUESTS_PENDING, go ahead and wait in a loop
1918 	 * until the Tx queues have emptied, or until a number of retries. If
1919 	 * we fail to clear within the retry loop, we will issue a warning
1920 	 * indicating that Tx DMA is probably hung. Note this means we call
1921 	 * .stop_hw() twice but this shouldn't cause any problems.
1922 	 */
1923 	err = hw->mac.ops.stop_hw(hw);
1924 	if (err != FM10K_ERR_REQUESTS_PENDING)
1925 		goto skip_tx_dma_drain;
1926 
1927 #define TX_DMA_DRAIN_RETRIES 25
1928 	for (count = 0; count < TX_DMA_DRAIN_RETRIES; count++) {
1929 		usleep_range(10000, 20000);
1930 
1931 		/* start checking at the last ring to have pending Tx */
1932 		for (; i < interface->num_tx_queues; i++)
1933 			if (fm10k_get_tx_pending(interface->tx_ring[i], false))
1934 				break;
1935 
1936 		/* if all the queues are drained, we can break now */
1937 		if (i == interface->num_tx_queues)
1938 			break;
1939 	}
1940 
1941 	if (count >= TX_DMA_DRAIN_RETRIES)
1942 		dev_err(&interface->pdev->dev,
1943 			"Tx queues failed to drain after %d tries. Tx DMA is probably hung.\n",
1944 			count);
1945 skip_tx_dma_drain:
1946 	/* Disable DMA engine for Tx/Rx */
1947 	err = hw->mac.ops.stop_hw(hw);
1948 	if (err == FM10K_ERR_REQUESTS_PENDING)
1949 		dev_err(&interface->pdev->dev,
1950 			"due to pending requests hw was not shut down gracefully\n");
1951 	else if (err)
1952 		dev_err(&interface->pdev->dev, "stop_hw failed: %d\n", err);
1953 
1954 	/* free any buffers still on the rings */
1955 	fm10k_clean_all_tx_rings(interface);
1956 	fm10k_clean_all_rx_rings(interface);
1957 }
1958 
1959 /**
1960  * fm10k_sw_init - Initialize general software structures
1961  * @interface: host interface private structure to initialize
1962  * @ent: PCI device ID entry
1963  *
1964  * fm10k_sw_init initializes the interface private data structure.
1965  * Fields are initialized based on PCI device information and
1966  * OS network device settings (MTU size).
1967  **/
1968 static int fm10k_sw_init(struct fm10k_intfc *interface,
1969 			 const struct pci_device_id *ent)
1970 {
1971 	const struct fm10k_info *fi = fm10k_info_tbl[ent->driver_data];
1972 	struct fm10k_hw *hw = &interface->hw;
1973 	struct pci_dev *pdev = interface->pdev;
1974 	struct net_device *netdev = interface->netdev;
1975 	u32 rss_key[FM10K_RSSRK_SIZE];
1976 	unsigned int rss;
1977 	int err;
1978 
1979 	/* initialize back pointer */
1980 	hw->back = interface;
1981 	hw->hw_addr = interface->uc_addr;
1982 
1983 	/* PCI config space info */
1984 	hw->vendor_id = pdev->vendor;
1985 	hw->device_id = pdev->device;
1986 	hw->revision_id = pdev->revision;
1987 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
1988 	hw->subsystem_device_id = pdev->subsystem_device;
1989 
1990 	/* Setup hw api */
1991 	memcpy(&hw->mac.ops, fi->mac_ops, sizeof(hw->mac.ops));
1992 	hw->mac.type = fi->mac;
1993 
1994 	/* Setup IOV handlers */
1995 	if (fi->iov_ops)
1996 		memcpy(&hw->iov.ops, fi->iov_ops, sizeof(hw->iov.ops));
1997 
1998 	/* Set common capability flags and settings */
1999 	rss = min_t(int, FM10K_MAX_RSS_INDICES, num_online_cpus());
2000 	interface->ring_feature[RING_F_RSS].limit = rss;
2001 	fi->get_invariants(hw);
2002 
2003 	/* pick up the PCIe bus settings for reporting later */
2004 	if (hw->mac.ops.get_bus_info)
2005 		hw->mac.ops.get_bus_info(hw);
2006 
2007 	/* limit the usable DMA range */
2008 	if (hw->mac.ops.set_dma_mask)
2009 		hw->mac.ops.set_dma_mask(hw, dma_get_mask(&pdev->dev));
2010 
2011 	/* update netdev with DMA restrictions */
2012 	if (dma_get_mask(&pdev->dev) > DMA_BIT_MASK(32)) {
2013 		netdev->features |= NETIF_F_HIGHDMA;
2014 		netdev->vlan_features |= NETIF_F_HIGHDMA;
2015 	}
2016 
2017 	/* reset and initialize the hardware so it is in a known state */
2018 	err = hw->mac.ops.reset_hw(hw);
2019 	if (err) {
2020 		dev_err(&pdev->dev, "reset_hw failed: %d\n", err);
2021 		return err;
2022 	}
2023 
2024 	err = hw->mac.ops.init_hw(hw);
2025 	if (err) {
2026 		dev_err(&pdev->dev, "init_hw failed: %d\n", err);
2027 		return err;
2028 	}
2029 
2030 	/* initialize hardware statistics */
2031 	hw->mac.ops.update_hw_stats(hw, &interface->stats);
2032 
2033 	/* Set upper limit on IOV VFs that can be allocated */
2034 	pci_sriov_set_totalvfs(pdev, hw->iov.total_vfs);
2035 
2036 	/* Start with random Ethernet address */
2037 	eth_random_addr(hw->mac.addr);
2038 
2039 	/* Initialize MAC address from hardware */
2040 	err = hw->mac.ops.read_mac_addr(hw);
2041 	if (err) {
2042 		dev_warn(&pdev->dev,
2043 			 "Failed to obtain MAC address defaulting to random\n");
2044 		/* tag address assignment as random */
2045 		netdev->addr_assign_type |= NET_ADDR_RANDOM;
2046 	}
2047 
2048 	ether_addr_copy(netdev->dev_addr, hw->mac.addr);
2049 	ether_addr_copy(netdev->perm_addr, hw->mac.addr);
2050 
2051 	if (!is_valid_ether_addr(netdev->perm_addr)) {
2052 		dev_err(&pdev->dev, "Invalid MAC Address\n");
2053 		return -EIO;
2054 	}
2055 
2056 	/* initialize DCBNL interface */
2057 	fm10k_dcbnl_set_ops(netdev);
2058 
2059 	/* set default ring sizes */
2060 	interface->tx_ring_count = FM10K_DEFAULT_TXD;
2061 	interface->rx_ring_count = FM10K_DEFAULT_RXD;
2062 
2063 	/* set default interrupt moderation */
2064 	interface->tx_itr = FM10K_TX_ITR_DEFAULT;
2065 	interface->rx_itr = FM10K_ITR_ADAPTIVE | FM10K_RX_ITR_DEFAULT;
2066 
2067 	/* Initialize the MAC/VLAN queue */
2068 	INIT_LIST_HEAD(&interface->macvlan_requests);
2069 
2070 	netdev_rss_key_fill(rss_key, sizeof(rss_key));
2071 	memcpy(interface->rssrk, rss_key, sizeof(rss_key));
2072 
2073 	/* Initialize the mailbox lock */
2074 	spin_lock_init(&interface->mbx_lock);
2075 	spin_lock_init(&interface->macvlan_lock);
2076 
2077 	/* Start off interface as being down */
2078 	set_bit(__FM10K_DOWN, interface->state);
2079 	set_bit(__FM10K_UPDATING_STATS, interface->state);
2080 
2081 	return 0;
2082 }
2083 
2084 /**
2085  * fm10k_probe - Device Initialization Routine
2086  * @pdev: PCI device information struct
2087  * @ent: entry in fm10k_pci_tbl
2088  *
2089  * Returns 0 on success, negative on failure
2090  *
2091  * fm10k_probe initializes an interface identified by a pci_dev structure.
2092  * The OS initialization, configuring of the interface private structure,
2093  * and a hardware reset occur.
2094  **/
2095 static int fm10k_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2096 {
2097 	struct net_device *netdev;
2098 	struct fm10k_intfc *interface;
2099 	int err;
2100 
2101 	if (pdev->error_state != pci_channel_io_normal) {
2102 		dev_err(&pdev->dev,
2103 			"PCI device still in an error state. Unable to load...\n");
2104 		return -EIO;
2105 	}
2106 
2107 	err = pci_enable_device_mem(pdev);
2108 	if (err) {
2109 		dev_err(&pdev->dev,
2110 			"PCI enable device failed: %d\n", err);
2111 		return err;
2112 	}
2113 
2114 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
2115 	if (err)
2116 		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
2117 	if (err) {
2118 		dev_err(&pdev->dev,
2119 			"DMA configuration failed: %d\n", err);
2120 		goto err_dma;
2121 	}
2122 
2123 	err = pci_request_mem_regions(pdev, fm10k_driver_name);
2124 	if (err) {
2125 		dev_err(&pdev->dev,
2126 			"pci_request_selected_regions failed: %d\n", err);
2127 		goto err_pci_reg;
2128 	}
2129 
2130 	pci_enable_pcie_error_reporting(pdev);
2131 
2132 	pci_set_master(pdev);
2133 	pci_save_state(pdev);
2134 
2135 	netdev = fm10k_alloc_netdev(fm10k_info_tbl[ent->driver_data]);
2136 	if (!netdev) {
2137 		err = -ENOMEM;
2138 		goto err_alloc_netdev;
2139 	}
2140 
2141 	SET_NETDEV_DEV(netdev, &pdev->dev);
2142 
2143 	interface = netdev_priv(netdev);
2144 	pci_set_drvdata(pdev, interface);
2145 
2146 	interface->netdev = netdev;
2147 	interface->pdev = pdev;
2148 
2149 	interface->uc_addr = ioremap(pci_resource_start(pdev, 0),
2150 				     FM10K_UC_ADDR_SIZE);
2151 	if (!interface->uc_addr) {
2152 		err = -EIO;
2153 		goto err_ioremap;
2154 	}
2155 
2156 	err = fm10k_sw_init(interface, ent);
2157 	if (err)
2158 		goto err_sw_init;
2159 
2160 	/* enable debugfs support */
2161 	fm10k_dbg_intfc_init(interface);
2162 
2163 	err = fm10k_init_queueing_scheme(interface);
2164 	if (err)
2165 		goto err_sw_init;
2166 
2167 	/* the mbx interrupt might attempt to schedule the service task, so we
2168 	 * must ensure it is disabled since we haven't yet requested the timer
2169 	 * or work item.
2170 	 */
2171 	set_bit(__FM10K_SERVICE_DISABLE, interface->state);
2172 
2173 	err = fm10k_mbx_request_irq(interface);
2174 	if (err)
2175 		goto err_mbx_interrupt;
2176 
2177 	/* final check of hardware state before registering the interface */
2178 	err = fm10k_hw_ready(interface);
2179 	if (err)
2180 		goto err_register;
2181 
2182 	err = register_netdev(netdev);
2183 	if (err)
2184 		goto err_register;
2185 
2186 	/* carrier off reporting is important to ethtool even BEFORE open */
2187 	netif_carrier_off(netdev);
2188 
2189 	/* stop all the transmit queues from transmitting until link is up */
2190 	netif_tx_stop_all_queues(netdev);
2191 
2192 	/* Initialize service timer and service task late in order to avoid
2193 	 * cleanup issues.
2194 	 */
2195 	timer_setup(&interface->service_timer, fm10k_service_timer, 0);
2196 	INIT_WORK(&interface->service_task, fm10k_service_task);
2197 
2198 	/* Setup the MAC/VLAN queue */
2199 	INIT_DELAYED_WORK(&interface->macvlan_task, fm10k_macvlan_task);
2200 
2201 	/* kick off service timer now, even when interface is down */
2202 	mod_timer(&interface->service_timer, (HZ * 2) + jiffies);
2203 
2204 	/* print warning for non-optimal configurations */
2205 	pcie_print_link_status(interface->pdev);
2206 
2207 	/* report MAC address for logging */
2208 	dev_info(&pdev->dev, "%pM\n", netdev->dev_addr);
2209 
2210 	/* enable SR-IOV after registering netdev to enforce PF/VF ordering */
2211 	fm10k_iov_configure(pdev, 0);
2212 
2213 	/* clear the service task disable bit and kick off service task */
2214 	clear_bit(__FM10K_SERVICE_DISABLE, interface->state);
2215 	fm10k_service_event_schedule(interface);
2216 
2217 	return 0;
2218 
2219 err_register:
2220 	fm10k_mbx_free_irq(interface);
2221 err_mbx_interrupt:
2222 	fm10k_clear_queueing_scheme(interface);
2223 err_sw_init:
2224 	if (interface->sw_addr)
2225 		iounmap(interface->sw_addr);
2226 	iounmap(interface->uc_addr);
2227 err_ioremap:
2228 	free_netdev(netdev);
2229 err_alloc_netdev:
2230 	pci_release_mem_regions(pdev);
2231 err_pci_reg:
2232 err_dma:
2233 	pci_disable_device(pdev);
2234 	return err;
2235 }
2236 
2237 /**
2238  * fm10k_remove - Device Removal Routine
2239  * @pdev: PCI device information struct
2240  *
2241  * fm10k_remove is called by the PCI subsystem to alert the driver
2242  * that it should release a PCI device.  The could be caused by a
2243  * Hot-Plug event, or because the driver is going to be removed from
2244  * memory.
2245  **/
2246 static void fm10k_remove(struct pci_dev *pdev)
2247 {
2248 	struct fm10k_intfc *interface = pci_get_drvdata(pdev);
2249 	struct net_device *netdev = interface->netdev;
2250 
2251 	del_timer_sync(&interface->service_timer);
2252 
2253 	fm10k_stop_service_event(interface);
2254 	fm10k_stop_macvlan_task(interface);
2255 
2256 	/* Remove all pending MAC/VLAN requests */
2257 	fm10k_clear_macvlan_queue(interface, interface->glort, true);
2258 
2259 	/* free netdev, this may bounce the interrupts due to setup_tc */
2260 	if (netdev->reg_state == NETREG_REGISTERED)
2261 		unregister_netdev(netdev);
2262 
2263 	/* release VFs */
2264 	fm10k_iov_disable(pdev);
2265 
2266 	/* disable mailbox interrupt */
2267 	fm10k_mbx_free_irq(interface);
2268 
2269 	/* free interrupts */
2270 	fm10k_clear_queueing_scheme(interface);
2271 
2272 	/* remove any debugfs interfaces */
2273 	fm10k_dbg_intfc_exit(interface);
2274 
2275 	if (interface->sw_addr)
2276 		iounmap(interface->sw_addr);
2277 	iounmap(interface->uc_addr);
2278 
2279 	free_netdev(netdev);
2280 
2281 	pci_release_mem_regions(pdev);
2282 
2283 	pci_disable_pcie_error_reporting(pdev);
2284 
2285 	pci_disable_device(pdev);
2286 }
2287 
2288 static void fm10k_prepare_suspend(struct fm10k_intfc *interface)
2289 {
2290 	/* the watchdog task reads from registers, which might appear like
2291 	 * a surprise remove if the PCIe device is disabled while we're
2292 	 * stopped. We stop the watchdog task until after we resume software
2293 	 * activity.
2294 	 *
2295 	 * Note that the MAC/VLAN task will be stopped as part of preparing
2296 	 * for reset so we don't need to handle it here.
2297 	 */
2298 	fm10k_stop_service_event(interface);
2299 
2300 	if (fm10k_prepare_for_reset(interface))
2301 		set_bit(__FM10K_RESET_SUSPENDED, interface->state);
2302 }
2303 
2304 static int fm10k_handle_resume(struct fm10k_intfc *interface)
2305 {
2306 	struct fm10k_hw *hw = &interface->hw;
2307 	int err;
2308 
2309 	/* Even if we didn't properly prepare for reset in
2310 	 * fm10k_prepare_suspend, we'll attempt to resume anyways.
2311 	 */
2312 	if (!test_and_clear_bit(__FM10K_RESET_SUSPENDED, interface->state))
2313 		dev_warn(&interface->pdev->dev,
2314 			 "Device was shut down as part of suspend... Attempting to recover\n");
2315 
2316 	/* reset statistics starting values */
2317 	hw->mac.ops.rebind_hw_stats(hw, &interface->stats);
2318 
2319 	err = fm10k_handle_reset(interface);
2320 	if (err)
2321 		return err;
2322 
2323 	/* assume host is not ready, to prevent race with watchdog in case we
2324 	 * actually don't have connection to the switch
2325 	 */
2326 	interface->host_ready = false;
2327 	fm10k_watchdog_host_not_ready(interface);
2328 
2329 	/* force link to stay down for a second to prevent link flutter */
2330 	interface->link_down_event = jiffies + (HZ);
2331 	set_bit(__FM10K_LINK_DOWN, interface->state);
2332 
2333 	/* restart the service task */
2334 	fm10k_start_service_event(interface);
2335 
2336 	/* Restart the MAC/VLAN request queue in-case of outstanding events */
2337 	fm10k_macvlan_schedule(interface);
2338 
2339 	return 0;
2340 }
2341 
2342 /**
2343  * fm10k_resume - Generic PM resume hook
2344  * @dev: generic device structure
2345  *
2346  * Generic PM hook used when waking the device from a low power state after
2347  * suspend or hibernation. This function does not need to handle lower PCIe
2348  * device state as the stack takes care of that for us.
2349  **/
2350 static int __maybe_unused fm10k_resume(struct device *dev)
2351 {
2352 	struct fm10k_intfc *interface = dev_get_drvdata(dev);
2353 	struct net_device *netdev = interface->netdev;
2354 	struct fm10k_hw *hw = &interface->hw;
2355 	int err;
2356 
2357 	/* refresh hw_addr in case it was dropped */
2358 	hw->hw_addr = interface->uc_addr;
2359 
2360 	err = fm10k_handle_resume(interface);
2361 	if (err)
2362 		return err;
2363 
2364 	netif_device_attach(netdev);
2365 
2366 	return 0;
2367 }
2368 
2369 /**
2370  * fm10k_suspend - Generic PM suspend hook
2371  * @dev: generic device structure
2372  *
2373  * Generic PM hook used when setting the device into a low power state for
2374  * system suspend or hibernation. This function does not need to handle lower
2375  * PCIe device state as the stack takes care of that for us.
2376  **/
2377 static int __maybe_unused fm10k_suspend(struct device *dev)
2378 {
2379 	struct fm10k_intfc *interface = dev_get_drvdata(dev);
2380 	struct net_device *netdev = interface->netdev;
2381 
2382 	netif_device_detach(netdev);
2383 
2384 	fm10k_prepare_suspend(interface);
2385 
2386 	return 0;
2387 }
2388 
2389 /**
2390  * fm10k_io_error_detected - called when PCI error is detected
2391  * @pdev: Pointer to PCI device
2392  * @state: The current pci connection state
2393  *
2394  * This function is called after a PCI bus error affecting
2395  * this device has been detected.
2396  */
2397 static pci_ers_result_t fm10k_io_error_detected(struct pci_dev *pdev,
2398 						pci_channel_state_t state)
2399 {
2400 	struct fm10k_intfc *interface = pci_get_drvdata(pdev);
2401 	struct net_device *netdev = interface->netdev;
2402 
2403 	netif_device_detach(netdev);
2404 
2405 	if (state == pci_channel_io_perm_failure)
2406 		return PCI_ERS_RESULT_DISCONNECT;
2407 
2408 	fm10k_prepare_suspend(interface);
2409 
2410 	/* Request a slot reset. */
2411 	return PCI_ERS_RESULT_NEED_RESET;
2412 }
2413 
2414 /**
2415  * fm10k_io_slot_reset - called after the pci bus has been reset.
2416  * @pdev: Pointer to PCI device
2417  *
2418  * Restart the card from scratch, as if from a cold-boot.
2419  */
2420 static pci_ers_result_t fm10k_io_slot_reset(struct pci_dev *pdev)
2421 {
2422 	pci_ers_result_t result;
2423 
2424 	if (pci_reenable_device(pdev)) {
2425 		dev_err(&pdev->dev,
2426 			"Cannot re-enable PCI device after reset.\n");
2427 		result = PCI_ERS_RESULT_DISCONNECT;
2428 	} else {
2429 		pci_set_master(pdev);
2430 		pci_restore_state(pdev);
2431 
2432 		/* After second error pci->state_saved is false, this
2433 		 * resets it so EEH doesn't break.
2434 		 */
2435 		pci_save_state(pdev);
2436 
2437 		pci_wake_from_d3(pdev, false);
2438 
2439 		result = PCI_ERS_RESULT_RECOVERED;
2440 	}
2441 
2442 	return result;
2443 }
2444 
2445 /**
2446  * fm10k_io_resume - called when traffic can start flowing again.
2447  * @pdev: Pointer to PCI device
2448  *
2449  * This callback is called when the error recovery driver tells us that
2450  * its OK to resume normal operation.
2451  */
2452 static void fm10k_io_resume(struct pci_dev *pdev)
2453 {
2454 	struct fm10k_intfc *interface = pci_get_drvdata(pdev);
2455 	struct net_device *netdev = interface->netdev;
2456 	int err;
2457 
2458 	err = fm10k_handle_resume(interface);
2459 
2460 	if (err)
2461 		dev_warn(&pdev->dev,
2462 			 "%s failed: %d\n", __func__, err);
2463 	else
2464 		netif_device_attach(netdev);
2465 }
2466 
2467 /**
2468  * fm10k_io_reset_prepare - called when PCI function is about to be reset
2469  * @pdev: Pointer to PCI device
2470  *
2471  * This callback is called when the PCI function is about to be reset,
2472  * allowing the device driver to prepare for it.
2473  */
2474 static void fm10k_io_reset_prepare(struct pci_dev *pdev)
2475 {
2476 	/* warn incase we have any active VF devices */
2477 	if (pci_num_vf(pdev))
2478 		dev_warn(&pdev->dev,
2479 			 "PCIe FLR may cause issues for any active VF devices\n");
2480 	fm10k_prepare_suspend(pci_get_drvdata(pdev));
2481 }
2482 
2483 /**
2484  * fm10k_io_reset_done - called when PCI function has finished resetting
2485  * @pdev: Pointer to PCI device
2486  *
2487  * This callback is called just after the PCI function is reset, such as via
2488  * /sys/class/net/<enpX>/device/reset or similar.
2489  */
2490 static void fm10k_io_reset_done(struct pci_dev *pdev)
2491 {
2492 	struct fm10k_intfc *interface = pci_get_drvdata(pdev);
2493 	int err = fm10k_handle_resume(interface);
2494 
2495 	if (err) {
2496 		dev_warn(&pdev->dev,
2497 			 "%s failed: %d\n", __func__, err);
2498 		netif_device_detach(interface->netdev);
2499 	}
2500 }
2501 
2502 static const struct pci_error_handlers fm10k_err_handler = {
2503 	.error_detected = fm10k_io_error_detected,
2504 	.slot_reset = fm10k_io_slot_reset,
2505 	.resume = fm10k_io_resume,
2506 	.reset_prepare = fm10k_io_reset_prepare,
2507 	.reset_done = fm10k_io_reset_done,
2508 };
2509 
2510 static SIMPLE_DEV_PM_OPS(fm10k_pm_ops, fm10k_suspend, fm10k_resume);
2511 
2512 static struct pci_driver fm10k_driver = {
2513 	.name			= fm10k_driver_name,
2514 	.id_table		= fm10k_pci_tbl,
2515 	.probe			= fm10k_probe,
2516 	.remove			= fm10k_remove,
2517 	.driver = {
2518 		.pm		= &fm10k_pm_ops,
2519 	},
2520 	.sriov_configure	= fm10k_iov_configure,
2521 	.err_handler		= &fm10k_err_handler
2522 };
2523 
2524 /**
2525  * fm10k_register_pci_driver - register driver interface
2526  *
2527  * This function is called on module load in order to register the driver.
2528  **/
2529 int fm10k_register_pci_driver(void)
2530 {
2531 	return pci_register_driver(&fm10k_driver);
2532 }
2533 
2534 /**
2535  * fm10k_unregister_pci_driver - unregister driver interface
2536  *
2537  * This function is called on module unload in order to remove the driver.
2538  **/
2539 void fm10k_unregister_pci_driver(void)
2540 {
2541 	pci_unregister_driver(&fm10k_driver);
2542 }
2543