1 /**********************************************************************
2  * Author: Cavium, Inc.
3  *
4  * Contact: support@cavium.com
5  *          Please include "LiquidIO" in the subject.
6  *
7  * Copyright (c) 2003-2016 Cavium, Inc.
8  *
9  * This file is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License, Version 2, as
11  * published by the Free Software Foundation.
12  *
13  * This file is distributed in the hope that it will be useful, but
14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16  * NONINFRINGEMENT.  See the GNU General Public License for more details.
17  ***********************************************************************/
18 #include <linux/module.h>
19 #include <linux/interrupt.h>
20 #include <linux/pci.h>
21 #include <linux/firmware.h>
22 #include <net/vxlan.h>
23 #include <linux/kthread.h>
24 #include "liquidio_common.h"
25 #include "octeon_droq.h"
26 #include "octeon_iq.h"
27 #include "response_manager.h"
28 #include "octeon_device.h"
29 #include "octeon_nic.h"
30 #include "octeon_main.h"
31 #include "octeon_network.h"
32 #include "cn66xx_regs.h"
33 #include "cn66xx_device.h"
34 #include "cn68xx_device.h"
35 #include "cn23xx_pf_device.h"
36 #include "liquidio_image.h"
37 #include "lio_vf_rep.h"
38 
39 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
40 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
41 MODULE_LICENSE("GPL");
42 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME
43 		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
44 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME
45 		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
46 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME
47 		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
48 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME
49 		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
50 
51 static int ddr_timeout = 10000;
52 module_param(ddr_timeout, int, 0644);
53 MODULE_PARM_DESC(ddr_timeout,
54 		 "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
55 
56 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
57 
58 static int debug = -1;
59 module_param(debug, int, 0644);
60 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
61 
62 static char fw_type[LIO_MAX_FW_TYPE_LEN] = LIO_FW_NAME_TYPE_AUTO;
63 module_param_string(fw_type, fw_type, sizeof(fw_type), 0444);
64 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded (default is \"auto\"), which uses firmware in flash, if present, else loads \"nic\".");
65 
66 static u32 console_bitmask;
67 module_param(console_bitmask, int, 0644);
68 MODULE_PARM_DESC(console_bitmask,
69 		 "Bitmask indicating which consoles have debug output redirected to syslog.");
70 
71 /**
72  * octeon_console_debug_enabled - determines if a given console has debug enabled.
73  * @console: console to check
74  * Return:  1 = enabled. 0 otherwise
75  */
76 static int octeon_console_debug_enabled(u32 console)
77 {
78 	return (console_bitmask >> (console)) & 0x1;
79 }
80 
81 /* Polling interval for determining when NIC application is alive */
82 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
83 
84 /* runtime link query interval */
85 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS         1000
86 /* update localtime to octeon firmware every 60 seconds.
87  * make firmware to use same time reference, so that it will be easy to
88  * correlate firmware logged events/errors with host events, for debugging.
89  */
90 #define LIO_SYNC_OCTEON_TIME_INTERVAL_MS 60000
91 
92 /* time to wait for possible in-flight requests in milliseconds */
93 #define WAIT_INFLIGHT_REQUEST	msecs_to_jiffies(1000)
94 
95 struct oct_link_status_resp {
96 	u64 rh;
97 	struct oct_link_info link_info;
98 	u64 status;
99 };
100 
101 struct oct_timestamp_resp {
102 	u64 rh;
103 	u64 timestamp;
104 	u64 status;
105 };
106 
107 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
108 
109 union tx_info {
110 	u64 u64;
111 	struct {
112 #ifdef __BIG_ENDIAN_BITFIELD
113 		u16 gso_size;
114 		u16 gso_segs;
115 		u32 reserved;
116 #else
117 		u32 reserved;
118 		u16 gso_segs;
119 		u16 gso_size;
120 #endif
121 	} s;
122 };
123 
124 /* Octeon device properties to be used by the NIC module.
125  * Each octeon device in the system will be represented
126  * by this structure in the NIC module.
127  */
128 
129 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
130 #define OCTNIC_GSO_MAX_SIZE                                                    \
131 	(CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
132 
133 struct handshake {
134 	struct completion init;
135 	struct completion started;
136 	struct pci_dev *pci_dev;
137 	int init_ok;
138 	int started_ok;
139 };
140 
141 #ifdef CONFIG_PCI_IOV
142 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs);
143 #endif
144 
145 static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
146 				    char *prefix, char *suffix);
147 
148 static int octeon_device_init(struct octeon_device *);
149 static int liquidio_stop(struct net_device *netdev);
150 static void liquidio_remove(struct pci_dev *pdev);
151 static int liquidio_probe(struct pci_dev *pdev,
152 			  const struct pci_device_id *ent);
153 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
154 				      int linkstate);
155 
156 static struct handshake handshake[MAX_OCTEON_DEVICES];
157 static struct completion first_stage;
158 
159 static void octeon_droq_bh(struct tasklet_struct *t)
160 {
161 	int q_no;
162 	int reschedule = 0;
163 	struct octeon_device_priv *oct_priv = from_tasklet(oct_priv, t,
164 							  droq_tasklet);
165 	struct octeon_device *oct = oct_priv->dev;
166 
167 	for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
168 		if (!(oct->io_qmask.oq & BIT_ULL(q_no)))
169 			continue;
170 		reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
171 							  MAX_PACKET_BUDGET);
172 		lio_enable_irq(oct->droq[q_no], NULL);
173 
174 		if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
175 			/* set time and cnt interrupt thresholds for this DROQ
176 			 * for NAPI
177 			 */
178 			int adjusted_q_no = q_no + oct->sriov_info.pf_srn;
179 
180 			octeon_write_csr64(
181 			    oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(adjusted_q_no),
182 			    0x5700000040ULL);
183 			octeon_write_csr64(
184 			    oct, CN23XX_SLI_OQ_PKTS_SENT(adjusted_q_no), 0);
185 		}
186 	}
187 
188 	if (reschedule)
189 		tasklet_schedule(&oct_priv->droq_tasklet);
190 }
191 
192 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
193 {
194 	struct octeon_device_priv *oct_priv = oct->priv;
195 	int retry = 100, pkt_cnt = 0, pending_pkts = 0;
196 	int i;
197 
198 	do {
199 		pending_pkts = 0;
200 
201 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
202 			if (!(oct->io_qmask.oq & BIT_ULL(i)))
203 				continue;
204 			pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
205 		}
206 		if (pkt_cnt > 0) {
207 			pending_pkts += pkt_cnt;
208 			tasklet_schedule(&oct_priv->droq_tasklet);
209 		}
210 		pkt_cnt = 0;
211 		schedule_timeout_uninterruptible(1);
212 
213 	} while (retry-- && pending_pkts);
214 
215 	return pkt_cnt;
216 }
217 
218 /**
219  * force_io_queues_off - Forces all IO queues off on a given device
220  * @oct: Pointer to Octeon device
221  */
222 static void force_io_queues_off(struct octeon_device *oct)
223 {
224 	if ((oct->chip_id == OCTEON_CN66XX) ||
225 	    (oct->chip_id == OCTEON_CN68XX)) {
226 		/* Reset the Enable bits for Input Queues. */
227 		octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
228 
229 		/* Reset the Enable bits for Output Queues. */
230 		octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
231 	}
232 }
233 
234 /**
235  * pcierror_quiesce_device - Cause device to go quiet so it can be safely removed/reset/etc
236  * @oct: Pointer to Octeon device
237  */
238 static inline void pcierror_quiesce_device(struct octeon_device *oct)
239 {
240 	int i;
241 
242 	/* Disable the input and output queues now. No more packets will
243 	 * arrive from Octeon, but we should wait for all packet processing
244 	 * to finish.
245 	 */
246 	force_io_queues_off(oct);
247 
248 	/* To allow for in-flight requests */
249 	schedule_timeout_uninterruptible(WAIT_INFLIGHT_REQUEST);
250 
251 	if (wait_for_pending_requests(oct))
252 		dev_err(&oct->pci_dev->dev, "There were pending requests\n");
253 
254 	/* Force all requests waiting to be fetched by OCTEON to complete. */
255 	for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
256 		struct octeon_instr_queue *iq;
257 
258 		if (!(oct->io_qmask.iq & BIT_ULL(i)))
259 			continue;
260 		iq = oct->instr_queue[i];
261 
262 		if (atomic_read(&iq->instr_pending)) {
263 			spin_lock_bh(&iq->lock);
264 			iq->fill_cnt = 0;
265 			iq->octeon_read_index = iq->host_write_index;
266 			iq->stats.instr_processed +=
267 				atomic_read(&iq->instr_pending);
268 			lio_process_iq_request_list(oct, iq, 0);
269 			spin_unlock_bh(&iq->lock);
270 		}
271 	}
272 
273 	/* Force all pending ordered list requests to time out. */
274 	lio_process_ordered_list(oct, 1);
275 
276 	/* We do not need to wait for output queue packets to be processed. */
277 }
278 
279 /**
280  * cleanup_aer_uncorrect_error_status - Cleanup PCI AER uncorrectable error status
281  * @dev: Pointer to PCI device
282  */
283 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
284 {
285 	int pos = 0x100;
286 	u32 status, mask;
287 
288 	pr_info("%s :\n", __func__);
289 
290 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
291 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
292 	if (dev->error_state == pci_channel_io_normal)
293 		status &= ~mask;        /* Clear corresponding nonfatal bits */
294 	else
295 		status &= mask;         /* Clear corresponding fatal bits */
296 	pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
297 }
298 
299 /**
300  * stop_pci_io - Stop all PCI IO to a given device
301  * @oct: Pointer to Octeon device
302  */
303 static void stop_pci_io(struct octeon_device *oct)
304 {
305 	/* No more instructions will be forwarded. */
306 	atomic_set(&oct->status, OCT_DEV_IN_RESET);
307 
308 	pci_disable_device(oct->pci_dev);
309 
310 	/* Disable interrupts  */
311 	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
312 
313 	pcierror_quiesce_device(oct);
314 
315 	/* Release the interrupt line */
316 	free_irq(oct->pci_dev->irq, oct);
317 
318 	if (oct->flags & LIO_FLAG_MSI_ENABLED)
319 		pci_disable_msi(oct->pci_dev);
320 
321 	dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
322 		lio_get_state_string(&oct->status));
323 
324 	/* making it a common function for all OCTEON models */
325 	cleanup_aer_uncorrect_error_status(oct->pci_dev);
326 }
327 
328 /**
329  * liquidio_pcie_error_detected - called when PCI error is detected
330  * @pdev: Pointer to PCI device
331  * @state: The current pci connection state
332  *
333  * This function is called after a PCI bus error affecting
334  * this device has been detected.
335  */
336 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
337 						     pci_channel_state_t state)
338 {
339 	struct octeon_device *oct = pci_get_drvdata(pdev);
340 
341 	/* Non-correctable Non-fatal errors */
342 	if (state == pci_channel_io_normal) {
343 		dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
344 		cleanup_aer_uncorrect_error_status(oct->pci_dev);
345 		return PCI_ERS_RESULT_CAN_RECOVER;
346 	}
347 
348 	/* Non-correctable Fatal errors */
349 	dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
350 	stop_pci_io(oct);
351 
352 	/* Always return a DISCONNECT. There is no support for recovery but only
353 	 * for a clean shutdown.
354 	 */
355 	return PCI_ERS_RESULT_DISCONNECT;
356 }
357 
358 /**
359  * liquidio_pcie_mmio_enabled - mmio handler
360  * @pdev: Pointer to PCI device
361  */
362 static pci_ers_result_t liquidio_pcie_mmio_enabled(struct pci_dev __maybe_unused *pdev)
363 {
364 	/* We should never hit this since we never ask for a reset for a Fatal
365 	 * Error. We always return DISCONNECT in io_error above.
366 	 * But play safe and return RECOVERED for now.
367 	 */
368 	return PCI_ERS_RESULT_RECOVERED;
369 }
370 
371 /**
372  * liquidio_pcie_slot_reset - called after the pci bus has been reset.
373  * @pdev: Pointer to PCI device
374  *
375  * Restart the card from scratch, as if from a cold-boot. Implementation
376  * resembles the first-half of the octeon_resume routine.
377  */
378 static pci_ers_result_t liquidio_pcie_slot_reset(struct pci_dev __maybe_unused *pdev)
379 {
380 	/* We should never hit this since we never ask for a reset for a Fatal
381 	 * Error. We always return DISCONNECT in io_error above.
382 	 * But play safe and return RECOVERED for now.
383 	 */
384 	return PCI_ERS_RESULT_RECOVERED;
385 }
386 
387 /**
388  * liquidio_pcie_resume - called when traffic can start flowing again.
389  * @pdev: Pointer to PCI device
390  *
391  * This callback is called when the error recovery driver tells us that
392  * its OK to resume normal operation. Implementation resembles the
393  * second-half of the octeon_resume routine.
394  */
395 static void liquidio_pcie_resume(struct pci_dev __maybe_unused *pdev)
396 {
397 	/* Nothing to be done here. */
398 }
399 
400 #define liquidio_suspend NULL
401 #define liquidio_resume NULL
402 
403 /* For PCI-E Advanced Error Recovery (AER) Interface */
404 static const struct pci_error_handlers liquidio_err_handler = {
405 	.error_detected = liquidio_pcie_error_detected,
406 	.mmio_enabled	= liquidio_pcie_mmio_enabled,
407 	.slot_reset	= liquidio_pcie_slot_reset,
408 	.resume		= liquidio_pcie_resume,
409 };
410 
411 static const struct pci_device_id liquidio_pci_tbl[] = {
412 	{       /* 68xx */
413 		PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
414 	},
415 	{       /* 66xx */
416 		PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
417 	},
418 	{       /* 23xx pf */
419 		PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
420 	},
421 	{
422 		0, 0, 0, 0, 0, 0, 0
423 	}
424 };
425 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
426 
427 static SIMPLE_DEV_PM_OPS(liquidio_pm_ops, liquidio_suspend, liquidio_resume);
428 
429 static struct pci_driver liquidio_pci_driver = {
430 	.name		= "LiquidIO",
431 	.id_table	= liquidio_pci_tbl,
432 	.probe		= liquidio_probe,
433 	.remove		= liquidio_remove,
434 	.err_handler	= &liquidio_err_handler,    /* For AER */
435 	.driver.pm	= &liquidio_pm_ops,
436 #ifdef CONFIG_PCI_IOV
437 	.sriov_configure = liquidio_enable_sriov,
438 #endif
439 };
440 
441 /**
442  * liquidio_init_pci - register PCI driver
443  */
444 static int liquidio_init_pci(void)
445 {
446 	return pci_register_driver(&liquidio_pci_driver);
447 }
448 
449 /**
450  * liquidio_deinit_pci - unregister PCI driver
451  */
452 static void liquidio_deinit_pci(void)
453 {
454 	pci_unregister_driver(&liquidio_pci_driver);
455 }
456 
457 /**
458  * check_txq_status - Check Tx queue status, and take appropriate action
459  * @lio: per-network private data
460  * Return: 0 if full, number of queues woken up otherwise
461  */
462 static inline int check_txq_status(struct lio *lio)
463 {
464 	int numqs = lio->netdev->real_num_tx_queues;
465 	int ret_val = 0;
466 	int q, iq;
467 
468 	/* check each sub-queue state */
469 	for (q = 0; q < numqs; q++) {
470 		iq = lio->linfo.txpciq[q %
471 			lio->oct_dev->num_iqs].s.q_no;
472 		if (octnet_iq_is_full(lio->oct_dev, iq))
473 			continue;
474 		if (__netif_subqueue_stopped(lio->netdev, q)) {
475 			netif_wake_subqueue(lio->netdev, q);
476 			INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
477 						  tx_restart, 1);
478 			ret_val++;
479 		}
480 	}
481 
482 	return ret_val;
483 }
484 
485 /**
486  * print_link_info -  Print link information
487  * @netdev: network device
488  */
489 static void print_link_info(struct net_device *netdev)
490 {
491 	struct lio *lio = GET_LIO(netdev);
492 
493 	if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
494 	    ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
495 		struct oct_link_info *linfo = &lio->linfo;
496 
497 		if (linfo->link.s.link_up) {
498 			netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
499 				   linfo->link.s.speed,
500 				   (linfo->link.s.duplex) ? "Full" : "Half");
501 		} else {
502 			netif_info(lio, link, lio->netdev, "Link Down\n");
503 		}
504 	}
505 }
506 
507 /**
508  * octnet_link_status_change - Routine to notify MTU change
509  * @work: work_struct data structure
510  */
511 static void octnet_link_status_change(struct work_struct *work)
512 {
513 	struct cavium_wk *wk = (struct cavium_wk *)work;
514 	struct lio *lio = (struct lio *)wk->ctxptr;
515 
516 	/* lio->linfo.link.s.mtu always contains max MTU of the lio interface.
517 	 * this API is invoked only when new max-MTU of the interface is
518 	 * less than current MTU.
519 	 */
520 	rtnl_lock();
521 	dev_set_mtu(lio->netdev, lio->linfo.link.s.mtu);
522 	rtnl_unlock();
523 }
524 
525 /**
526  * setup_link_status_change_wq - Sets up the mtu status change work
527  * @netdev: network device
528  */
529 static inline int setup_link_status_change_wq(struct net_device *netdev)
530 {
531 	struct lio *lio = GET_LIO(netdev);
532 	struct octeon_device *oct = lio->oct_dev;
533 
534 	lio->link_status_wq.wq = alloc_workqueue("link-status",
535 						 WQ_MEM_RECLAIM, 0);
536 	if (!lio->link_status_wq.wq) {
537 		dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");
538 		return -1;
539 	}
540 	INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,
541 			  octnet_link_status_change);
542 	lio->link_status_wq.wk.ctxptr = lio;
543 
544 	return 0;
545 }
546 
547 static inline void cleanup_link_status_change_wq(struct net_device *netdev)
548 {
549 	struct lio *lio = GET_LIO(netdev);
550 
551 	if (lio->link_status_wq.wq) {
552 		cancel_delayed_work_sync(&lio->link_status_wq.wk.work);
553 		destroy_workqueue(lio->link_status_wq.wq);
554 	}
555 }
556 
557 /**
558  * update_link_status - Update link status
559  * @netdev: network device
560  * @ls: link status structure
561  *
562  * Called on receipt of a link status response from the core application to
563  * update each interface's link status.
564  */
565 static inline void update_link_status(struct net_device *netdev,
566 				      union oct_link_status *ls)
567 {
568 	struct lio *lio = GET_LIO(netdev);
569 	int changed = (lio->linfo.link.u64 != ls->u64);
570 	int current_max_mtu = lio->linfo.link.s.mtu;
571 	struct octeon_device *oct = lio->oct_dev;
572 
573 	dev_dbg(&oct->pci_dev->dev, "%s: lio->linfo.link.u64=%llx, ls->u64=%llx\n",
574 		__func__, lio->linfo.link.u64, ls->u64);
575 	lio->linfo.link.u64 = ls->u64;
576 
577 	if ((lio->intf_open) && (changed)) {
578 		print_link_info(netdev);
579 		lio->link_changes++;
580 
581 		if (lio->linfo.link.s.link_up) {
582 			dev_dbg(&oct->pci_dev->dev, "%s: link_up", __func__);
583 			netif_carrier_on(netdev);
584 			wake_txqs(netdev);
585 		} else {
586 			dev_dbg(&oct->pci_dev->dev, "%s: link_off", __func__);
587 			netif_carrier_off(netdev);
588 			stop_txqs(netdev);
589 		}
590 		if (lio->linfo.link.s.mtu != current_max_mtu) {
591 			netif_info(lio, probe, lio->netdev, "Max MTU changed from %d to %d\n",
592 				   current_max_mtu, lio->linfo.link.s.mtu);
593 			netdev->max_mtu = lio->linfo.link.s.mtu;
594 		}
595 		if (lio->linfo.link.s.mtu < netdev->mtu) {
596 			dev_warn(&oct->pci_dev->dev,
597 				 "Current MTU is higher than new max MTU; Reducing the current mtu from %d to %d\n",
598 				     netdev->mtu, lio->linfo.link.s.mtu);
599 			queue_delayed_work(lio->link_status_wq.wq,
600 					   &lio->link_status_wq.wk.work, 0);
601 		}
602 	}
603 }
604 
605 /**
606  * lio_sync_octeon_time - send latest localtime to octeon firmware so that
607  * firmware will correct it's time, in case there is a time skew
608  *
609  * @work: work scheduled to send time update to octeon firmware
610  **/
611 static void lio_sync_octeon_time(struct work_struct *work)
612 {
613 	struct cavium_wk *wk = (struct cavium_wk *)work;
614 	struct lio *lio = (struct lio *)wk->ctxptr;
615 	struct octeon_device *oct = lio->oct_dev;
616 	struct octeon_soft_command *sc;
617 	struct timespec64 ts;
618 	struct lio_time *lt;
619 	int ret;
620 
621 	sc = octeon_alloc_soft_command(oct, sizeof(struct lio_time), 16, 0);
622 	if (!sc) {
623 		dev_err(&oct->pci_dev->dev,
624 			"Failed to sync time to octeon: soft command allocation failed\n");
625 		return;
626 	}
627 
628 	lt = (struct lio_time *)sc->virtdptr;
629 
630 	/* Get time of the day */
631 	ktime_get_real_ts64(&ts);
632 	lt->sec = ts.tv_sec;
633 	lt->nsec = ts.tv_nsec;
634 	octeon_swap_8B_data((u64 *)lt, (sizeof(struct lio_time)) / 8);
635 
636 	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
637 	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
638 				    OPCODE_NIC_SYNC_OCTEON_TIME, 0, 0, 0);
639 
640 	init_completion(&sc->complete);
641 	sc->sc_status = OCTEON_REQUEST_PENDING;
642 
643 	ret = octeon_send_soft_command(oct, sc);
644 	if (ret == IQ_SEND_FAILED) {
645 		dev_err(&oct->pci_dev->dev,
646 			"Failed to sync time to octeon: failed to send soft command\n");
647 		octeon_free_soft_command(oct, sc);
648 	} else {
649 		WRITE_ONCE(sc->caller_is_done, true);
650 	}
651 
652 	queue_delayed_work(lio->sync_octeon_time_wq.wq,
653 			   &lio->sync_octeon_time_wq.wk.work,
654 			   msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
655 }
656 
657 /**
658  * setup_sync_octeon_time_wq - prepare work to periodically update local time to octeon firmware
659  *
660  * @netdev: network device which should send time update to firmware
661  **/
662 static inline int setup_sync_octeon_time_wq(struct net_device *netdev)
663 {
664 	struct lio *lio = GET_LIO(netdev);
665 	struct octeon_device *oct = lio->oct_dev;
666 
667 	lio->sync_octeon_time_wq.wq =
668 		alloc_workqueue("update-octeon-time", WQ_MEM_RECLAIM, 0);
669 	if (!lio->sync_octeon_time_wq.wq) {
670 		dev_err(&oct->pci_dev->dev, "Unable to create wq to update octeon time\n");
671 		return -1;
672 	}
673 	INIT_DELAYED_WORK(&lio->sync_octeon_time_wq.wk.work,
674 			  lio_sync_octeon_time);
675 	lio->sync_octeon_time_wq.wk.ctxptr = lio;
676 	queue_delayed_work(lio->sync_octeon_time_wq.wq,
677 			   &lio->sync_octeon_time_wq.wk.work,
678 			   msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
679 
680 	return 0;
681 }
682 
683 /**
684  * cleanup_sync_octeon_time_wq - destroy wq
685  *
686  * @netdev: network device which should send time update to firmware
687  *
688  * Stop scheduling and destroy the work created to periodically update local
689  * time to octeon firmware.
690  **/
691 static inline void cleanup_sync_octeon_time_wq(struct net_device *netdev)
692 {
693 	struct lio *lio = GET_LIO(netdev);
694 	struct cavium_wq *time_wq = &lio->sync_octeon_time_wq;
695 
696 	if (time_wq->wq) {
697 		cancel_delayed_work_sync(&time_wq->wk.work);
698 		destroy_workqueue(time_wq->wq);
699 	}
700 }
701 
702 static struct octeon_device *get_other_octeon_device(struct octeon_device *oct)
703 {
704 	struct octeon_device *other_oct;
705 
706 	other_oct = lio_get_device(oct->octeon_id + 1);
707 
708 	if (other_oct && other_oct->pci_dev) {
709 		int oct_busnum, other_oct_busnum;
710 
711 		oct_busnum = oct->pci_dev->bus->number;
712 		other_oct_busnum = other_oct->pci_dev->bus->number;
713 
714 		if (oct_busnum == other_oct_busnum) {
715 			int oct_slot, other_oct_slot;
716 
717 			oct_slot = PCI_SLOT(oct->pci_dev->devfn);
718 			other_oct_slot = PCI_SLOT(other_oct->pci_dev->devfn);
719 
720 			if (oct_slot == other_oct_slot)
721 				return other_oct;
722 		}
723 	}
724 
725 	return NULL;
726 }
727 
728 static void disable_all_vf_links(struct octeon_device *oct)
729 {
730 	struct net_device *netdev;
731 	int max_vfs, vf, i;
732 
733 	if (!oct)
734 		return;
735 
736 	max_vfs = oct->sriov_info.max_vfs;
737 
738 	for (i = 0; i < oct->ifcount; i++) {
739 		netdev = oct->props[i].netdev;
740 		if (!netdev)
741 			continue;
742 
743 		for (vf = 0; vf < max_vfs; vf++)
744 			liquidio_set_vf_link_state(netdev, vf,
745 						   IFLA_VF_LINK_STATE_DISABLE);
746 	}
747 }
748 
749 static int liquidio_watchdog(void *param)
750 {
751 	bool err_msg_was_printed[LIO_MAX_CORES];
752 	u16 mask_of_crashed_or_stuck_cores = 0;
753 	bool all_vf_links_are_disabled = false;
754 	struct octeon_device *oct = param;
755 	struct octeon_device *other_oct;
756 #ifdef CONFIG_MODULE_UNLOAD
757 	long refcount, vfs_referencing_pf;
758 	u64 vfs_mask1, vfs_mask2;
759 #endif
760 	int core;
761 
762 	memset(err_msg_was_printed, 0, sizeof(err_msg_was_printed));
763 
764 	while (!kthread_should_stop()) {
765 		/* sleep for a couple of seconds so that we don't hog the CPU */
766 		set_current_state(TASK_INTERRUPTIBLE);
767 		schedule_timeout(msecs_to_jiffies(2000));
768 
769 		mask_of_crashed_or_stuck_cores =
770 		    (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);
771 
772 		if (!mask_of_crashed_or_stuck_cores)
773 			continue;
774 
775 		WRITE_ONCE(oct->cores_crashed, true);
776 		other_oct = get_other_octeon_device(oct);
777 		if (other_oct)
778 			WRITE_ONCE(other_oct->cores_crashed, true);
779 
780 		for (core = 0; core < LIO_MAX_CORES; core++) {
781 			bool core_crashed_or_got_stuck;
782 
783 			core_crashed_or_got_stuck =
784 						(mask_of_crashed_or_stuck_cores
785 						 >> core) & 1;
786 
787 			if (core_crashed_or_got_stuck &&
788 			    !err_msg_was_printed[core]) {
789 				dev_err(&oct->pci_dev->dev,
790 					"ERROR: Octeon core %d crashed or got stuck!  See oct-fwdump for details.\n",
791 					core);
792 				err_msg_was_printed[core] = true;
793 			}
794 		}
795 
796 		if (all_vf_links_are_disabled)
797 			continue;
798 
799 		disable_all_vf_links(oct);
800 		disable_all_vf_links(other_oct);
801 		all_vf_links_are_disabled = true;
802 
803 #ifdef CONFIG_MODULE_UNLOAD
804 		vfs_mask1 = READ_ONCE(oct->sriov_info.vf_drv_loaded_mask);
805 		vfs_mask2 = READ_ONCE(other_oct->sriov_info.vf_drv_loaded_mask);
806 
807 		vfs_referencing_pf  = hweight64(vfs_mask1);
808 		vfs_referencing_pf += hweight64(vfs_mask2);
809 
810 		refcount = module_refcount(THIS_MODULE);
811 		if (refcount >= vfs_referencing_pf) {
812 			while (vfs_referencing_pf) {
813 				module_put(THIS_MODULE);
814 				vfs_referencing_pf--;
815 			}
816 		}
817 #endif
818 	}
819 
820 	return 0;
821 }
822 
823 /**
824  * liquidio_probe - PCI probe handler
825  * @pdev: PCI device structure
826  * @ent: unused
827  */
828 static int
829 liquidio_probe(struct pci_dev *pdev, const struct pci_device_id __maybe_unused *ent)
830 {
831 	struct octeon_device *oct_dev = NULL;
832 	struct handshake *hs;
833 
834 	oct_dev = octeon_allocate_device(pdev->device,
835 					 sizeof(struct octeon_device_priv));
836 	if (!oct_dev) {
837 		dev_err(&pdev->dev, "Unable to allocate device\n");
838 		return -ENOMEM;
839 	}
840 
841 	if (pdev->device == OCTEON_CN23XX_PF_VID)
842 		oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
843 
844 	/* Enable PTP for 6XXX Device */
845 	if (((pdev->device == OCTEON_CN66XX) ||
846 	     (pdev->device == OCTEON_CN68XX)))
847 		oct_dev->ptp_enable = true;
848 	else
849 		oct_dev->ptp_enable = false;
850 
851 	dev_info(&pdev->dev, "Initializing device %x:%x.\n",
852 		 (u32)pdev->vendor, (u32)pdev->device);
853 
854 	/* Assign octeon_device for this device to the private data area. */
855 	pci_set_drvdata(pdev, oct_dev);
856 
857 	/* set linux specific device pointer */
858 	oct_dev->pci_dev = (void *)pdev;
859 
860 	oct_dev->subsystem_id = pdev->subsystem_vendor |
861 		(pdev->subsystem_device << 16);
862 
863 	hs = &handshake[oct_dev->octeon_id];
864 	init_completion(&hs->init);
865 	init_completion(&hs->started);
866 	hs->pci_dev = pdev;
867 
868 	if (oct_dev->octeon_id == 0)
869 		/* first LiquidIO NIC is detected */
870 		complete(&first_stage);
871 
872 	if (octeon_device_init(oct_dev)) {
873 		complete(&hs->init);
874 		liquidio_remove(pdev);
875 		return -ENOMEM;
876 	}
877 
878 	if (OCTEON_CN23XX_PF(oct_dev)) {
879 		u8 bus, device, function;
880 
881 		if (atomic_read(oct_dev->adapter_refcount) == 1) {
882 			/* Each NIC gets one watchdog kernel thread.  The first
883 			 * PF (of each NIC) that gets pci_driver->probe()'d
884 			 * creates that thread.
885 			 */
886 			bus = pdev->bus->number;
887 			device = PCI_SLOT(pdev->devfn);
888 			function = PCI_FUNC(pdev->devfn);
889 			oct_dev->watchdog_task = kthread_run(liquidio_watchdog,
890 							     oct_dev,
891 							     "liowd/%02hhx:%02hhx.%hhx",
892 							     bus, device, function);
893 			if (IS_ERR(oct_dev->watchdog_task)) {
894 				oct_dev->watchdog_task = NULL;
895 				dev_err(&oct_dev->pci_dev->dev,
896 					"failed to create kernel_thread\n");
897 				liquidio_remove(pdev);
898 				return -1;
899 			}
900 		}
901 	}
902 
903 	oct_dev->rx_pause = 1;
904 	oct_dev->tx_pause = 1;
905 
906 	dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
907 
908 	return 0;
909 }
910 
911 static bool fw_type_is_auto(void)
912 {
913 	return strncmp(fw_type, LIO_FW_NAME_TYPE_AUTO,
914 		       sizeof(LIO_FW_NAME_TYPE_AUTO)) == 0;
915 }
916 
917 /**
918  * octeon_pci_flr - PCI FLR for each Octeon device.
919  * @oct: octeon device
920  */
921 static void octeon_pci_flr(struct octeon_device *oct)
922 {
923 	int rc;
924 
925 	pci_save_state(oct->pci_dev);
926 
927 	pci_cfg_access_lock(oct->pci_dev);
928 
929 	/* Quiesce the device completely */
930 	pci_write_config_word(oct->pci_dev, PCI_COMMAND,
931 			      PCI_COMMAND_INTX_DISABLE);
932 
933 	rc = __pci_reset_function_locked(oct->pci_dev);
934 
935 	if (rc != 0)
936 		dev_err(&oct->pci_dev->dev, "Error %d resetting PCI function %d\n",
937 			rc, oct->pf_num);
938 
939 	pci_cfg_access_unlock(oct->pci_dev);
940 
941 	pci_restore_state(oct->pci_dev);
942 }
943 
944 /**
945  * octeon_destroy_resources - Destroy resources associated with octeon device
946  * @oct: octeon device
947  */
948 static void octeon_destroy_resources(struct octeon_device *oct)
949 {
950 	int i, refcount;
951 	struct msix_entry *msix_entries;
952 	struct octeon_device_priv *oct_priv = oct->priv;
953 
954 	struct handshake *hs;
955 
956 	switch (atomic_read(&oct->status)) {
957 	case OCT_DEV_RUNNING:
958 	case OCT_DEV_CORE_OK:
959 
960 		/* No more instructions will be forwarded. */
961 		atomic_set(&oct->status, OCT_DEV_IN_RESET);
962 
963 		oct->app_mode = CVM_DRV_INVALID_APP;
964 		dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
965 			lio_get_state_string(&oct->status));
966 
967 		schedule_timeout_uninterruptible(HZ / 10);
968 
969 		fallthrough;
970 	case OCT_DEV_HOST_OK:
971 
972 	case OCT_DEV_CONSOLE_INIT_DONE:
973 		/* Remove any consoles */
974 		octeon_remove_consoles(oct);
975 
976 		fallthrough;
977 	case OCT_DEV_IO_QUEUES_DONE:
978 		if (lio_wait_for_instr_fetch(oct))
979 			dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
980 
981 		if (wait_for_pending_requests(oct))
982 			dev_err(&oct->pci_dev->dev, "There were pending requests\n");
983 
984 		/* Disable the input and output queues now. No more packets will
985 		 * arrive from Octeon, but we should wait for all packet
986 		 * processing to finish.
987 		 */
988 		oct->fn_list.disable_io_queues(oct);
989 
990 		if (lio_wait_for_oq_pkts(oct))
991 			dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
992 
993 		/* Force all requests waiting to be fetched by OCTEON to
994 		 * complete.
995 		 */
996 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
997 			struct octeon_instr_queue *iq;
998 
999 			if (!(oct->io_qmask.iq & BIT_ULL(i)))
1000 				continue;
1001 			iq = oct->instr_queue[i];
1002 
1003 			if (atomic_read(&iq->instr_pending)) {
1004 				spin_lock_bh(&iq->lock);
1005 				iq->fill_cnt = 0;
1006 				iq->octeon_read_index = iq->host_write_index;
1007 				iq->stats.instr_processed +=
1008 					atomic_read(&iq->instr_pending);
1009 				lio_process_iq_request_list(oct, iq, 0);
1010 				spin_unlock_bh(&iq->lock);
1011 			}
1012 		}
1013 
1014 		lio_process_ordered_list(oct, 1);
1015 		octeon_free_sc_done_list(oct);
1016 		octeon_free_sc_zombie_list(oct);
1017 
1018 		fallthrough;
1019 	case OCT_DEV_INTR_SET_DONE:
1020 		/* Disable interrupts  */
1021 		oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
1022 
1023 		if (oct->msix_on) {
1024 			msix_entries = (struct msix_entry *)oct->msix_entries;
1025 			for (i = 0; i < oct->num_msix_irqs - 1; i++) {
1026 				if (oct->ioq_vector[i].vector) {
1027 					/* clear the affinity_cpumask */
1028 					irq_set_affinity_hint(
1029 							msix_entries[i].vector,
1030 							NULL);
1031 					free_irq(msix_entries[i].vector,
1032 						 &oct->ioq_vector[i]);
1033 					oct->ioq_vector[i].vector = 0;
1034 				}
1035 			}
1036 			/* non-iov vector's argument is oct struct */
1037 			free_irq(msix_entries[i].vector, oct);
1038 
1039 			pci_disable_msix(oct->pci_dev);
1040 			kfree(oct->msix_entries);
1041 			oct->msix_entries = NULL;
1042 		} else {
1043 			/* Release the interrupt line */
1044 			free_irq(oct->pci_dev->irq, oct);
1045 
1046 			if (oct->flags & LIO_FLAG_MSI_ENABLED)
1047 				pci_disable_msi(oct->pci_dev);
1048 		}
1049 
1050 		kfree(oct->irq_name_storage);
1051 		oct->irq_name_storage = NULL;
1052 
1053 		fallthrough;
1054 	case OCT_DEV_MSIX_ALLOC_VECTOR_DONE:
1055 		if (OCTEON_CN23XX_PF(oct))
1056 			octeon_free_ioq_vector(oct);
1057 
1058 		fallthrough;
1059 	case OCT_DEV_MBOX_SETUP_DONE:
1060 		if (OCTEON_CN23XX_PF(oct))
1061 			oct->fn_list.free_mbox(oct);
1062 
1063 		fallthrough;
1064 	case OCT_DEV_IN_RESET:
1065 	case OCT_DEV_DROQ_INIT_DONE:
1066 		/* Wait for any pending operations */
1067 		mdelay(100);
1068 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
1069 			if (!(oct->io_qmask.oq & BIT_ULL(i)))
1070 				continue;
1071 			octeon_delete_droq(oct, i);
1072 		}
1073 
1074 		/* Force any pending handshakes to complete */
1075 		for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1076 			hs = &handshake[i];
1077 
1078 			if (hs->pci_dev) {
1079 				handshake[oct->octeon_id].init_ok = 0;
1080 				complete(&handshake[oct->octeon_id].init);
1081 				handshake[oct->octeon_id].started_ok = 0;
1082 				complete(&handshake[oct->octeon_id].started);
1083 			}
1084 		}
1085 
1086 		fallthrough;
1087 	case OCT_DEV_RESP_LIST_INIT_DONE:
1088 		octeon_delete_response_list(oct);
1089 
1090 		fallthrough;
1091 	case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1092 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1093 			if (!(oct->io_qmask.iq & BIT_ULL(i)))
1094 				continue;
1095 			octeon_delete_instr_queue(oct, i);
1096 		}
1097 #ifdef CONFIG_PCI_IOV
1098 		if (oct->sriov_info.sriov_enabled)
1099 			pci_disable_sriov(oct->pci_dev);
1100 #endif
1101 		fallthrough;
1102 	case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1103 		octeon_free_sc_buffer_pool(oct);
1104 
1105 		fallthrough;
1106 	case OCT_DEV_DISPATCH_INIT_DONE:
1107 		octeon_delete_dispatch_list(oct);
1108 		cancel_delayed_work_sync(&oct->nic_poll_work.work);
1109 
1110 		fallthrough;
1111 	case OCT_DEV_PCI_MAP_DONE:
1112 		refcount = octeon_deregister_device(oct);
1113 
1114 		/* Soft reset the octeon device before exiting.
1115 		 * However, if fw was loaded from card (i.e. autoboot),
1116 		 * perform an FLR instead.
1117 		 * Implementation note: only soft-reset the device
1118 		 * if it is a CN6XXX OR the LAST CN23XX device.
1119 		 */
1120 		if (atomic_read(oct->adapter_fw_state) == FW_IS_PRELOADED)
1121 			octeon_pci_flr(oct);
1122 		else if (OCTEON_CN6XXX(oct) || !refcount)
1123 			oct->fn_list.soft_reset(oct);
1124 
1125 		octeon_unmap_pci_barx(oct, 0);
1126 		octeon_unmap_pci_barx(oct, 1);
1127 
1128 		fallthrough;
1129 	case OCT_DEV_PCI_ENABLE_DONE:
1130 		/* Disable the device, releasing the PCI INT */
1131 		pci_disable_device(oct->pci_dev);
1132 
1133 		fallthrough;
1134 	case OCT_DEV_BEGIN_STATE:
1135 		/* Nothing to be done here either */
1136 		break;
1137 	}                       /* end switch (oct->status) */
1138 
1139 	tasklet_kill(&oct_priv->droq_tasklet);
1140 }
1141 
1142 /**
1143  * send_rx_ctrl_cmd - Send Rx control command
1144  * @lio: per-network private data
1145  * @start_stop: whether to start or stop
1146  */
1147 static int send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1148 {
1149 	struct octeon_soft_command *sc;
1150 	union octnet_cmd *ncmd;
1151 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1152 	int retval;
1153 
1154 	if (oct->props[lio->ifidx].rx_on == start_stop)
1155 		return 0;
1156 
1157 	sc = (struct octeon_soft_command *)
1158 		octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
1159 					  16, 0);
1160 	if (!sc) {
1161 		netif_info(lio, rx_err, lio->netdev,
1162 			   "Failed to allocate octeon_soft_command struct\n");
1163 		return -ENOMEM;
1164 	}
1165 
1166 	ncmd = (union octnet_cmd *)sc->virtdptr;
1167 
1168 	ncmd->u64 = 0;
1169 	ncmd->s.cmd = OCTNET_CMD_RX_CTL;
1170 	ncmd->s.param1 = start_stop;
1171 
1172 	octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
1173 
1174 	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1175 
1176 	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
1177 				    OPCODE_NIC_CMD, 0, 0, 0);
1178 
1179 	init_completion(&sc->complete);
1180 	sc->sc_status = OCTEON_REQUEST_PENDING;
1181 
1182 	retval = octeon_send_soft_command(oct, sc);
1183 	if (retval == IQ_SEND_FAILED) {
1184 		netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1185 		octeon_free_soft_command(oct, sc);
1186 	} else {
1187 		/* Sleep on a wait queue till the cond flag indicates that the
1188 		 * response arrived or timed-out.
1189 		 */
1190 		retval = wait_for_sc_completion_timeout(oct, sc, 0);
1191 		if (retval)
1192 			return retval;
1193 
1194 		oct->props[lio->ifidx].rx_on = start_stop;
1195 		WRITE_ONCE(sc->caller_is_done, true);
1196 	}
1197 
1198 	return retval;
1199 }
1200 
1201 /**
1202  * liquidio_destroy_nic_device - Destroy NIC device interface
1203  * @oct: octeon device
1204  * @ifidx: which interface to destroy
1205  *
1206  * Cleanup associated with each interface for an Octeon device  when NIC
1207  * module is being unloaded or if initialization fails during load.
1208  */
1209 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1210 {
1211 	struct net_device *netdev = oct->props[ifidx].netdev;
1212 	struct octeon_device_priv *oct_priv = oct->priv;
1213 	struct napi_struct *napi, *n;
1214 	struct lio *lio;
1215 
1216 	if (!netdev) {
1217 		dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1218 			__func__, ifidx);
1219 		return;
1220 	}
1221 
1222 	lio = GET_LIO(netdev);
1223 
1224 	dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1225 
1226 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1227 		liquidio_stop(netdev);
1228 
1229 	if (oct->props[lio->ifidx].napi_enabled == 1) {
1230 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1231 			napi_disable(napi);
1232 
1233 		oct->props[lio->ifidx].napi_enabled = 0;
1234 
1235 		if (OCTEON_CN23XX_PF(oct))
1236 			oct->droq[0]->ops.poll_mode = 0;
1237 	}
1238 
1239 	/* Delete NAPI */
1240 	list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1241 		netif_napi_del(napi);
1242 
1243 	tasklet_enable(&oct_priv->droq_tasklet);
1244 
1245 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1246 		unregister_netdev(netdev);
1247 
1248 	cleanup_sync_octeon_time_wq(netdev);
1249 	cleanup_link_status_change_wq(netdev);
1250 
1251 	cleanup_rx_oom_poll_fn(netdev);
1252 
1253 	lio_delete_glists(lio);
1254 
1255 	free_netdev(netdev);
1256 
1257 	oct->props[ifidx].gmxport = -1;
1258 
1259 	oct->props[ifidx].netdev = NULL;
1260 }
1261 
1262 /**
1263  * liquidio_stop_nic_module - Stop complete NIC functionality
1264  * @oct: octeon device
1265  */
1266 static int liquidio_stop_nic_module(struct octeon_device *oct)
1267 {
1268 	int i, j;
1269 	struct lio *lio;
1270 
1271 	dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1272 	device_lock(&oct->pci_dev->dev);
1273 	if (oct->devlink) {
1274 		devlink_unregister(oct->devlink);
1275 		devlink_free(oct->devlink);
1276 		oct->devlink = NULL;
1277 	}
1278 	device_unlock(&oct->pci_dev->dev);
1279 
1280 	if (!oct->ifcount) {
1281 		dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1282 		return 1;
1283 	}
1284 
1285 	spin_lock_bh(&oct->cmd_resp_wqlock);
1286 	oct->cmd_resp_state = OCT_DRV_OFFLINE;
1287 	spin_unlock_bh(&oct->cmd_resp_wqlock);
1288 
1289 	lio_vf_rep_destroy(oct);
1290 
1291 	for (i = 0; i < oct->ifcount; i++) {
1292 		lio = GET_LIO(oct->props[i].netdev);
1293 		for (j = 0; j < oct->num_oqs; j++)
1294 			octeon_unregister_droq_ops(oct,
1295 						   lio->linfo.rxpciq[j].s.q_no);
1296 	}
1297 
1298 	for (i = 0; i < oct->ifcount; i++)
1299 		liquidio_destroy_nic_device(oct, i);
1300 
1301 	dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1302 	return 0;
1303 }
1304 
1305 /**
1306  * liquidio_remove - Cleans up resources at unload time
1307  * @pdev: PCI device structure
1308  */
1309 static void liquidio_remove(struct pci_dev *pdev)
1310 {
1311 	struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1312 
1313 	dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1314 
1315 	if (oct_dev->watchdog_task)
1316 		kthread_stop(oct_dev->watchdog_task);
1317 
1318 	if (!oct_dev->octeon_id &&
1319 	    oct_dev->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP)
1320 		lio_vf_rep_modexit();
1321 
1322 	if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1323 		liquidio_stop_nic_module(oct_dev);
1324 
1325 	/* Reset the octeon device and cleanup all memory allocated for
1326 	 * the octeon device by driver.
1327 	 */
1328 	octeon_destroy_resources(oct_dev);
1329 
1330 	dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1331 
1332 	/* This octeon device has been removed. Update the global
1333 	 * data structure to reflect this. Free the device structure.
1334 	 */
1335 	octeon_free_device_mem(oct_dev);
1336 }
1337 
1338 /**
1339  * octeon_chip_specific_setup - Identify the Octeon device and to map the BAR address space
1340  * @oct: octeon device
1341  */
1342 static int octeon_chip_specific_setup(struct octeon_device *oct)
1343 {
1344 	u32 dev_id, rev_id;
1345 	int ret = 1;
1346 
1347 	pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1348 	pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1349 	oct->rev_id = rev_id & 0xff;
1350 
1351 	switch (dev_id) {
1352 	case OCTEON_CN68XX_PCIID:
1353 		oct->chip_id = OCTEON_CN68XX;
1354 		ret = lio_setup_cn68xx_octeon_device(oct);
1355 		break;
1356 
1357 	case OCTEON_CN66XX_PCIID:
1358 		oct->chip_id = OCTEON_CN66XX;
1359 		ret = lio_setup_cn66xx_octeon_device(oct);
1360 		break;
1361 
1362 	case OCTEON_CN23XX_PCIID_PF:
1363 		oct->chip_id = OCTEON_CN23XX_PF_VID;
1364 		ret = setup_cn23xx_octeon_pf_device(oct);
1365 		if (ret)
1366 			break;
1367 #ifdef CONFIG_PCI_IOV
1368 		if (!ret)
1369 			pci_sriov_set_totalvfs(oct->pci_dev,
1370 					       oct->sriov_info.max_vfs);
1371 #endif
1372 		break;
1373 
1374 	default:
1375 		dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1376 			dev_id);
1377 	}
1378 
1379 	return ret;
1380 }
1381 
1382 /**
1383  * octeon_pci_os_setup - PCI initialization for each Octeon device.
1384  * @oct: octeon device
1385  */
1386 static int octeon_pci_os_setup(struct octeon_device *oct)
1387 {
1388 	/* setup PCI stuff first */
1389 	if (pci_enable_device(oct->pci_dev)) {
1390 		dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1391 		return 1;
1392 	}
1393 
1394 	if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1395 		dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1396 		pci_disable_device(oct->pci_dev);
1397 		return 1;
1398 	}
1399 
1400 	/* Enable PCI DMA Master. */
1401 	pci_set_master(oct->pci_dev);
1402 
1403 	return 0;
1404 }
1405 
1406 /**
1407  * free_netbuf - Unmap and free network buffer
1408  * @buf: buffer
1409  */
1410 static void free_netbuf(void *buf)
1411 {
1412 	struct sk_buff *skb;
1413 	struct octnet_buf_free_info *finfo;
1414 	struct lio *lio;
1415 
1416 	finfo = (struct octnet_buf_free_info *)buf;
1417 	skb = finfo->skb;
1418 	lio = finfo->lio;
1419 
1420 	dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1421 			 DMA_TO_DEVICE);
1422 
1423 	tx_buffer_free(skb);
1424 }
1425 
1426 /**
1427  * free_netsgbuf - Unmap and free gather buffer
1428  * @buf: buffer
1429  */
1430 static void free_netsgbuf(void *buf)
1431 {
1432 	struct octnet_buf_free_info *finfo;
1433 	struct sk_buff *skb;
1434 	struct lio *lio;
1435 	struct octnic_gather *g;
1436 	int i, frags, iq;
1437 
1438 	finfo = (struct octnet_buf_free_info *)buf;
1439 	skb = finfo->skb;
1440 	lio = finfo->lio;
1441 	g = finfo->g;
1442 	frags = skb_shinfo(skb)->nr_frags;
1443 
1444 	dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1445 			 g->sg[0].ptr[0], (skb->len - skb->data_len),
1446 			 DMA_TO_DEVICE);
1447 
1448 	i = 1;
1449 	while (frags--) {
1450 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
1451 
1452 		dma_unmap_page(&lio->oct_dev->pci_dev->dev,
1453 			       g->sg[(i >> 2)].ptr[(i & 3)],
1454 			       skb_frag_size(frag), DMA_TO_DEVICE);
1455 		i++;
1456 	}
1457 
1458 	iq = skb_iq(lio->oct_dev, skb);
1459 	spin_lock(&lio->glist_lock[iq]);
1460 	list_add_tail(&g->list, &lio->glist[iq]);
1461 	spin_unlock(&lio->glist_lock[iq]);
1462 
1463 	tx_buffer_free(skb);
1464 }
1465 
1466 /**
1467  * free_netsgbuf_with_resp - Unmap and free gather buffer with response
1468  * @buf: buffer
1469  */
1470 static void free_netsgbuf_with_resp(void *buf)
1471 {
1472 	struct octeon_soft_command *sc;
1473 	struct octnet_buf_free_info *finfo;
1474 	struct sk_buff *skb;
1475 	struct lio *lio;
1476 	struct octnic_gather *g;
1477 	int i, frags, iq;
1478 
1479 	sc = (struct octeon_soft_command *)buf;
1480 	skb = (struct sk_buff *)sc->callback_arg;
1481 	finfo = (struct octnet_buf_free_info *)&skb->cb;
1482 
1483 	lio = finfo->lio;
1484 	g = finfo->g;
1485 	frags = skb_shinfo(skb)->nr_frags;
1486 
1487 	dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1488 			 g->sg[0].ptr[0], (skb->len - skb->data_len),
1489 			 DMA_TO_DEVICE);
1490 
1491 	i = 1;
1492 	while (frags--) {
1493 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
1494 
1495 		dma_unmap_page(&lio->oct_dev->pci_dev->dev,
1496 			       g->sg[(i >> 2)].ptr[(i & 3)],
1497 			       skb_frag_size(frag), DMA_TO_DEVICE);
1498 		i++;
1499 	}
1500 
1501 	iq = skb_iq(lio->oct_dev, skb);
1502 
1503 	spin_lock(&lio->glist_lock[iq]);
1504 	list_add_tail(&g->list, &lio->glist[iq]);
1505 	spin_unlock(&lio->glist_lock[iq]);
1506 
1507 	/* Don't free the skb yet */
1508 }
1509 
1510 /**
1511  * liquidio_ptp_adjfine - Adjust ptp frequency
1512  * @ptp: PTP clock info
1513  * @scaled_ppm: how much to adjust by, in scaled parts-per-million
1514  *
1515  * Scaled parts per million is ppm with a 16-bit binary fractional field.
1516  */
1517 static int liquidio_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
1518 {
1519 	struct lio *lio = container_of(ptp, struct lio, ptp_info);
1520 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1521 	s32 ppb = scaled_ppm_to_ppb(scaled_ppm);
1522 	u64 comp, delta;
1523 	unsigned long flags;
1524 	bool neg_adj = false;
1525 
1526 	if (ppb < 0) {
1527 		neg_adj = true;
1528 		ppb = -ppb;
1529 	}
1530 
1531 	/* The hardware adds the clock compensation value to the
1532 	 * PTP clock on every coprocessor clock cycle, so we
1533 	 * compute the delta in terms of coprocessor clocks.
1534 	 */
1535 	delta = (u64)ppb << 32;
1536 	do_div(delta, oct->coproc_clock_rate);
1537 
1538 	spin_lock_irqsave(&lio->ptp_lock, flags);
1539 	comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1540 	if (neg_adj)
1541 		comp -= delta;
1542 	else
1543 		comp += delta;
1544 	lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1545 	spin_unlock_irqrestore(&lio->ptp_lock, flags);
1546 
1547 	return 0;
1548 }
1549 
1550 /**
1551  * liquidio_ptp_adjtime - Adjust ptp time
1552  * @ptp: PTP clock info
1553  * @delta: how much to adjust by, in nanosecs
1554  */
1555 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1556 {
1557 	unsigned long flags;
1558 	struct lio *lio = container_of(ptp, struct lio, ptp_info);
1559 
1560 	spin_lock_irqsave(&lio->ptp_lock, flags);
1561 	lio->ptp_adjust += delta;
1562 	spin_unlock_irqrestore(&lio->ptp_lock, flags);
1563 
1564 	return 0;
1565 }
1566 
1567 /**
1568  * liquidio_ptp_gettime - Get hardware clock time, including any adjustment
1569  * @ptp: PTP clock info
1570  * @ts: timespec
1571  */
1572 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1573 				struct timespec64 *ts)
1574 {
1575 	u64 ns;
1576 	unsigned long flags;
1577 	struct lio *lio = container_of(ptp, struct lio, ptp_info);
1578 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1579 
1580 	spin_lock_irqsave(&lio->ptp_lock, flags);
1581 	ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1582 	ns += lio->ptp_adjust;
1583 	spin_unlock_irqrestore(&lio->ptp_lock, flags);
1584 
1585 	*ts = ns_to_timespec64(ns);
1586 
1587 	return 0;
1588 }
1589 
1590 /**
1591  * liquidio_ptp_settime - Set hardware clock time. Reset adjustment
1592  * @ptp: PTP clock info
1593  * @ts: timespec
1594  */
1595 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
1596 				const struct timespec64 *ts)
1597 {
1598 	u64 ns;
1599 	unsigned long flags;
1600 	struct lio *lio = container_of(ptp, struct lio, ptp_info);
1601 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1602 
1603 	ns = timespec64_to_ns(ts);
1604 
1605 	spin_lock_irqsave(&lio->ptp_lock, flags);
1606 	lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
1607 	lio->ptp_adjust = 0;
1608 	spin_unlock_irqrestore(&lio->ptp_lock, flags);
1609 
1610 	return 0;
1611 }
1612 
1613 /**
1614  * liquidio_ptp_enable - Check if PTP is enabled
1615  * @ptp: PTP clock info
1616  * @rq: request
1617  * @on: is it on
1618  */
1619 static int
1620 liquidio_ptp_enable(struct ptp_clock_info __maybe_unused *ptp,
1621 		    struct ptp_clock_request __maybe_unused *rq,
1622 		    int __maybe_unused on)
1623 {
1624 	return -EOPNOTSUPP;
1625 }
1626 
1627 /**
1628  * oct_ptp_open - Open PTP clock source
1629  * @netdev: network device
1630  */
1631 static void oct_ptp_open(struct net_device *netdev)
1632 {
1633 	struct lio *lio = GET_LIO(netdev);
1634 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1635 
1636 	spin_lock_init(&lio->ptp_lock);
1637 
1638 	snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
1639 	lio->ptp_info.owner = THIS_MODULE;
1640 	lio->ptp_info.max_adj = 250000000;
1641 	lio->ptp_info.n_alarm = 0;
1642 	lio->ptp_info.n_ext_ts = 0;
1643 	lio->ptp_info.n_per_out = 0;
1644 	lio->ptp_info.pps = 0;
1645 	lio->ptp_info.adjfine = liquidio_ptp_adjfine;
1646 	lio->ptp_info.adjtime = liquidio_ptp_adjtime;
1647 	lio->ptp_info.gettime64 = liquidio_ptp_gettime;
1648 	lio->ptp_info.settime64 = liquidio_ptp_settime;
1649 	lio->ptp_info.enable = liquidio_ptp_enable;
1650 
1651 	lio->ptp_adjust = 0;
1652 
1653 	lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
1654 					     &oct->pci_dev->dev);
1655 
1656 	if (IS_ERR(lio->ptp_clock))
1657 		lio->ptp_clock = NULL;
1658 }
1659 
1660 /**
1661  * liquidio_ptp_init - Init PTP clock
1662  * @oct: octeon device
1663  */
1664 static void liquidio_ptp_init(struct octeon_device *oct)
1665 {
1666 	u64 clock_comp, cfg;
1667 
1668 	clock_comp = (u64)NSEC_PER_SEC << 32;
1669 	do_div(clock_comp, oct->coproc_clock_rate);
1670 	lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1671 
1672 	/* Enable */
1673 	cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
1674 	lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
1675 }
1676 
1677 /**
1678  * load_firmware - Load firmware to device
1679  * @oct: octeon device
1680  *
1681  * Maps device to firmware filename, requests firmware, and downloads it
1682  */
1683 static int load_firmware(struct octeon_device *oct)
1684 {
1685 	int ret = 0;
1686 	const struct firmware *fw;
1687 	char fw_name[LIO_MAX_FW_FILENAME_LEN];
1688 	char *tmp_fw_type;
1689 
1690 	if (fw_type_is_auto()) {
1691 		tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
1692 		strncpy(fw_type, tmp_fw_type, sizeof(fw_type));
1693 	} else {
1694 		tmp_fw_type = fw_type;
1695 	}
1696 
1697 	sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,
1698 		octeon_get_conf(oct)->card_name, tmp_fw_type,
1699 		LIO_FW_NAME_SUFFIX);
1700 
1701 	ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);
1702 	if (ret) {
1703 		dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n",
1704 			fw_name);
1705 		release_firmware(fw);
1706 		return ret;
1707 	}
1708 
1709 	ret = octeon_download_firmware(oct, fw->data, fw->size);
1710 
1711 	release_firmware(fw);
1712 
1713 	return ret;
1714 }
1715 
1716 /**
1717  * octnet_poll_check_txq_status - Poll routine for checking transmit queue status
1718  * @work: work_struct data structure
1719  */
1720 static void octnet_poll_check_txq_status(struct work_struct *work)
1721 {
1722 	struct cavium_wk *wk = (struct cavium_wk *)work;
1723 	struct lio *lio = (struct lio *)wk->ctxptr;
1724 
1725 	if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
1726 		return;
1727 
1728 	check_txq_status(lio);
1729 	queue_delayed_work(lio->txq_status_wq.wq,
1730 			   &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
1731 }
1732 
1733 /**
1734  * setup_tx_poll_fn - Sets up the txq poll check
1735  * @netdev: network device
1736  */
1737 static inline int setup_tx_poll_fn(struct net_device *netdev)
1738 {
1739 	struct lio *lio = GET_LIO(netdev);
1740 	struct octeon_device *oct = lio->oct_dev;
1741 
1742 	lio->txq_status_wq.wq = alloc_workqueue("txq-status",
1743 						WQ_MEM_RECLAIM, 0);
1744 	if (!lio->txq_status_wq.wq) {
1745 		dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
1746 		return -1;
1747 	}
1748 	INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
1749 			  octnet_poll_check_txq_status);
1750 	lio->txq_status_wq.wk.ctxptr = lio;
1751 	queue_delayed_work(lio->txq_status_wq.wq,
1752 			   &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
1753 	return 0;
1754 }
1755 
1756 static inline void cleanup_tx_poll_fn(struct net_device *netdev)
1757 {
1758 	struct lio *lio = GET_LIO(netdev);
1759 
1760 	if (lio->txq_status_wq.wq) {
1761 		cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
1762 		destroy_workqueue(lio->txq_status_wq.wq);
1763 	}
1764 }
1765 
1766 /**
1767  * liquidio_open - Net device open for LiquidIO
1768  * @netdev: network device
1769  */
1770 static int liquidio_open(struct net_device *netdev)
1771 {
1772 	struct lio *lio = GET_LIO(netdev);
1773 	struct octeon_device *oct = lio->oct_dev;
1774 	struct octeon_device_priv *oct_priv = oct->priv;
1775 	struct napi_struct *napi, *n;
1776 	int ret = 0;
1777 
1778 	if (oct->props[lio->ifidx].napi_enabled == 0) {
1779 		tasklet_disable(&oct_priv->droq_tasklet);
1780 
1781 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1782 			napi_enable(napi);
1783 
1784 		oct->props[lio->ifidx].napi_enabled = 1;
1785 
1786 		if (OCTEON_CN23XX_PF(oct))
1787 			oct->droq[0]->ops.poll_mode = 1;
1788 	}
1789 
1790 	if (oct->ptp_enable)
1791 		oct_ptp_open(netdev);
1792 
1793 	ifstate_set(lio, LIO_IFSTATE_RUNNING);
1794 
1795 	if (!OCTEON_CN23XX_PF(oct) || !oct->msix_on) {
1796 		ret = setup_tx_poll_fn(netdev);
1797 		if (ret)
1798 			goto err_poll;
1799 	}
1800 
1801 	netif_tx_start_all_queues(netdev);
1802 
1803 	/* Ready for link status updates */
1804 	lio->intf_open = 1;
1805 
1806 	netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
1807 
1808 	/* tell Octeon to start forwarding packets to host */
1809 	ret = send_rx_ctrl_cmd(lio, 1);
1810 	if (ret)
1811 		goto err_rx_ctrl;
1812 
1813 	/* start periodical statistics fetch */
1814 	INIT_DELAYED_WORK(&lio->stats_wk.work, lio_fetch_stats);
1815 	lio->stats_wk.ctxptr = lio;
1816 	schedule_delayed_work(&lio->stats_wk.work, msecs_to_jiffies
1817 					(LIQUIDIO_NDEV_STATS_POLL_TIME_MS));
1818 
1819 	dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
1820 		 netdev->name);
1821 
1822 	return 0;
1823 
1824 err_rx_ctrl:
1825 	if (!OCTEON_CN23XX_PF(oct) || !oct->msix_on)
1826 		cleanup_tx_poll_fn(netdev);
1827 err_poll:
1828 	if (lio->ptp_clock) {
1829 		ptp_clock_unregister(lio->ptp_clock);
1830 		lio->ptp_clock = NULL;
1831 	}
1832 
1833 	if (oct->props[lio->ifidx].napi_enabled == 1) {
1834 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1835 			napi_disable(napi);
1836 
1837 		oct->props[lio->ifidx].napi_enabled = 0;
1838 
1839 		if (OCTEON_CN23XX_PF(oct))
1840 			oct->droq[0]->ops.poll_mode = 0;
1841 	}
1842 
1843 	return ret;
1844 }
1845 
1846 /**
1847  * liquidio_stop - Net device stop for LiquidIO
1848  * @netdev: network device
1849  */
1850 static int liquidio_stop(struct net_device *netdev)
1851 {
1852 	struct lio *lio = GET_LIO(netdev);
1853 	struct octeon_device *oct = lio->oct_dev;
1854 	struct octeon_device_priv *oct_priv = oct->priv;
1855 	struct napi_struct *napi, *n;
1856 	int ret = 0;
1857 
1858 	ifstate_reset(lio, LIO_IFSTATE_RUNNING);
1859 
1860 	/* Stop any link updates */
1861 	lio->intf_open = 0;
1862 
1863 	stop_txqs(netdev);
1864 
1865 	/* Inform that netif carrier is down */
1866 	netif_carrier_off(netdev);
1867 	netif_tx_disable(netdev);
1868 
1869 	lio->linfo.link.s.link_up = 0;
1870 	lio->link_changes++;
1871 
1872 	/* Tell Octeon that nic interface is down. */
1873 	ret = send_rx_ctrl_cmd(lio, 0);
1874 	if (ret)
1875 		return ret;
1876 
1877 	if (OCTEON_CN23XX_PF(oct)) {
1878 		if (!oct->msix_on)
1879 			cleanup_tx_poll_fn(netdev);
1880 	} else {
1881 		cleanup_tx_poll_fn(netdev);
1882 	}
1883 
1884 	cancel_delayed_work_sync(&lio->stats_wk.work);
1885 
1886 	if (lio->ptp_clock) {
1887 		ptp_clock_unregister(lio->ptp_clock);
1888 		lio->ptp_clock = NULL;
1889 	}
1890 
1891 	/* Wait for any pending Rx descriptors */
1892 	if (lio_wait_for_clean_oq(oct))
1893 		netif_info(lio, rx_err, lio->netdev,
1894 			   "Proceeding with stop interface after partial RX desc processing\n");
1895 
1896 	if (oct->props[lio->ifidx].napi_enabled == 1) {
1897 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1898 			napi_disable(napi);
1899 
1900 		oct->props[lio->ifidx].napi_enabled = 0;
1901 
1902 		if (OCTEON_CN23XX_PF(oct))
1903 			oct->droq[0]->ops.poll_mode = 0;
1904 
1905 		tasklet_enable(&oct_priv->droq_tasklet);
1906 	}
1907 
1908 	dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
1909 
1910 	return ret;
1911 }
1912 
1913 /**
1914  * get_new_flags - Converts a mask based on net device flags
1915  * @netdev: network device
1916  *
1917  * This routine generates a octnet_ifflags mask from the net device flags
1918  * received from the OS.
1919  */
1920 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
1921 {
1922 	enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
1923 
1924 	if (netdev->flags & IFF_PROMISC)
1925 		f |= OCTNET_IFFLAG_PROMISC;
1926 
1927 	if (netdev->flags & IFF_ALLMULTI)
1928 		f |= OCTNET_IFFLAG_ALLMULTI;
1929 
1930 	if (netdev->flags & IFF_MULTICAST) {
1931 		f |= OCTNET_IFFLAG_MULTICAST;
1932 
1933 		/* Accept all multicast addresses if there are more than we
1934 		 * can handle
1935 		 */
1936 		if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
1937 			f |= OCTNET_IFFLAG_ALLMULTI;
1938 	}
1939 
1940 	if (netdev->flags & IFF_BROADCAST)
1941 		f |= OCTNET_IFFLAG_BROADCAST;
1942 
1943 	return f;
1944 }
1945 
1946 /**
1947  * liquidio_set_mcast_list - Net device set_multicast_list
1948  * @netdev: network device
1949  */
1950 static void liquidio_set_mcast_list(struct net_device *netdev)
1951 {
1952 	struct lio *lio = GET_LIO(netdev);
1953 	struct octeon_device *oct = lio->oct_dev;
1954 	struct octnic_ctrl_pkt nctrl;
1955 	struct netdev_hw_addr *ha;
1956 	u64 *mc;
1957 	int ret;
1958 	int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
1959 
1960 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1961 
1962 	/* Create a ctrl pkt command to be sent to core app. */
1963 	nctrl.ncmd.u64 = 0;
1964 	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
1965 	nctrl.ncmd.s.param1 = get_new_flags(netdev);
1966 	nctrl.ncmd.s.param2 = mc_count;
1967 	nctrl.ncmd.s.more = mc_count;
1968 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1969 	nctrl.netpndev = (u64)netdev;
1970 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1971 
1972 	/* copy all the addresses into the udd */
1973 	mc = &nctrl.udd[0];
1974 	netdev_for_each_mc_addr(ha, netdev) {
1975 		*mc = 0;
1976 		memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
1977 		/* no need to swap bytes */
1978 
1979 		if (++mc > &nctrl.udd[mc_count])
1980 			break;
1981 	}
1982 
1983 	/* Apparently, any activity in this call from the kernel has to
1984 	 * be atomic. So we won't wait for response.
1985 	 */
1986 
1987 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1988 	if (ret) {
1989 		dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
1990 			ret);
1991 	}
1992 }
1993 
1994 /**
1995  * liquidio_set_mac - Net device set_mac_address
1996  * @netdev: network device
1997  * @p: pointer to sockaddr
1998  */
1999 static int liquidio_set_mac(struct net_device *netdev, void *p)
2000 {
2001 	int ret = 0;
2002 	struct lio *lio = GET_LIO(netdev);
2003 	struct octeon_device *oct = lio->oct_dev;
2004 	struct sockaddr *addr = (struct sockaddr *)p;
2005 	struct octnic_ctrl_pkt nctrl;
2006 
2007 	if (!is_valid_ether_addr(addr->sa_data))
2008 		return -EADDRNOTAVAIL;
2009 
2010 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2011 
2012 	nctrl.ncmd.u64 = 0;
2013 	nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2014 	nctrl.ncmd.s.param1 = 0;
2015 	nctrl.ncmd.s.more = 1;
2016 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2017 	nctrl.netpndev = (u64)netdev;
2018 
2019 	nctrl.udd[0] = 0;
2020 	/* The MAC Address is presented in network byte order. */
2021 	memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2022 
2023 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2024 	if (ret < 0) {
2025 		dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2026 		return -ENOMEM;
2027 	}
2028 
2029 	if (nctrl.sc_status) {
2030 		dev_err(&oct->pci_dev->dev,
2031 			"%s: MAC Address change failed. sc return=%x\n",
2032 			 __func__, nctrl.sc_status);
2033 		return -EIO;
2034 	}
2035 
2036 	eth_hw_addr_set(netdev, addr->sa_data);
2037 	memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2038 
2039 	return 0;
2040 }
2041 
2042 static void
2043 liquidio_get_stats64(struct net_device *netdev,
2044 		     struct rtnl_link_stats64 *lstats)
2045 {
2046 	struct lio *lio = GET_LIO(netdev);
2047 	struct octeon_device *oct;
2048 	u64 pkts = 0, drop = 0, bytes = 0;
2049 	struct oct_droq_stats *oq_stats;
2050 	struct oct_iq_stats *iq_stats;
2051 	int i, iq_no, oq_no;
2052 
2053 	oct = lio->oct_dev;
2054 
2055 	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
2056 		return;
2057 
2058 	for (i = 0; i < oct->num_iqs; i++) {
2059 		iq_no = lio->linfo.txpciq[i].s.q_no;
2060 		iq_stats = &oct->instr_queue[iq_no]->stats;
2061 		pkts += iq_stats->tx_done;
2062 		drop += iq_stats->tx_dropped;
2063 		bytes += iq_stats->tx_tot_bytes;
2064 	}
2065 
2066 	lstats->tx_packets = pkts;
2067 	lstats->tx_bytes = bytes;
2068 	lstats->tx_dropped = drop;
2069 
2070 	pkts = 0;
2071 	drop = 0;
2072 	bytes = 0;
2073 
2074 	for (i = 0; i < oct->num_oqs; i++) {
2075 		oq_no = lio->linfo.rxpciq[i].s.q_no;
2076 		oq_stats = &oct->droq[oq_no]->stats;
2077 		pkts += oq_stats->rx_pkts_received;
2078 		drop += (oq_stats->rx_dropped +
2079 			 oq_stats->dropped_nodispatch +
2080 			 oq_stats->dropped_toomany +
2081 			 oq_stats->dropped_nomem);
2082 		bytes += oq_stats->rx_bytes_received;
2083 	}
2084 
2085 	lstats->rx_bytes = bytes;
2086 	lstats->rx_packets = pkts;
2087 	lstats->rx_dropped = drop;
2088 
2089 	lstats->multicast = oct->link_stats.fromwire.fw_total_mcast;
2090 	lstats->collisions = oct->link_stats.fromhost.total_collisions;
2091 
2092 	/* detailed rx_errors: */
2093 	lstats->rx_length_errors = oct->link_stats.fromwire.l2_err;
2094 	/* recved pkt with crc error    */
2095 	lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err;
2096 	/* recv'd frame alignment error */
2097 	lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err;
2098 	/* recv'r fifo overrun */
2099 	lstats->rx_fifo_errors = oct->link_stats.fromwire.fifo_err;
2100 
2101 	lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors +
2102 		lstats->rx_frame_errors + lstats->rx_fifo_errors;
2103 
2104 	/* detailed tx_errors */
2105 	lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko;
2106 	lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link;
2107 	lstats->tx_fifo_errors = oct->link_stats.fromhost.fifo_err;
2108 
2109 	lstats->tx_errors = lstats->tx_aborted_errors +
2110 		lstats->tx_carrier_errors +
2111 		lstats->tx_fifo_errors;
2112 }
2113 
2114 /**
2115  * hwtstamp_ioctl - Handler for SIOCSHWTSTAMP ioctl
2116  * @netdev: network device
2117  * @ifr: interface request
2118  */
2119 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
2120 {
2121 	struct hwtstamp_config conf;
2122 	struct lio *lio = GET_LIO(netdev);
2123 
2124 	if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2125 		return -EFAULT;
2126 
2127 	switch (conf.tx_type) {
2128 	case HWTSTAMP_TX_ON:
2129 	case HWTSTAMP_TX_OFF:
2130 		break;
2131 	default:
2132 		return -ERANGE;
2133 	}
2134 
2135 	switch (conf.rx_filter) {
2136 	case HWTSTAMP_FILTER_NONE:
2137 		break;
2138 	case HWTSTAMP_FILTER_ALL:
2139 	case HWTSTAMP_FILTER_SOME:
2140 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2141 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2142 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2143 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2144 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2145 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2146 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2147 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2148 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2149 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
2150 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
2151 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2152 	case HWTSTAMP_FILTER_NTP_ALL:
2153 		conf.rx_filter = HWTSTAMP_FILTER_ALL;
2154 		break;
2155 	default:
2156 		return -ERANGE;
2157 	}
2158 
2159 	if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2160 		ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2161 
2162 	else
2163 		ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2164 
2165 	return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2166 }
2167 
2168 /**
2169  * liquidio_ioctl - ioctl handler
2170  * @netdev: network device
2171  * @ifr: interface request
2172  * @cmd: command
2173  */
2174 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2175 {
2176 	struct lio *lio = GET_LIO(netdev);
2177 
2178 	switch (cmd) {
2179 	case SIOCSHWTSTAMP:
2180 		if (lio->oct_dev->ptp_enable)
2181 			return hwtstamp_ioctl(netdev, ifr);
2182 		fallthrough;
2183 	default:
2184 		return -EOPNOTSUPP;
2185 	}
2186 }
2187 
2188 /**
2189  * handle_timestamp - handle a Tx timestamp response
2190  * @oct: octeon device
2191  * @status: response status
2192  * @buf: pointer to skb
2193  */
2194 static void handle_timestamp(struct octeon_device *oct,
2195 			     u32 status,
2196 			     void *buf)
2197 {
2198 	struct octnet_buf_free_info *finfo;
2199 	struct octeon_soft_command *sc;
2200 	struct oct_timestamp_resp *resp;
2201 	struct lio *lio;
2202 	struct sk_buff *skb = (struct sk_buff *)buf;
2203 
2204 	finfo = (struct octnet_buf_free_info *)skb->cb;
2205 	lio = finfo->lio;
2206 	sc = finfo->sc;
2207 	oct = lio->oct_dev;
2208 	resp = (struct oct_timestamp_resp *)sc->virtrptr;
2209 
2210 	if (status != OCTEON_REQUEST_DONE) {
2211 		dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2212 			CVM_CAST64(status));
2213 		resp->timestamp = 0;
2214 	}
2215 
2216 	octeon_swap_8B_data(&resp->timestamp, 1);
2217 
2218 	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
2219 		struct skb_shared_hwtstamps ts;
2220 		u64 ns = resp->timestamp;
2221 
2222 		netif_info(lio, tx_done, lio->netdev,
2223 			   "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2224 			   skb, (unsigned long long)ns);
2225 		ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
2226 		skb_tstamp_tx(skb, &ts);
2227 	}
2228 
2229 	octeon_free_soft_command(oct, sc);
2230 	tx_buffer_free(skb);
2231 }
2232 
2233 /**
2234  * send_nic_timestamp_pkt - Send a data packet that will be timestamped
2235  * @oct: octeon device
2236  * @ndata: pointer to network data
2237  * @finfo: pointer to private network data
2238  * @xmit_more: more is coming
2239  */
2240 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
2241 					 struct octnic_data_pkt *ndata,
2242 					 struct octnet_buf_free_info *finfo,
2243 					 int xmit_more)
2244 {
2245 	int retval;
2246 	struct octeon_soft_command *sc;
2247 	struct lio *lio;
2248 	int ring_doorbell;
2249 	u32 len;
2250 
2251 	lio = finfo->lio;
2252 
2253 	sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
2254 					    sizeof(struct oct_timestamp_resp));
2255 	finfo->sc = sc;
2256 
2257 	if (!sc) {
2258 		dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
2259 		return IQ_SEND_FAILED;
2260 	}
2261 
2262 	if (ndata->reqtype == REQTYPE_NORESP_NET)
2263 		ndata->reqtype = REQTYPE_RESP_NET;
2264 	else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
2265 		ndata->reqtype = REQTYPE_RESP_NET_SG;
2266 
2267 	sc->callback = handle_timestamp;
2268 	sc->callback_arg = finfo->skb;
2269 	sc->iq_no = ndata->q_no;
2270 
2271 	if (OCTEON_CN23XX_PF(oct))
2272 		len = (u32)((struct octeon_instr_ih3 *)
2273 			    (&sc->cmd.cmd3.ih3))->dlengsz;
2274 	else
2275 		len = (u32)((struct octeon_instr_ih2 *)
2276 			    (&sc->cmd.cmd2.ih2))->dlengsz;
2277 
2278 	ring_doorbell = !xmit_more;
2279 
2280 	retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
2281 				     sc, len, ndata->reqtype);
2282 
2283 	if (retval == IQ_SEND_FAILED) {
2284 		dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
2285 			retval);
2286 		octeon_free_soft_command(oct, sc);
2287 	} else {
2288 		netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
2289 	}
2290 
2291 	return retval;
2292 }
2293 
2294 /**
2295  * liquidio_xmit - Transmit networks packets to the Octeon interface
2296  * @skb: skbuff struct to be passed to network layer.
2297  * @netdev: pointer to network device
2298  *
2299  * Return: whether the packet was transmitted to the device okay or not
2300  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
2301  */
2302 static netdev_tx_t liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
2303 {
2304 	struct lio *lio;
2305 	struct octnet_buf_free_info *finfo;
2306 	union octnic_cmd_setup cmdsetup;
2307 	struct octnic_data_pkt ndata;
2308 	struct octeon_device *oct;
2309 	struct oct_iq_stats *stats;
2310 	struct octeon_instr_irh *irh;
2311 	union tx_info *tx_info;
2312 	int status = 0;
2313 	int q_idx = 0, iq_no = 0;
2314 	int j, xmit_more = 0;
2315 	u64 dptr = 0;
2316 	u32 tag = 0;
2317 
2318 	lio = GET_LIO(netdev);
2319 	oct = lio->oct_dev;
2320 
2321 	q_idx = skb_iq(oct, skb);
2322 	tag = q_idx;
2323 	iq_no = lio->linfo.txpciq[q_idx].s.q_no;
2324 
2325 	stats = &oct->instr_queue[iq_no]->stats;
2326 
2327 	/* Check for all conditions in which the current packet cannot be
2328 	 * transmitted.
2329 	 */
2330 	if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
2331 	    (!lio->linfo.link.s.link_up) ||
2332 	    (skb->len <= 0)) {
2333 		netif_info(lio, tx_err, lio->netdev,
2334 			   "Transmit failed link_status : %d\n",
2335 			   lio->linfo.link.s.link_up);
2336 		goto lio_xmit_failed;
2337 	}
2338 
2339 	/* Use space in skb->cb to store info used to unmap and
2340 	 * free the buffers.
2341 	 */
2342 	finfo = (struct octnet_buf_free_info *)skb->cb;
2343 	finfo->lio = lio;
2344 	finfo->skb = skb;
2345 	finfo->sc = NULL;
2346 
2347 	/* Prepare the attributes for the data to be passed to OSI. */
2348 	memset(&ndata, 0, sizeof(struct octnic_data_pkt));
2349 
2350 	ndata.buf = (void *)finfo;
2351 
2352 	ndata.q_no = iq_no;
2353 
2354 	if (octnet_iq_is_full(oct, ndata.q_no)) {
2355 		/* defer sending if queue is full */
2356 		netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2357 			   ndata.q_no);
2358 		stats->tx_iq_busy++;
2359 		return NETDEV_TX_BUSY;
2360 	}
2361 
2362 	/* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",
2363 	 *	lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
2364 	 */
2365 
2366 	ndata.datasize = skb->len;
2367 
2368 	cmdsetup.u64 = 0;
2369 	cmdsetup.s.iq_no = iq_no;
2370 
2371 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
2372 		if (skb->encapsulation) {
2373 			cmdsetup.s.tnl_csum = 1;
2374 			stats->tx_vxlan++;
2375 		} else {
2376 			cmdsetup.s.transport_csum = 1;
2377 		}
2378 	}
2379 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
2380 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2381 		cmdsetup.s.timestamp = 1;
2382 	}
2383 
2384 	if (skb_shinfo(skb)->nr_frags == 0) {
2385 		cmdsetup.s.u.datasize = skb->len;
2386 		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2387 
2388 		/* Offload checksum calculation for TCP/UDP packets */
2389 		dptr = dma_map_single(&oct->pci_dev->dev,
2390 				      skb->data,
2391 				      skb->len,
2392 				      DMA_TO_DEVICE);
2393 		if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
2394 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
2395 				__func__);
2396 			stats->tx_dmamap_fail++;
2397 			return NETDEV_TX_BUSY;
2398 		}
2399 
2400 		if (OCTEON_CN23XX_PF(oct))
2401 			ndata.cmd.cmd3.dptr = dptr;
2402 		else
2403 			ndata.cmd.cmd2.dptr = dptr;
2404 		finfo->dptr = dptr;
2405 		ndata.reqtype = REQTYPE_NORESP_NET;
2406 
2407 	} else {
2408 		int i, frags;
2409 		skb_frag_t *frag;
2410 		struct octnic_gather *g;
2411 
2412 		spin_lock(&lio->glist_lock[q_idx]);
2413 		g = (struct octnic_gather *)
2414 			lio_list_delete_head(&lio->glist[q_idx]);
2415 		spin_unlock(&lio->glist_lock[q_idx]);
2416 
2417 		if (!g) {
2418 			netif_info(lio, tx_err, lio->netdev,
2419 				   "Transmit scatter gather: glist null!\n");
2420 			goto lio_xmit_failed;
2421 		}
2422 
2423 		cmdsetup.s.gather = 1;
2424 		cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
2425 		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2426 
2427 		memset(g->sg, 0, g->sg_size);
2428 
2429 		g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
2430 						 skb->data,
2431 						 (skb->len - skb->data_len),
2432 						 DMA_TO_DEVICE);
2433 		if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
2434 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
2435 				__func__);
2436 			stats->tx_dmamap_fail++;
2437 			return NETDEV_TX_BUSY;
2438 		}
2439 		add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
2440 
2441 		frags = skb_shinfo(skb)->nr_frags;
2442 		i = 1;
2443 		while (frags--) {
2444 			frag = &skb_shinfo(skb)->frags[i - 1];
2445 
2446 			g->sg[(i >> 2)].ptr[(i & 3)] =
2447 				skb_frag_dma_map(&oct->pci_dev->dev,
2448 					         frag, 0, skb_frag_size(frag),
2449 						 DMA_TO_DEVICE);
2450 
2451 			if (dma_mapping_error(&oct->pci_dev->dev,
2452 					      g->sg[i >> 2].ptr[i & 3])) {
2453 				dma_unmap_single(&oct->pci_dev->dev,
2454 						 g->sg[0].ptr[0],
2455 						 skb->len - skb->data_len,
2456 						 DMA_TO_DEVICE);
2457 				for (j = 1; j < i; j++) {
2458 					frag = &skb_shinfo(skb)->frags[j - 1];
2459 					dma_unmap_page(&oct->pci_dev->dev,
2460 						       g->sg[j >> 2].ptr[j & 3],
2461 						       skb_frag_size(frag),
2462 						       DMA_TO_DEVICE);
2463 				}
2464 				dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
2465 					__func__);
2466 				return NETDEV_TX_BUSY;
2467 			}
2468 
2469 			add_sg_size(&g->sg[(i >> 2)], skb_frag_size(frag),
2470 				    (i & 3));
2471 			i++;
2472 		}
2473 
2474 		dptr = g->sg_dma_ptr;
2475 
2476 		if (OCTEON_CN23XX_PF(oct))
2477 			ndata.cmd.cmd3.dptr = dptr;
2478 		else
2479 			ndata.cmd.cmd2.dptr = dptr;
2480 		finfo->dptr = dptr;
2481 		finfo->g = g;
2482 
2483 		ndata.reqtype = REQTYPE_NORESP_NET_SG;
2484 	}
2485 
2486 	if (OCTEON_CN23XX_PF(oct)) {
2487 		irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
2488 		tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];
2489 	} else {
2490 		irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
2491 		tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
2492 	}
2493 
2494 	if (skb_shinfo(skb)->gso_size) {
2495 		tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
2496 		tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
2497 		stats->tx_gso++;
2498 	}
2499 
2500 	/* HW insert VLAN tag */
2501 	if (skb_vlan_tag_present(skb)) {
2502 		irh->priority = skb_vlan_tag_get(skb) >> 13;
2503 		irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
2504 	}
2505 
2506 	xmit_more = netdev_xmit_more();
2507 
2508 	if (unlikely(cmdsetup.s.timestamp))
2509 		status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
2510 	else
2511 		status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
2512 	if (status == IQ_SEND_FAILED)
2513 		goto lio_xmit_failed;
2514 
2515 	netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
2516 
2517 	if (status == IQ_SEND_STOP)
2518 		netif_stop_subqueue(netdev, q_idx);
2519 
2520 	netif_trans_update(netdev);
2521 
2522 	if (tx_info->s.gso_segs)
2523 		stats->tx_done += tx_info->s.gso_segs;
2524 	else
2525 		stats->tx_done++;
2526 	stats->tx_tot_bytes += ndata.datasize;
2527 
2528 	return NETDEV_TX_OK;
2529 
2530 lio_xmit_failed:
2531 	stats->tx_dropped++;
2532 	netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
2533 		   iq_no, stats->tx_dropped);
2534 	if (dptr)
2535 		dma_unmap_single(&oct->pci_dev->dev, dptr,
2536 				 ndata.datasize, DMA_TO_DEVICE);
2537 
2538 	octeon_ring_doorbell_locked(oct, iq_no);
2539 
2540 	tx_buffer_free(skb);
2541 	return NETDEV_TX_OK;
2542 }
2543 
2544 /**
2545  * liquidio_tx_timeout - Network device Tx timeout
2546  * @netdev:    pointer to network device
2547  * @txqueue: index of the hung transmit queue
2548  */
2549 static void liquidio_tx_timeout(struct net_device *netdev, unsigned int txqueue)
2550 {
2551 	struct lio *lio;
2552 
2553 	lio = GET_LIO(netdev);
2554 
2555 	netif_info(lio, tx_err, lio->netdev,
2556 		   "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2557 		   netdev->stats.tx_dropped);
2558 	netif_trans_update(netdev);
2559 	wake_txqs(netdev);
2560 }
2561 
2562 static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
2563 				    __be16 proto __attribute__((unused)),
2564 				    u16 vid)
2565 {
2566 	struct lio *lio = GET_LIO(netdev);
2567 	struct octeon_device *oct = lio->oct_dev;
2568 	struct octnic_ctrl_pkt nctrl;
2569 	int ret = 0;
2570 
2571 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2572 
2573 	nctrl.ncmd.u64 = 0;
2574 	nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2575 	nctrl.ncmd.s.param1 = vid;
2576 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2577 	nctrl.netpndev = (u64)netdev;
2578 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2579 
2580 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2581 	if (ret) {
2582 		dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2583 			ret);
2584 		if (ret > 0)
2585 			ret = -EIO;
2586 	}
2587 
2588 	return ret;
2589 }
2590 
2591 static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
2592 				     __be16 proto __attribute__((unused)),
2593 				     u16 vid)
2594 {
2595 	struct lio *lio = GET_LIO(netdev);
2596 	struct octeon_device *oct = lio->oct_dev;
2597 	struct octnic_ctrl_pkt nctrl;
2598 	int ret = 0;
2599 
2600 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2601 
2602 	nctrl.ncmd.u64 = 0;
2603 	nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2604 	nctrl.ncmd.s.param1 = vid;
2605 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2606 	nctrl.netpndev = (u64)netdev;
2607 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2608 
2609 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2610 	if (ret) {
2611 		dev_err(&oct->pci_dev->dev, "Del VLAN filter failed in core (ret: 0x%x)\n",
2612 			ret);
2613 		if (ret > 0)
2614 			ret = -EIO;
2615 	}
2616 	return ret;
2617 }
2618 
2619 /**
2620  * liquidio_set_rxcsum_command - Sending command to enable/disable RX checksum offload
2621  * @netdev:                pointer to network device
2622  * @command:               OCTNET_CMD_TNL_RX_CSUM_CTL
2623  * @rx_cmd:                OCTNET_CMD_RXCSUM_ENABLE/OCTNET_CMD_RXCSUM_DISABLE
2624  * Returns:                SUCCESS or FAILURE
2625  */
2626 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
2627 				       u8 rx_cmd)
2628 {
2629 	struct lio *lio = GET_LIO(netdev);
2630 	struct octeon_device *oct = lio->oct_dev;
2631 	struct octnic_ctrl_pkt nctrl;
2632 	int ret = 0;
2633 
2634 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2635 
2636 	nctrl.ncmd.u64 = 0;
2637 	nctrl.ncmd.s.cmd = command;
2638 	nctrl.ncmd.s.param1 = rx_cmd;
2639 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2640 	nctrl.netpndev = (u64)netdev;
2641 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2642 
2643 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2644 	if (ret) {
2645 		dev_err(&oct->pci_dev->dev,
2646 			"DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
2647 			ret);
2648 		if (ret > 0)
2649 			ret = -EIO;
2650 	}
2651 	return ret;
2652 }
2653 
2654 /**
2655  * liquidio_vxlan_port_command - Sending command to add/delete VxLAN UDP port to firmware
2656  * @netdev:                pointer to network device
2657  * @command:               OCTNET_CMD_VXLAN_PORT_CONFIG
2658  * @vxlan_port:            VxLAN port to be added or deleted
2659  * @vxlan_cmd_bit:         OCTNET_CMD_VXLAN_PORT_ADD,
2660  *                              OCTNET_CMD_VXLAN_PORT_DEL
2661  * Return:                     SUCCESS or FAILURE
2662  */
2663 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
2664 				       u16 vxlan_port, u8 vxlan_cmd_bit)
2665 {
2666 	struct lio *lio = GET_LIO(netdev);
2667 	struct octeon_device *oct = lio->oct_dev;
2668 	struct octnic_ctrl_pkt nctrl;
2669 	int ret = 0;
2670 
2671 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2672 
2673 	nctrl.ncmd.u64 = 0;
2674 	nctrl.ncmd.s.cmd = command;
2675 	nctrl.ncmd.s.more = vxlan_cmd_bit;
2676 	nctrl.ncmd.s.param1 = vxlan_port;
2677 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2678 	nctrl.netpndev = (u64)netdev;
2679 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2680 
2681 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2682 	if (ret) {
2683 		dev_err(&oct->pci_dev->dev,
2684 			"VxLAN port add/delete failed in core (ret:0x%x)\n",
2685 			ret);
2686 		if (ret > 0)
2687 			ret = -EIO;
2688 	}
2689 	return ret;
2690 }
2691 
2692 static int liquidio_udp_tunnel_set_port(struct net_device *netdev,
2693 					unsigned int table, unsigned int entry,
2694 					struct udp_tunnel_info *ti)
2695 {
2696 	return liquidio_vxlan_port_command(netdev,
2697 					   OCTNET_CMD_VXLAN_PORT_CONFIG,
2698 					   htons(ti->port),
2699 					   OCTNET_CMD_VXLAN_PORT_ADD);
2700 }
2701 
2702 static int liquidio_udp_tunnel_unset_port(struct net_device *netdev,
2703 					  unsigned int table,
2704 					  unsigned int entry,
2705 					  struct udp_tunnel_info *ti)
2706 {
2707 	return liquidio_vxlan_port_command(netdev,
2708 					   OCTNET_CMD_VXLAN_PORT_CONFIG,
2709 					   htons(ti->port),
2710 					   OCTNET_CMD_VXLAN_PORT_DEL);
2711 }
2712 
2713 static const struct udp_tunnel_nic_info liquidio_udp_tunnels = {
2714 	.set_port	= liquidio_udp_tunnel_set_port,
2715 	.unset_port	= liquidio_udp_tunnel_unset_port,
2716 	.tables		= {
2717 		{ .n_entries = 1024, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
2718 	},
2719 };
2720 
2721 /**
2722  * liquidio_fix_features - Net device fix features
2723  * @netdev:  pointer to network device
2724  * @request: features requested
2725  * Return: updated features list
2726  */
2727 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
2728 					       netdev_features_t request)
2729 {
2730 	struct lio *lio = netdev_priv(netdev);
2731 
2732 	if ((request & NETIF_F_RXCSUM) &&
2733 	    !(lio->dev_capability & NETIF_F_RXCSUM))
2734 		request &= ~NETIF_F_RXCSUM;
2735 
2736 	if ((request & NETIF_F_HW_CSUM) &&
2737 	    !(lio->dev_capability & NETIF_F_HW_CSUM))
2738 		request &= ~NETIF_F_HW_CSUM;
2739 
2740 	if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
2741 		request &= ~NETIF_F_TSO;
2742 
2743 	if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
2744 		request &= ~NETIF_F_TSO6;
2745 
2746 	if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
2747 		request &= ~NETIF_F_LRO;
2748 
2749 	/*Disable LRO if RXCSUM is off */
2750 	if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
2751 	    (lio->dev_capability & NETIF_F_LRO))
2752 		request &= ~NETIF_F_LRO;
2753 
2754 	if ((request & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2755 	    !(lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER))
2756 		request &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
2757 
2758 	return request;
2759 }
2760 
2761 /**
2762  * liquidio_set_features - Net device set features
2763  * @netdev:  pointer to network device
2764  * @features: features to enable/disable
2765  */
2766 static int liquidio_set_features(struct net_device *netdev,
2767 				 netdev_features_t features)
2768 {
2769 	struct lio *lio = netdev_priv(netdev);
2770 
2771 	if ((features & NETIF_F_LRO) &&
2772 	    (lio->dev_capability & NETIF_F_LRO) &&
2773 	    !(netdev->features & NETIF_F_LRO))
2774 		liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
2775 				     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2776 	else if (!(features & NETIF_F_LRO) &&
2777 		 (lio->dev_capability & NETIF_F_LRO) &&
2778 		 (netdev->features & NETIF_F_LRO))
2779 		liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
2780 				     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2781 
2782 	/* Sending command to firmware to enable/disable RX checksum
2783 	 * offload settings using ethtool
2784 	 */
2785 	if (!(netdev->features & NETIF_F_RXCSUM) &&
2786 	    (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
2787 	    (features & NETIF_F_RXCSUM))
2788 		liquidio_set_rxcsum_command(netdev,
2789 					    OCTNET_CMD_TNL_RX_CSUM_CTL,
2790 					    OCTNET_CMD_RXCSUM_ENABLE);
2791 	else if ((netdev->features & NETIF_F_RXCSUM) &&
2792 		 (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
2793 		 !(features & NETIF_F_RXCSUM))
2794 		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
2795 					    OCTNET_CMD_RXCSUM_DISABLE);
2796 
2797 	if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2798 	    (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2799 	    !(netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
2800 		liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
2801 				     OCTNET_CMD_VLAN_FILTER_ENABLE);
2802 	else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2803 		 (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2804 		 (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
2805 		liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
2806 				     OCTNET_CMD_VLAN_FILTER_DISABLE);
2807 
2808 	return 0;
2809 }
2810 
2811 static int __liquidio_set_vf_mac(struct net_device *netdev, int vfidx,
2812 				 u8 *mac, bool is_admin_assigned)
2813 {
2814 	struct lio *lio = GET_LIO(netdev);
2815 	struct octeon_device *oct = lio->oct_dev;
2816 	struct octnic_ctrl_pkt nctrl;
2817 	int ret = 0;
2818 
2819 	if (!is_valid_ether_addr(mac))
2820 		return -EINVAL;
2821 
2822 	if (vfidx < 0 || vfidx >= oct->sriov_info.max_vfs)
2823 		return -EINVAL;
2824 
2825 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2826 
2827 	nctrl.ncmd.u64 = 0;
2828 	nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2829 	/* vfidx is 0 based, but vf_num (param1) is 1 based */
2830 	nctrl.ncmd.s.param1 = vfidx + 1;
2831 	nctrl.ncmd.s.more = 1;
2832 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2833 	nctrl.netpndev = (u64)netdev;
2834 	if (is_admin_assigned) {
2835 		nctrl.ncmd.s.param2 = true;
2836 		nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2837 	}
2838 
2839 	nctrl.udd[0] = 0;
2840 	/* The MAC Address is presented in network byte order. */
2841 	ether_addr_copy((u8 *)&nctrl.udd[0] + 2, mac);
2842 
2843 	oct->sriov_info.vf_macaddr[vfidx] = nctrl.udd[0];
2844 
2845 	ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2846 	if (ret > 0)
2847 		ret = -EIO;
2848 
2849 	return ret;
2850 }
2851 
2852 static int liquidio_set_vf_mac(struct net_device *netdev, int vfidx, u8 *mac)
2853 {
2854 	struct lio *lio = GET_LIO(netdev);
2855 	struct octeon_device *oct = lio->oct_dev;
2856 	int retval;
2857 
2858 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2859 		return -EINVAL;
2860 
2861 	retval = __liquidio_set_vf_mac(netdev, vfidx, mac, true);
2862 	if (!retval)
2863 		cn23xx_tell_vf_its_macaddr_changed(oct, vfidx, mac);
2864 
2865 	return retval;
2866 }
2867 
2868 static int liquidio_set_vf_spoofchk(struct net_device *netdev, int vfidx,
2869 				    bool enable)
2870 {
2871 	struct lio *lio = GET_LIO(netdev);
2872 	struct octeon_device *oct = lio->oct_dev;
2873 	struct octnic_ctrl_pkt nctrl;
2874 	int retval;
2875 
2876 	if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SPOOFCHK_CAP)) {
2877 		netif_info(lio, drv, lio->netdev,
2878 			   "firmware does not support spoofchk\n");
2879 		return -EOPNOTSUPP;
2880 	}
2881 
2882 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {
2883 		netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);
2884 		return -EINVAL;
2885 	}
2886 
2887 	if (enable) {
2888 		if (oct->sriov_info.vf_spoofchk[vfidx])
2889 			return 0;
2890 	} else {
2891 		/* Clear */
2892 		if (!oct->sriov_info.vf_spoofchk[vfidx])
2893 			return 0;
2894 	}
2895 
2896 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2897 	nctrl.ncmd.s.cmdgroup = OCTNET_CMD_GROUP1;
2898 	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_SPOOFCHK;
2899 	nctrl.ncmd.s.param1 =
2900 		vfidx + 1; /* vfidx is 0 based,
2901 			    * but vf_num (param1) is 1 based
2902 			    */
2903 	nctrl.ncmd.s.param2 = enable;
2904 	nctrl.ncmd.s.more = 0;
2905 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2906 	nctrl.cb_fn = NULL;
2907 
2908 	retval = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2909 
2910 	if (retval) {
2911 		netif_info(lio, drv, lio->netdev,
2912 			   "Failed to set VF %d spoofchk %s\n", vfidx,
2913 			enable ? "on" : "off");
2914 		return -1;
2915 	}
2916 
2917 	oct->sriov_info.vf_spoofchk[vfidx] = enable;
2918 	netif_info(lio, drv, lio->netdev, "VF %u spoofchk is %s\n", vfidx,
2919 		   enable ? "on" : "off");
2920 
2921 	return 0;
2922 }
2923 
2924 static int liquidio_set_vf_vlan(struct net_device *netdev, int vfidx,
2925 				u16 vlan, u8 qos, __be16 vlan_proto)
2926 {
2927 	struct lio *lio = GET_LIO(netdev);
2928 	struct octeon_device *oct = lio->oct_dev;
2929 	struct octnic_ctrl_pkt nctrl;
2930 	u16 vlantci;
2931 	int ret = 0;
2932 
2933 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2934 		return -EINVAL;
2935 
2936 	if (vlan_proto != htons(ETH_P_8021Q))
2937 		return -EPROTONOSUPPORT;
2938 
2939 	if (vlan >= VLAN_N_VID || qos > 7)
2940 		return -EINVAL;
2941 
2942 	if (vlan)
2943 		vlantci = vlan | (u16)qos << VLAN_PRIO_SHIFT;
2944 	else
2945 		vlantci = 0;
2946 
2947 	if (oct->sriov_info.vf_vlantci[vfidx] == vlantci)
2948 		return 0;
2949 
2950 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2951 
2952 	if (vlan)
2953 		nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2954 	else
2955 		nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2956 
2957 	nctrl.ncmd.s.param1 = vlantci;
2958 	nctrl.ncmd.s.param2 =
2959 	    vfidx + 1; /* vfidx is 0 based, but vf_num (param2) is 1 based */
2960 	nctrl.ncmd.s.more = 0;
2961 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2962 	nctrl.cb_fn = NULL;
2963 
2964 	ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2965 	if (ret) {
2966 		if (ret > 0)
2967 			ret = -EIO;
2968 		return ret;
2969 	}
2970 
2971 	oct->sriov_info.vf_vlantci[vfidx] = vlantci;
2972 
2973 	return ret;
2974 }
2975 
2976 static int liquidio_get_vf_config(struct net_device *netdev, int vfidx,
2977 				  struct ifla_vf_info *ivi)
2978 {
2979 	struct lio *lio = GET_LIO(netdev);
2980 	struct octeon_device *oct = lio->oct_dev;
2981 	u8 *macaddr;
2982 
2983 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2984 		return -EINVAL;
2985 
2986 	memset(ivi, 0, sizeof(struct ifla_vf_info));
2987 
2988 	ivi->vf = vfidx;
2989 	macaddr = 2 + (u8 *)&oct->sriov_info.vf_macaddr[vfidx];
2990 	ether_addr_copy(&ivi->mac[0], macaddr);
2991 	ivi->vlan = oct->sriov_info.vf_vlantci[vfidx] & VLAN_VID_MASK;
2992 	ivi->qos = oct->sriov_info.vf_vlantci[vfidx] >> VLAN_PRIO_SHIFT;
2993 	if (oct->sriov_info.trusted_vf.active &&
2994 	    oct->sriov_info.trusted_vf.id == vfidx)
2995 		ivi->trusted = true;
2996 	else
2997 		ivi->trusted = false;
2998 	ivi->linkstate = oct->sriov_info.vf_linkstate[vfidx];
2999 	ivi->spoofchk = oct->sriov_info.vf_spoofchk[vfidx];
3000 	ivi->max_tx_rate = lio->linfo.link.s.speed;
3001 	ivi->min_tx_rate = 0;
3002 
3003 	return 0;
3004 }
3005 
3006 static int liquidio_send_vf_trust_cmd(struct lio *lio, int vfidx, bool trusted)
3007 {
3008 	struct octeon_device *oct = lio->oct_dev;
3009 	struct octeon_soft_command *sc;
3010 	int retval;
3011 
3012 	sc = octeon_alloc_soft_command(oct, 0, 16, 0);
3013 	if (!sc)
3014 		return -ENOMEM;
3015 
3016 	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
3017 
3018 	/* vfidx is 0 based, but vf_num (param1) is 1 based */
3019 	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
3020 				    OPCODE_NIC_SET_TRUSTED_VF, 0, vfidx + 1,
3021 				    trusted);
3022 
3023 	init_completion(&sc->complete);
3024 	sc->sc_status = OCTEON_REQUEST_PENDING;
3025 
3026 	retval = octeon_send_soft_command(oct, sc);
3027 	if (retval == IQ_SEND_FAILED) {
3028 		octeon_free_soft_command(oct, sc);
3029 		retval = -1;
3030 	} else {
3031 		/* Wait for response or timeout */
3032 		retval = wait_for_sc_completion_timeout(oct, sc, 0);
3033 		if (retval)
3034 			return (retval);
3035 
3036 		WRITE_ONCE(sc->caller_is_done, true);
3037 	}
3038 
3039 	return retval;
3040 }
3041 
3042 static int liquidio_set_vf_trust(struct net_device *netdev, int vfidx,
3043 				 bool setting)
3044 {
3045 	struct lio *lio = GET_LIO(netdev);
3046 	struct octeon_device *oct = lio->oct_dev;
3047 
3048 	if (strcmp(oct->fw_info.liquidio_firmware_version, "1.7.1") < 0) {
3049 		/* trusted vf is not supported by firmware older than 1.7.1 */
3050 		return -EOPNOTSUPP;
3051 	}
3052 
3053 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {
3054 		netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);
3055 		return -EINVAL;
3056 	}
3057 
3058 	if (setting) {
3059 		/* Set */
3060 
3061 		if (oct->sriov_info.trusted_vf.active &&
3062 		    oct->sriov_info.trusted_vf.id == vfidx)
3063 			return 0;
3064 
3065 		if (oct->sriov_info.trusted_vf.active) {
3066 			netif_info(lio, drv, lio->netdev, "More than one trusted VF is not allowed\n");
3067 			return -EPERM;
3068 		}
3069 	} else {
3070 		/* Clear */
3071 
3072 		if (!oct->sriov_info.trusted_vf.active)
3073 			return 0;
3074 	}
3075 
3076 	if (!liquidio_send_vf_trust_cmd(lio, vfidx, setting)) {
3077 		if (setting) {
3078 			oct->sriov_info.trusted_vf.id = vfidx;
3079 			oct->sriov_info.trusted_vf.active = true;
3080 		} else {
3081 			oct->sriov_info.trusted_vf.active = false;
3082 		}
3083 
3084 		netif_info(lio, drv, lio->netdev, "VF %u is %strusted\n", vfidx,
3085 			   setting ? "" : "not ");
3086 	} else {
3087 		netif_info(lio, drv, lio->netdev, "Failed to set VF trusted\n");
3088 		return -1;
3089 	}
3090 
3091 	return 0;
3092 }
3093 
3094 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
3095 				      int linkstate)
3096 {
3097 	struct lio *lio = GET_LIO(netdev);
3098 	struct octeon_device *oct = lio->oct_dev;
3099 	struct octnic_ctrl_pkt nctrl;
3100 	int ret = 0;
3101 
3102 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3103 		return -EINVAL;
3104 
3105 	if (oct->sriov_info.vf_linkstate[vfidx] == linkstate)
3106 		return 0;
3107 
3108 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3109 	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_LINKSTATE;
3110 	nctrl.ncmd.s.param1 =
3111 	    vfidx + 1; /* vfidx is 0 based, but vf_num (param1) is 1 based */
3112 	nctrl.ncmd.s.param2 = linkstate;
3113 	nctrl.ncmd.s.more = 0;
3114 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3115 	nctrl.cb_fn = NULL;
3116 
3117 	ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
3118 
3119 	if (!ret)
3120 		oct->sriov_info.vf_linkstate[vfidx] = linkstate;
3121 	else if (ret > 0)
3122 		ret = -EIO;
3123 
3124 	return ret;
3125 }
3126 
3127 static int
3128 liquidio_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3129 {
3130 	struct lio_devlink_priv *priv;
3131 	struct octeon_device *oct;
3132 
3133 	priv = devlink_priv(devlink);
3134 	oct = priv->oct;
3135 
3136 	*mode = oct->eswitch_mode;
3137 
3138 	return 0;
3139 }
3140 
3141 static int
3142 liquidio_eswitch_mode_set(struct devlink *devlink, u16 mode,
3143 			  struct netlink_ext_ack *extack)
3144 {
3145 	struct lio_devlink_priv *priv;
3146 	struct octeon_device *oct;
3147 	int ret = 0;
3148 
3149 	priv = devlink_priv(devlink);
3150 	oct = priv->oct;
3151 
3152 	if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP))
3153 		return -EINVAL;
3154 
3155 	if (oct->eswitch_mode == mode)
3156 		return 0;
3157 
3158 	switch (mode) {
3159 	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3160 		oct->eswitch_mode = mode;
3161 		ret = lio_vf_rep_create(oct);
3162 		break;
3163 
3164 	case DEVLINK_ESWITCH_MODE_LEGACY:
3165 		lio_vf_rep_destroy(oct);
3166 		oct->eswitch_mode = mode;
3167 		break;
3168 
3169 	default:
3170 		ret = -EINVAL;
3171 	}
3172 
3173 	return ret;
3174 }
3175 
3176 static const struct devlink_ops liquidio_devlink_ops = {
3177 	.eswitch_mode_get = liquidio_eswitch_mode_get,
3178 	.eswitch_mode_set = liquidio_eswitch_mode_set,
3179 };
3180 
3181 static int
3182 liquidio_get_port_parent_id(struct net_device *dev,
3183 			    struct netdev_phys_item_id *ppid)
3184 {
3185 	struct lio *lio = GET_LIO(dev);
3186 	struct octeon_device *oct = lio->oct_dev;
3187 
3188 	if (oct->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
3189 		return -EOPNOTSUPP;
3190 
3191 	ppid->id_len = ETH_ALEN;
3192 	ether_addr_copy(ppid->id, (void *)&lio->linfo.hw_addr + 2);
3193 
3194 	return 0;
3195 }
3196 
3197 static int liquidio_get_vf_stats(struct net_device *netdev, int vfidx,
3198 				 struct ifla_vf_stats *vf_stats)
3199 {
3200 	struct lio *lio = GET_LIO(netdev);
3201 	struct octeon_device *oct = lio->oct_dev;
3202 	struct oct_vf_stats stats;
3203 	int ret;
3204 
3205 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3206 		return -EINVAL;
3207 
3208 	memset(&stats, 0, sizeof(struct oct_vf_stats));
3209 	ret = cn23xx_get_vf_stats(oct, vfidx, &stats);
3210 	if (!ret) {
3211 		vf_stats->rx_packets = stats.rx_packets;
3212 		vf_stats->tx_packets = stats.tx_packets;
3213 		vf_stats->rx_bytes = stats.rx_bytes;
3214 		vf_stats->tx_bytes = stats.tx_bytes;
3215 		vf_stats->broadcast = stats.broadcast;
3216 		vf_stats->multicast = stats.multicast;
3217 	}
3218 
3219 	return ret;
3220 }
3221 
3222 static const struct net_device_ops lionetdevops = {
3223 	.ndo_open		= liquidio_open,
3224 	.ndo_stop		= liquidio_stop,
3225 	.ndo_start_xmit		= liquidio_xmit,
3226 	.ndo_get_stats64	= liquidio_get_stats64,
3227 	.ndo_set_mac_address	= liquidio_set_mac,
3228 	.ndo_set_rx_mode	= liquidio_set_mcast_list,
3229 	.ndo_tx_timeout		= liquidio_tx_timeout,
3230 
3231 	.ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
3232 	.ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
3233 	.ndo_change_mtu		= liquidio_change_mtu,
3234 	.ndo_eth_ioctl		= liquidio_ioctl,
3235 	.ndo_fix_features	= liquidio_fix_features,
3236 	.ndo_set_features	= liquidio_set_features,
3237 	.ndo_set_vf_mac		= liquidio_set_vf_mac,
3238 	.ndo_set_vf_vlan	= liquidio_set_vf_vlan,
3239 	.ndo_get_vf_config	= liquidio_get_vf_config,
3240 	.ndo_set_vf_spoofchk	= liquidio_set_vf_spoofchk,
3241 	.ndo_set_vf_trust	= liquidio_set_vf_trust,
3242 	.ndo_set_vf_link_state  = liquidio_set_vf_link_state,
3243 	.ndo_get_vf_stats	= liquidio_get_vf_stats,
3244 	.ndo_get_port_parent_id	= liquidio_get_port_parent_id,
3245 };
3246 
3247 /**
3248  * liquidio_init - Entry point for the liquidio module
3249  */
3250 static int __init liquidio_init(void)
3251 {
3252 	int i;
3253 	struct handshake *hs;
3254 
3255 	init_completion(&first_stage);
3256 
3257 	octeon_init_device_list(OCTEON_CONFIG_TYPE_DEFAULT);
3258 
3259 	if (liquidio_init_pci())
3260 		return -EINVAL;
3261 
3262 	wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3263 
3264 	for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3265 		hs = &handshake[i];
3266 		if (hs->pci_dev) {
3267 			wait_for_completion(&hs->init);
3268 			if (!hs->init_ok) {
3269 				/* init handshake failed */
3270 				dev_err(&hs->pci_dev->dev,
3271 					"Failed to init device\n");
3272 				liquidio_deinit_pci();
3273 				return -EIO;
3274 			}
3275 		}
3276 	}
3277 
3278 	for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3279 		hs = &handshake[i];
3280 		if (hs->pci_dev) {
3281 			wait_for_completion_timeout(&hs->started,
3282 						    msecs_to_jiffies(30000));
3283 			if (!hs->started_ok) {
3284 				/* starter handshake failed */
3285 				dev_err(&hs->pci_dev->dev,
3286 					"Firmware failed to start\n");
3287 				liquidio_deinit_pci();
3288 				return -EIO;
3289 			}
3290 		}
3291 	}
3292 
3293 	return 0;
3294 }
3295 
3296 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3297 {
3298 	struct octeon_device *oct = (struct octeon_device *)buf;
3299 	struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3300 	int gmxport = 0;
3301 	union oct_link_status *ls;
3302 	int i;
3303 
3304 	if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) {
3305 		dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3306 			recv_pkt->buffer_size[0],
3307 			recv_pkt->rh.r_nic_info.gmxport);
3308 		goto nic_info_err;
3309 	}
3310 
3311 	gmxport = recv_pkt->rh.r_nic_info.gmxport;
3312 	ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) +
3313 		OCT_DROQ_INFO_SIZE);
3314 
3315 	octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3316 	for (i = 0; i < oct->ifcount; i++) {
3317 		if (oct->props[i].gmxport == gmxport) {
3318 			update_link_status(oct->props[i].netdev, ls);
3319 			break;
3320 		}
3321 	}
3322 
3323 nic_info_err:
3324 	for (i = 0; i < recv_pkt->buffer_count; i++)
3325 		recv_buffer_free(recv_pkt->buffer_ptr[i]);
3326 	octeon_free_recv_info(recv_info);
3327 	return 0;
3328 }
3329 
3330 /**
3331  * setup_nic_devices - Setup network interfaces
3332  * @octeon_dev:  octeon device
3333  *
3334  * Called during init time for each device. It assumes the NIC
3335  * is already up and running.  The link information for each
3336  * interface is passed in link_info.
3337  */
3338 static int setup_nic_devices(struct octeon_device *octeon_dev)
3339 {
3340 	struct lio *lio = NULL;
3341 	struct net_device *netdev;
3342 	u8 mac[6], i, j, *fw_ver, *micro_ver;
3343 	unsigned long micro;
3344 	u32 cur_ver;
3345 	struct octeon_soft_command *sc;
3346 	struct liquidio_if_cfg_resp *resp;
3347 	struct octdev_props *props;
3348 	int retval, num_iqueues, num_oqueues;
3349 	int max_num_queues = 0;
3350 	union oct_nic_if_cfg if_cfg;
3351 	unsigned int base_queue;
3352 	unsigned int gmx_port_id;
3353 	u32 resp_size, data_size;
3354 	u32 ifidx_or_pfnum;
3355 	struct lio_version *vdata;
3356 	struct devlink *devlink;
3357 	struct lio_devlink_priv *lio_devlink;
3358 
3359 	/* This is to handle link status changes */
3360 	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3361 				    OPCODE_NIC_INFO,
3362 				    lio_nic_info, octeon_dev);
3363 
3364 	/* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3365 	 * They are handled directly.
3366 	 */
3367 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3368 					free_netbuf);
3369 
3370 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3371 					free_netsgbuf);
3372 
3373 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3374 					free_netsgbuf_with_resp);
3375 
3376 	for (i = 0; i < octeon_dev->ifcount; i++) {
3377 		resp_size = sizeof(struct liquidio_if_cfg_resp);
3378 		data_size = sizeof(struct lio_version);
3379 		sc = (struct octeon_soft_command *)
3380 			octeon_alloc_soft_command(octeon_dev, data_size,
3381 						  resp_size, 0);
3382 		resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3383 		vdata = (struct lio_version *)sc->virtdptr;
3384 
3385 		*((u64 *)vdata) = 0;
3386 		vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
3387 		vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
3388 		vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
3389 
3390 		if (OCTEON_CN23XX_PF(octeon_dev)) {
3391 			num_iqueues = octeon_dev->sriov_info.num_pf_rings;
3392 			num_oqueues = octeon_dev->sriov_info.num_pf_rings;
3393 			base_queue = octeon_dev->sriov_info.pf_srn;
3394 
3395 			gmx_port_id = octeon_dev->pf_num;
3396 			ifidx_or_pfnum = octeon_dev->pf_num;
3397 		} else {
3398 			num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(
3399 						octeon_get_conf(octeon_dev), i);
3400 			num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(
3401 						octeon_get_conf(octeon_dev), i);
3402 			base_queue = CFG_GET_BASE_QUE_NIC_IF(
3403 						octeon_get_conf(octeon_dev), i);
3404 			gmx_port_id = CFG_GET_GMXID_NIC_IF(
3405 						octeon_get_conf(octeon_dev), i);
3406 			ifidx_or_pfnum = i;
3407 		}
3408 
3409 		dev_dbg(&octeon_dev->pci_dev->dev,
3410 			"requesting config for interface %d, iqs %d, oqs %d\n",
3411 			ifidx_or_pfnum, num_iqueues, num_oqueues);
3412 
3413 		if_cfg.u64 = 0;
3414 		if_cfg.s.num_iqueues = num_iqueues;
3415 		if_cfg.s.num_oqueues = num_oqueues;
3416 		if_cfg.s.base_queue = base_queue;
3417 		if_cfg.s.gmx_port_id = gmx_port_id;
3418 
3419 		sc->iq_no = 0;
3420 
3421 		octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
3422 					    OPCODE_NIC_IF_CFG, 0,
3423 					    if_cfg.u64, 0);
3424 
3425 		init_completion(&sc->complete);
3426 		sc->sc_status = OCTEON_REQUEST_PENDING;
3427 
3428 		retval = octeon_send_soft_command(octeon_dev, sc);
3429 		if (retval == IQ_SEND_FAILED) {
3430 			dev_err(&octeon_dev->pci_dev->dev,
3431 				"iq/oq config failed status: %x\n",
3432 				retval);
3433 			/* Soft instr is freed by driver in case of failure. */
3434 			octeon_free_soft_command(octeon_dev, sc);
3435 			return(-EIO);
3436 		}
3437 
3438 		/* Sleep on a wait queue till the cond flag indicates that the
3439 		 * response arrived or timed-out.
3440 		 */
3441 		retval = wait_for_sc_completion_timeout(octeon_dev, sc, 0);
3442 		if (retval)
3443 			return retval;
3444 
3445 		retval = resp->status;
3446 		if (retval) {
3447 			dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3448 			WRITE_ONCE(sc->caller_is_done, true);
3449 			goto setup_nic_dev_done;
3450 		}
3451 		snprintf(octeon_dev->fw_info.liquidio_firmware_version,
3452 			 32, "%s",
3453 			 resp->cfg_info.liquidio_firmware_version);
3454 
3455 		/* Verify f/w version (in case of 'auto' loading from flash) */
3456 		fw_ver = octeon_dev->fw_info.liquidio_firmware_version;
3457 		if (memcmp(LIQUIDIO_BASE_VERSION,
3458 			   fw_ver,
3459 			   strlen(LIQUIDIO_BASE_VERSION))) {
3460 			dev_err(&octeon_dev->pci_dev->dev,
3461 				"Unmatched firmware version. Expected %s.x, got %s.\n",
3462 				LIQUIDIO_BASE_VERSION, fw_ver);
3463 			WRITE_ONCE(sc->caller_is_done, true);
3464 			goto setup_nic_dev_done;
3465 		} else if (atomic_read(octeon_dev->adapter_fw_state) ==
3466 			   FW_IS_PRELOADED) {
3467 			dev_info(&octeon_dev->pci_dev->dev,
3468 				 "Using auto-loaded firmware version %s.\n",
3469 				 fw_ver);
3470 		}
3471 
3472 		/* extract micro version field; point past '<maj>.<min>.' */
3473 		micro_ver = fw_ver + strlen(LIQUIDIO_BASE_VERSION) + 1;
3474 		if (kstrtoul(micro_ver, 10, &micro) != 0)
3475 			micro = 0;
3476 		octeon_dev->fw_info.ver.maj = LIQUIDIO_BASE_MAJOR_VERSION;
3477 		octeon_dev->fw_info.ver.min = LIQUIDIO_BASE_MINOR_VERSION;
3478 		octeon_dev->fw_info.ver.rev = micro;
3479 
3480 		octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3481 				    (sizeof(struct liquidio_if_cfg_info)) >> 3);
3482 
3483 		num_iqueues = hweight64(resp->cfg_info.iqmask);
3484 		num_oqueues = hweight64(resp->cfg_info.oqmask);
3485 
3486 		if (!(num_iqueues) || !(num_oqueues)) {
3487 			dev_err(&octeon_dev->pci_dev->dev,
3488 				"Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3489 				resp->cfg_info.iqmask,
3490 				resp->cfg_info.oqmask);
3491 			WRITE_ONCE(sc->caller_is_done, true);
3492 			goto setup_nic_dev_done;
3493 		}
3494 
3495 		if (OCTEON_CN6XXX(octeon_dev)) {
3496 			max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
3497 								    cn6xxx));
3498 		} else if (OCTEON_CN23XX_PF(octeon_dev)) {
3499 			max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
3500 								    cn23xx_pf));
3501 		}
3502 
3503 		dev_dbg(&octeon_dev->pci_dev->dev,
3504 			"interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d max_num_queues: %d\n",
3505 			i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3506 			num_iqueues, num_oqueues, max_num_queues);
3507 		netdev = alloc_etherdev_mq(LIO_SIZE, max_num_queues);
3508 
3509 		if (!netdev) {
3510 			dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3511 			WRITE_ONCE(sc->caller_is_done, true);
3512 			goto setup_nic_dev_done;
3513 		}
3514 
3515 		SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
3516 
3517 		/* Associate the routines that will handle different
3518 		 * netdev tasks.
3519 		 */
3520 		netdev->netdev_ops = &lionetdevops;
3521 
3522 		retval = netif_set_real_num_rx_queues(netdev, num_oqueues);
3523 		if (retval) {
3524 			dev_err(&octeon_dev->pci_dev->dev,
3525 				"setting real number rx failed\n");
3526 			WRITE_ONCE(sc->caller_is_done, true);
3527 			goto setup_nic_dev_free;
3528 		}
3529 
3530 		retval = netif_set_real_num_tx_queues(netdev, num_iqueues);
3531 		if (retval) {
3532 			dev_err(&octeon_dev->pci_dev->dev,
3533 				"setting real number tx failed\n");
3534 			WRITE_ONCE(sc->caller_is_done, true);
3535 			goto setup_nic_dev_free;
3536 		}
3537 
3538 		lio = GET_LIO(netdev);
3539 
3540 		memset(lio, 0, sizeof(struct lio));
3541 
3542 		lio->ifidx = ifidx_or_pfnum;
3543 
3544 		props = &octeon_dev->props[i];
3545 		props->gmxport = resp->cfg_info.linfo.gmxport;
3546 		props->netdev = netdev;
3547 
3548 		lio->linfo.num_rxpciq = num_oqueues;
3549 		lio->linfo.num_txpciq = num_iqueues;
3550 		for (j = 0; j < num_oqueues; j++) {
3551 			lio->linfo.rxpciq[j].u64 =
3552 				resp->cfg_info.linfo.rxpciq[j].u64;
3553 		}
3554 		for (j = 0; j < num_iqueues; j++) {
3555 			lio->linfo.txpciq[j].u64 =
3556 				resp->cfg_info.linfo.txpciq[j].u64;
3557 		}
3558 		lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3559 		lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3560 		lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3561 
3562 		WRITE_ONCE(sc->caller_is_done, true);
3563 
3564 		lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3565 
3566 		if (OCTEON_CN23XX_PF(octeon_dev) ||
3567 		    OCTEON_CN6XXX(octeon_dev)) {
3568 			lio->dev_capability = NETIF_F_HIGHDMA
3569 					      | NETIF_F_IP_CSUM
3570 					      | NETIF_F_IPV6_CSUM
3571 					      | NETIF_F_SG | NETIF_F_RXCSUM
3572 					      | NETIF_F_GRO
3573 					      | NETIF_F_TSO | NETIF_F_TSO6
3574 					      | NETIF_F_LRO;
3575 		}
3576 		netif_set_tso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3577 
3578 		/*  Copy of transmit encapsulation capabilities:
3579 		 *  TSO, TSO6, Checksums for this device
3580 		 */
3581 		lio->enc_dev_capability = NETIF_F_IP_CSUM
3582 					  | NETIF_F_IPV6_CSUM
3583 					  | NETIF_F_GSO_UDP_TUNNEL
3584 					  | NETIF_F_HW_CSUM | NETIF_F_SG
3585 					  | NETIF_F_RXCSUM
3586 					  | NETIF_F_TSO | NETIF_F_TSO6
3587 					  | NETIF_F_LRO;
3588 
3589 		netdev->hw_enc_features = (lio->enc_dev_capability &
3590 					   ~NETIF_F_LRO);
3591 
3592 		netdev->udp_tunnel_nic_info = &liquidio_udp_tunnels;
3593 
3594 		lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;
3595 
3596 		netdev->vlan_features = lio->dev_capability;
3597 		/* Add any unchangeable hw features */
3598 		lio->dev_capability |=  NETIF_F_HW_VLAN_CTAG_FILTER |
3599 					NETIF_F_HW_VLAN_CTAG_RX |
3600 					NETIF_F_HW_VLAN_CTAG_TX;
3601 
3602 		netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
3603 
3604 		netdev->hw_features = lio->dev_capability;
3605 		/*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
3606 		netdev->hw_features = netdev->hw_features &
3607 			~NETIF_F_HW_VLAN_CTAG_RX;
3608 
3609 		/* MTU range: 68 - 16000 */
3610 		netdev->min_mtu = LIO_MIN_MTU_SIZE;
3611 		netdev->max_mtu = LIO_MAX_MTU_SIZE;
3612 
3613 		/* Point to the  properties for octeon device to which this
3614 		 * interface belongs.
3615 		 */
3616 		lio->oct_dev = octeon_dev;
3617 		lio->octprops = props;
3618 		lio->netdev = netdev;
3619 
3620 		dev_dbg(&octeon_dev->pci_dev->dev,
3621 			"if%d gmx: %d hw_addr: 0x%llx\n", i,
3622 			lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3623 
3624 		for (j = 0; j < octeon_dev->sriov_info.max_vfs; j++) {
3625 			u8 vfmac[ETH_ALEN];
3626 
3627 			eth_random_addr(vfmac);
3628 			if (__liquidio_set_vf_mac(netdev, j, vfmac, false)) {
3629 				dev_err(&octeon_dev->pci_dev->dev,
3630 					"Error setting VF%d MAC address\n",
3631 					j);
3632 				goto setup_nic_dev_free;
3633 			}
3634 		}
3635 
3636 		/* 64-bit swap required on LE machines */
3637 		octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3638 		for (j = 0; j < 6; j++)
3639 			mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3640 
3641 		/* Copy MAC Address to OS network device structure */
3642 
3643 		eth_hw_addr_set(netdev, mac);
3644 
3645 		/* By default all interfaces on a single Octeon uses the same
3646 		 * tx and rx queues
3647 		 */
3648 		lio->txq = lio->linfo.txpciq[0].s.q_no;
3649 		lio->rxq = lio->linfo.rxpciq[0].s.q_no;
3650 		if (liquidio_setup_io_queues(octeon_dev, i,
3651 					     lio->linfo.num_txpciq,
3652 					     lio->linfo.num_rxpciq)) {
3653 			dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3654 			goto setup_nic_dev_free;
3655 		}
3656 
3657 		ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3658 
3659 		lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3660 		lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3661 
3662 		if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {
3663 			dev_err(&octeon_dev->pci_dev->dev,
3664 				"Gather list allocation failed\n");
3665 			goto setup_nic_dev_free;
3666 		}
3667 
3668 		/* Register ethtool support */
3669 		liquidio_set_ethtool_ops(netdev);
3670 		if (lio->oct_dev->chip_id == OCTEON_CN23XX_PF_VID)
3671 			octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;
3672 		else
3673 			octeon_dev->priv_flags = 0x0;
3674 
3675 		if (netdev->features & NETIF_F_LRO)
3676 			liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3677 					     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3678 
3679 		liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
3680 				     OCTNET_CMD_VLAN_FILTER_ENABLE);
3681 
3682 		if ((debug != -1) && (debug & NETIF_MSG_HW))
3683 			liquidio_set_feature(netdev,
3684 					     OCTNET_CMD_VERBOSE_ENABLE, 0);
3685 
3686 		if (setup_link_status_change_wq(netdev))
3687 			goto setup_nic_dev_free;
3688 
3689 		if ((octeon_dev->fw_info.app_cap_flags &
3690 		     LIQUIDIO_TIME_SYNC_CAP) &&
3691 		    setup_sync_octeon_time_wq(netdev))
3692 			goto setup_nic_dev_free;
3693 
3694 		if (setup_rx_oom_poll_fn(netdev))
3695 			goto setup_nic_dev_free;
3696 
3697 		/* Register the network device with the OS */
3698 		if (register_netdev(netdev)) {
3699 			dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3700 			goto setup_nic_dev_free;
3701 		}
3702 
3703 		dev_dbg(&octeon_dev->pci_dev->dev,
3704 			"Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3705 			i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3706 		netif_carrier_off(netdev);
3707 		lio->link_changes++;
3708 
3709 		ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3710 
3711 		/* Sending command to firmware to enable Rx checksum offload
3712 		 * by default at the time of setup of Liquidio driver for
3713 		 * this device
3714 		 */
3715 		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3716 					    OCTNET_CMD_RXCSUM_ENABLE);
3717 		liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
3718 				     OCTNET_CMD_TXCSUM_ENABLE);
3719 
3720 		dev_dbg(&octeon_dev->pci_dev->dev,
3721 			"NIC ifidx:%d Setup successful\n", i);
3722 
3723 		if (octeon_dev->subsystem_id ==
3724 			OCTEON_CN2350_25GB_SUBSYS_ID ||
3725 		    octeon_dev->subsystem_id ==
3726 			OCTEON_CN2360_25GB_SUBSYS_ID) {
3727 			cur_ver = OCT_FW_VER(octeon_dev->fw_info.ver.maj,
3728 					     octeon_dev->fw_info.ver.min,
3729 					     octeon_dev->fw_info.ver.rev);
3730 
3731 			/* speed control unsupported in f/w older than 1.7.2 */
3732 			if (cur_ver < OCT_FW_VER(1, 7, 2)) {
3733 				dev_info(&octeon_dev->pci_dev->dev,
3734 					 "speed setting not supported by f/w.");
3735 				octeon_dev->speed_setting = 25;
3736 				octeon_dev->no_speed_setting = 1;
3737 			} else {
3738 				liquidio_get_speed(lio);
3739 			}
3740 
3741 			if (octeon_dev->speed_setting == 0) {
3742 				octeon_dev->speed_setting = 25;
3743 				octeon_dev->no_speed_setting = 1;
3744 			}
3745 		} else {
3746 			octeon_dev->no_speed_setting = 1;
3747 			octeon_dev->speed_setting = 10;
3748 		}
3749 		octeon_dev->speed_boot = octeon_dev->speed_setting;
3750 
3751 		/* don't read FEC setting if unsupported by f/w (see above) */
3752 		if (octeon_dev->speed_boot == 25 &&
3753 		    !octeon_dev->no_speed_setting) {
3754 			liquidio_get_fec(lio);
3755 			octeon_dev->props[lio->ifidx].fec_boot =
3756 				octeon_dev->props[lio->ifidx].fec;
3757 		}
3758 	}
3759 
3760 	device_lock(&octeon_dev->pci_dev->dev);
3761 	devlink = devlink_alloc(&liquidio_devlink_ops,
3762 				sizeof(struct lio_devlink_priv),
3763 				&octeon_dev->pci_dev->dev);
3764 	if (!devlink) {
3765 		device_unlock(&octeon_dev->pci_dev->dev);
3766 		dev_err(&octeon_dev->pci_dev->dev, "devlink alloc failed\n");
3767 		goto setup_nic_dev_free;
3768 	}
3769 
3770 	lio_devlink = devlink_priv(devlink);
3771 	lio_devlink->oct = octeon_dev;
3772 
3773 	octeon_dev->devlink = devlink;
3774 	octeon_dev->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
3775 	devlink_register(devlink);
3776 	device_unlock(&octeon_dev->pci_dev->dev);
3777 
3778 	return 0;
3779 
3780 setup_nic_dev_free:
3781 
3782 	while (i--) {
3783 		dev_err(&octeon_dev->pci_dev->dev,
3784 			"NIC ifidx:%d Setup failed\n", i);
3785 		liquidio_destroy_nic_device(octeon_dev, i);
3786 	}
3787 
3788 setup_nic_dev_done:
3789 
3790 	return -ENODEV;
3791 }
3792 
3793 #ifdef CONFIG_PCI_IOV
3794 static int octeon_enable_sriov(struct octeon_device *oct)
3795 {
3796 	unsigned int num_vfs_alloced = oct->sriov_info.num_vfs_alloced;
3797 	struct pci_dev *vfdev;
3798 	int err;
3799 	u32 u;
3800 
3801 	if (OCTEON_CN23XX_PF(oct) && num_vfs_alloced) {
3802 		err = pci_enable_sriov(oct->pci_dev,
3803 				       oct->sriov_info.num_vfs_alloced);
3804 		if (err) {
3805 			dev_err(&oct->pci_dev->dev,
3806 				"OCTEON: Failed to enable PCI sriov: %d\n",
3807 				err);
3808 			oct->sriov_info.num_vfs_alloced = 0;
3809 			return err;
3810 		}
3811 		oct->sriov_info.sriov_enabled = 1;
3812 
3813 		/* init lookup table that maps DPI ring number to VF pci_dev
3814 		 * struct pointer
3815 		 */
3816 		u = 0;
3817 		vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3818 				       OCTEON_CN23XX_VF_VID, NULL);
3819 		while (vfdev) {
3820 			if (vfdev->is_virtfn &&
3821 			    (vfdev->physfn == oct->pci_dev)) {
3822 				oct->sriov_info.dpiring_to_vfpcidev_lut[u] =
3823 					vfdev;
3824 				u += oct->sriov_info.rings_per_vf;
3825 			}
3826 			vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3827 					       OCTEON_CN23XX_VF_VID, vfdev);
3828 		}
3829 	}
3830 
3831 	return num_vfs_alloced;
3832 }
3833 
3834 static int lio_pci_sriov_disable(struct octeon_device *oct)
3835 {
3836 	int u;
3837 
3838 	if (pci_vfs_assigned(oct->pci_dev)) {
3839 		dev_err(&oct->pci_dev->dev, "VFs are still assigned to VMs.\n");
3840 		return -EPERM;
3841 	}
3842 
3843 	pci_disable_sriov(oct->pci_dev);
3844 
3845 	u = 0;
3846 	while (u < MAX_POSSIBLE_VFS) {
3847 		oct->sriov_info.dpiring_to_vfpcidev_lut[u] = NULL;
3848 		u += oct->sriov_info.rings_per_vf;
3849 	}
3850 
3851 	oct->sriov_info.num_vfs_alloced = 0;
3852 	dev_info(&oct->pci_dev->dev, "oct->pf_num:%d disabled VFs\n",
3853 		 oct->pf_num);
3854 
3855 	return 0;
3856 }
3857 
3858 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs)
3859 {
3860 	struct octeon_device *oct = pci_get_drvdata(dev);
3861 	int ret = 0;
3862 
3863 	if ((num_vfs == oct->sriov_info.num_vfs_alloced) &&
3864 	    (oct->sriov_info.sriov_enabled)) {
3865 		dev_info(&oct->pci_dev->dev, "oct->pf_num:%d already enabled num_vfs:%d\n",
3866 			 oct->pf_num, num_vfs);
3867 		return 0;
3868 	}
3869 
3870 	if (!num_vfs) {
3871 		lio_vf_rep_destroy(oct);
3872 		ret = lio_pci_sriov_disable(oct);
3873 	} else if (num_vfs > oct->sriov_info.max_vfs) {
3874 		dev_err(&oct->pci_dev->dev,
3875 			"OCTEON: Max allowed VFs:%d user requested:%d",
3876 			oct->sriov_info.max_vfs, num_vfs);
3877 		ret = -EPERM;
3878 	} else {
3879 		oct->sriov_info.num_vfs_alloced = num_vfs;
3880 		ret = octeon_enable_sriov(oct);
3881 		dev_info(&oct->pci_dev->dev, "oct->pf_num:%d num_vfs:%d\n",
3882 			 oct->pf_num, num_vfs);
3883 		ret = lio_vf_rep_create(oct);
3884 		if (ret)
3885 			dev_info(&oct->pci_dev->dev,
3886 				 "vf representor create failed");
3887 	}
3888 
3889 	return ret;
3890 }
3891 #endif
3892 
3893 /**
3894  * liquidio_init_nic_module - initialize the NIC
3895  * @oct: octeon device
3896  *
3897  * This initialization routine is called once the Octeon device application is
3898  * up and running
3899  */
3900 static int liquidio_init_nic_module(struct octeon_device *oct)
3901 {
3902 	int i, retval = 0;
3903 	int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
3904 
3905 	dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
3906 
3907 	/* only default iq and oq were initialized
3908 	 * initialize the rest as well
3909 	 */
3910 	/* run port_config command for each port */
3911 	oct->ifcount = num_nic_ports;
3912 
3913 	memset(oct->props, 0, sizeof(struct octdev_props) * num_nic_ports);
3914 
3915 	for (i = 0; i < MAX_OCTEON_LINKS; i++)
3916 		oct->props[i].gmxport = -1;
3917 
3918 	retval = setup_nic_devices(oct);
3919 	if (retval) {
3920 		dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
3921 		goto octnet_init_failure;
3922 	}
3923 
3924 	/* Call vf_rep_modinit if the firmware is switchdev capable
3925 	 * and do it from the first liquidio function probed.
3926 	 */
3927 	if (!oct->octeon_id &&
3928 	    oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP) {
3929 		retval = lio_vf_rep_modinit();
3930 		if (retval) {
3931 			liquidio_stop_nic_module(oct);
3932 			goto octnet_init_failure;
3933 		}
3934 	}
3935 
3936 	liquidio_ptp_init(oct);
3937 
3938 	dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
3939 
3940 	return retval;
3941 
3942 octnet_init_failure:
3943 
3944 	oct->ifcount = 0;
3945 
3946 	return retval;
3947 }
3948 
3949 /**
3950  * nic_starter - finish init
3951  * @work:  work struct work_struct
3952  *
3953  * starter callback that invokes the remaining initialization work after the NIC is up and running.
3954  */
3955 static void nic_starter(struct work_struct *work)
3956 {
3957 	struct octeon_device *oct;
3958 	struct cavium_wk *wk = (struct cavium_wk *)work;
3959 
3960 	oct = (struct octeon_device *)wk->ctxptr;
3961 
3962 	if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
3963 		return;
3964 
3965 	/* If the status of the device is CORE_OK, the core
3966 	 * application has reported its application type. Call
3967 	 * any registered handlers now and move to the RUNNING
3968 	 * state.
3969 	 */
3970 	if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
3971 		schedule_delayed_work(&oct->nic_poll_work.work,
3972 				      LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3973 		return;
3974 	}
3975 
3976 	atomic_set(&oct->status, OCT_DEV_RUNNING);
3977 
3978 	if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
3979 		dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
3980 
3981 		if (liquidio_init_nic_module(oct))
3982 			dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
3983 		else
3984 			handshake[oct->octeon_id].started_ok = 1;
3985 	} else {
3986 		dev_err(&oct->pci_dev->dev,
3987 			"Unexpected application running on NIC (%d). Check firmware.\n",
3988 			oct->app_mode);
3989 	}
3990 
3991 	complete(&handshake[oct->octeon_id].started);
3992 }
3993 
3994 static int
3995 octeon_recv_vf_drv_notice(struct octeon_recv_info *recv_info, void *buf)
3996 {
3997 	struct octeon_device *oct = (struct octeon_device *)buf;
3998 	struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3999 	int i, notice, vf_idx;
4000 	bool cores_crashed;
4001 	u64 *data, vf_num;
4002 
4003 	notice = recv_pkt->rh.r.ossp;
4004 	data = (u64 *)(get_rbd(recv_pkt->buffer_ptr[0]) + OCT_DROQ_INFO_SIZE);
4005 
4006 	/* the first 64-bit word of data is the vf_num */
4007 	vf_num = data[0];
4008 	octeon_swap_8B_data(&vf_num, 1);
4009 	vf_idx = (int)vf_num - 1;
4010 
4011 	cores_crashed = READ_ONCE(oct->cores_crashed);
4012 
4013 	if (notice == VF_DRV_LOADED) {
4014 		if (!(oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx))) {
4015 			oct->sriov_info.vf_drv_loaded_mask |= BIT_ULL(vf_idx);
4016 			dev_info(&oct->pci_dev->dev,
4017 				 "driver for VF%d was loaded\n", vf_idx);
4018 			if (!cores_crashed)
4019 				try_module_get(THIS_MODULE);
4020 		}
4021 	} else if (notice == VF_DRV_REMOVED) {
4022 		if (oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx)) {
4023 			oct->sriov_info.vf_drv_loaded_mask &= ~BIT_ULL(vf_idx);
4024 			dev_info(&oct->pci_dev->dev,
4025 				 "driver for VF%d was removed\n", vf_idx);
4026 			if (!cores_crashed)
4027 				module_put(THIS_MODULE);
4028 		}
4029 	} else if (notice == VF_DRV_MACADDR_CHANGED) {
4030 		u8 *b = (u8 *)&data[1];
4031 
4032 		oct->sriov_info.vf_macaddr[vf_idx] = data[1];
4033 		dev_info(&oct->pci_dev->dev,
4034 			 "VF driver changed VF%d's MAC address to %pM\n",
4035 			 vf_idx, b + 2);
4036 	}
4037 
4038 	for (i = 0; i < recv_pkt->buffer_count; i++)
4039 		recv_buffer_free(recv_pkt->buffer_ptr[i]);
4040 	octeon_free_recv_info(recv_info);
4041 
4042 	return 0;
4043 }
4044 
4045 /**
4046  * octeon_device_init - Device initialization for each Octeon device that is probed
4047  * @octeon_dev:  octeon device
4048  */
4049 static int octeon_device_init(struct octeon_device *octeon_dev)
4050 {
4051 	int j, ret;
4052 	char bootcmd[] = "\n";
4053 	char *dbg_enb = NULL;
4054 	enum lio_fw_state fw_state;
4055 	struct octeon_device_priv *oct_priv = octeon_dev->priv;
4056 	atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
4057 
4058 	/* Enable access to the octeon device and make its DMA capability
4059 	 * known to the OS.
4060 	 */
4061 	if (octeon_pci_os_setup(octeon_dev))
4062 		return 1;
4063 
4064 	atomic_set(&octeon_dev->status, OCT_DEV_PCI_ENABLE_DONE);
4065 
4066 	/* Identify the Octeon type and map the BAR address space. */
4067 	if (octeon_chip_specific_setup(octeon_dev)) {
4068 		dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
4069 		return 1;
4070 	}
4071 
4072 	atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
4073 
4074 	/* Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE',
4075 	 * since that is what is required for the reference to be removed
4076 	 * during de-initialization (see 'octeon_destroy_resources').
4077 	 */
4078 	octeon_register_device(octeon_dev, octeon_dev->pci_dev->bus->number,
4079 			       PCI_SLOT(octeon_dev->pci_dev->devfn),
4080 			       PCI_FUNC(octeon_dev->pci_dev->devfn),
4081 			       true);
4082 
4083 	octeon_dev->app_mode = CVM_DRV_INVALID_APP;
4084 
4085 	/* CN23XX supports preloaded firmware if the following is true:
4086 	 *
4087 	 * The adapter indicates that firmware is currently running AND
4088 	 * 'fw_type' is 'auto'.
4089 	 *
4090 	 * (default state is NEEDS_TO_BE_LOADED, override it if appropriate).
4091 	 */
4092 	if (OCTEON_CN23XX_PF(octeon_dev) &&
4093 	    cn23xx_fw_loaded(octeon_dev) && fw_type_is_auto()) {
4094 		atomic_cmpxchg(octeon_dev->adapter_fw_state,
4095 			       FW_NEEDS_TO_BE_LOADED, FW_IS_PRELOADED);
4096 	}
4097 
4098 	/* If loading firmware, only first device of adapter needs to do so. */
4099 	fw_state = atomic_cmpxchg(octeon_dev->adapter_fw_state,
4100 				  FW_NEEDS_TO_BE_LOADED,
4101 				  FW_IS_BEING_LOADED);
4102 
4103 	/* Here, [local variable] 'fw_state' is set to one of:
4104 	 *
4105 	 *   FW_IS_PRELOADED:       No firmware is to be loaded (see above)
4106 	 *   FW_NEEDS_TO_BE_LOADED: The driver's first instance will load
4107 	 *                          firmware to the adapter.
4108 	 *   FW_IS_BEING_LOADED:    The driver's second instance will not load
4109 	 *                          firmware to the adapter.
4110 	 */
4111 
4112 	/* Prior to f/w load, perform a soft reset of the Octeon device;
4113 	 * if error resetting, return w/error.
4114 	 */
4115 	if (fw_state == FW_NEEDS_TO_BE_LOADED)
4116 		if (octeon_dev->fn_list.soft_reset(octeon_dev))
4117 			return 1;
4118 
4119 	/* Initialize the dispatch mechanism used to push packets arriving on
4120 	 * Octeon Output queues.
4121 	 */
4122 	if (octeon_init_dispatch_list(octeon_dev))
4123 		return 1;
4124 
4125 	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4126 				    OPCODE_NIC_CORE_DRV_ACTIVE,
4127 				    octeon_core_drv_init,
4128 				    octeon_dev);
4129 
4130 	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4131 				    OPCODE_NIC_VF_DRV_NOTICE,
4132 				    octeon_recv_vf_drv_notice, octeon_dev);
4133 	INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
4134 	octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
4135 	schedule_delayed_work(&octeon_dev->nic_poll_work.work,
4136 			      LIQUIDIO_STARTER_POLL_INTERVAL_MS);
4137 
4138 	atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
4139 
4140 	if (octeon_set_io_queues_off(octeon_dev)) {
4141 		dev_err(&octeon_dev->pci_dev->dev, "setting io queues off failed\n");
4142 		return 1;
4143 	}
4144 
4145 	if (OCTEON_CN23XX_PF(octeon_dev)) {
4146 		ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4147 		if (ret) {
4148 			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Failed to configure device registers\n");
4149 			return ret;
4150 		}
4151 	}
4152 
4153 	/* Initialize soft command buffer pool
4154 	 */
4155 	if (octeon_setup_sc_buffer_pool(octeon_dev)) {
4156 		dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
4157 		return 1;
4158 	}
4159 	atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
4160 
4161 	/*  Setup the data structures that manage this Octeon's Input queues. */
4162 	if (octeon_setup_instr_queues(octeon_dev)) {
4163 		dev_err(&octeon_dev->pci_dev->dev,
4164 			"instruction queue initialization failed\n");
4165 		return 1;
4166 	}
4167 	atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
4168 
4169 	/* Initialize lists to manage the requests of different types that
4170 	 * arrive from user & kernel applications for this octeon device.
4171 	 */
4172 	if (octeon_setup_response_list(octeon_dev)) {
4173 		dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
4174 		return 1;
4175 	}
4176 	atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
4177 
4178 	if (octeon_setup_output_queues(octeon_dev)) {
4179 		dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
4180 		return 1;
4181 	}
4182 
4183 	atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
4184 
4185 	if (OCTEON_CN23XX_PF(octeon_dev)) {
4186 		if (octeon_dev->fn_list.setup_mbox(octeon_dev)) {
4187 			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Mailbox setup failed\n");
4188 			return 1;
4189 		}
4190 		atomic_set(&octeon_dev->status, OCT_DEV_MBOX_SETUP_DONE);
4191 
4192 		if (octeon_allocate_ioq_vector
4193 				(octeon_dev,
4194 				 octeon_dev->sriov_info.num_pf_rings)) {
4195 			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");
4196 			return 1;
4197 		}
4198 		atomic_set(&octeon_dev->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE);
4199 
4200 	} else {
4201 		/* The input and output queue registers were setup earlier (the
4202 		 * queues were not enabled). Any additional registers
4203 		 * that need to be programmed should be done now.
4204 		 */
4205 		ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4206 		if (ret) {
4207 			dev_err(&octeon_dev->pci_dev->dev,
4208 				"Failed to configure device registers\n");
4209 			return ret;
4210 		}
4211 	}
4212 
4213 	/* Initialize the tasklet that handles output queue packet processing.*/
4214 	dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
4215 	tasklet_setup(&oct_priv->droq_tasklet, octeon_droq_bh);
4216 
4217 	/* Setup the interrupt handler and record the INT SUM register address
4218 	 */
4219 	if (octeon_setup_interrupt(octeon_dev,
4220 				   octeon_dev->sriov_info.num_pf_rings))
4221 		return 1;
4222 
4223 	/* Enable Octeon device interrupts */
4224 	octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);
4225 
4226 	atomic_set(&octeon_dev->status, OCT_DEV_INTR_SET_DONE);
4227 
4228 	/* Send Credit for Octeon Output queues. Credits are always sent BEFORE
4229 	 * the output queue is enabled.
4230 	 * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in
4231 	 * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0.
4232 	 * Otherwise, it is possible that the DRV_ACTIVE message will be sent
4233 	 * before any credits have been issued, causing the ring to be reset
4234 	 * (and the f/w appear to never have started).
4235 	 */
4236 	for (j = 0; j < octeon_dev->num_oqs; j++)
4237 		writel(octeon_dev->droq[j]->max_count,
4238 		       octeon_dev->droq[j]->pkts_credit_reg);
4239 
4240 	/* Enable the input and output queues for this Octeon device */
4241 	ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
4242 	if (ret) {
4243 		dev_err(&octeon_dev->pci_dev->dev, "Failed to enable input/output queues");
4244 		return ret;
4245 	}
4246 
4247 	atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
4248 
4249 	if (fw_state == FW_NEEDS_TO_BE_LOADED) {
4250 		dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
4251 		if (!ddr_timeout) {
4252 			dev_info(&octeon_dev->pci_dev->dev,
4253 				 "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
4254 		}
4255 
4256 		schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
4257 
4258 		/* Wait for the octeon to initialize DDR after the soft-reset.*/
4259 		while (!ddr_timeout) {
4260 			set_current_state(TASK_INTERRUPTIBLE);
4261 			if (schedule_timeout(HZ / 10)) {
4262 				/* user probably pressed Control-C */
4263 				return 1;
4264 			}
4265 		}
4266 		ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
4267 		if (ret) {
4268 			dev_err(&octeon_dev->pci_dev->dev,
4269 				"DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
4270 				ret);
4271 			return 1;
4272 		}
4273 
4274 		if (octeon_wait_for_bootloader(octeon_dev, 1000)) {
4275 			dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
4276 			return 1;
4277 		}
4278 
4279 		/* Divert uboot to take commands from host instead. */
4280 		ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
4281 
4282 		dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
4283 		ret = octeon_init_consoles(octeon_dev);
4284 		if (ret) {
4285 			dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
4286 			return 1;
4287 		}
4288 		/* If console debug enabled, specify empty string to use default
4289 		 * enablement ELSE specify NULL string for 'disabled'.
4290 		 */
4291 		dbg_enb = octeon_console_debug_enabled(0) ? "" : NULL;
4292 		ret = octeon_add_console(octeon_dev, 0, dbg_enb);
4293 		if (ret) {
4294 			dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
4295 			return 1;
4296 		} else if (octeon_console_debug_enabled(0)) {
4297 			/* If console was added AND we're logging console output
4298 			 * then set our console print function.
4299 			 */
4300 			octeon_dev->console[0].print = octeon_dbg_console_print;
4301 		}
4302 
4303 		atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
4304 
4305 		dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
4306 		ret = load_firmware(octeon_dev);
4307 		if (ret) {
4308 			dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
4309 			return 1;
4310 		}
4311 
4312 		atomic_set(octeon_dev->adapter_fw_state, FW_HAS_BEEN_LOADED);
4313 	}
4314 
4315 	handshake[octeon_dev->octeon_id].init_ok = 1;
4316 	complete(&handshake[octeon_dev->octeon_id].init);
4317 
4318 	atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
4319 	oct_priv->dev = octeon_dev;
4320 
4321 	return 0;
4322 }
4323 
4324 /**
4325  * octeon_dbg_console_print - Debug console print function
4326  * @oct:  octeon device
4327  * @console_num: console number
4328  * @prefix:      first portion of line to display
4329  * @suffix:      second portion of line to display
4330  *
4331  * The OCTEON debug console outputs entire lines (excluding '\n').
4332  * Normally, the line will be passed in the 'prefix' parameter.
4333  * However, due to buffering, it is possible for a line to be split into two
4334  * parts, in which case they will be passed as the 'prefix' parameter and
4335  * 'suffix' parameter.
4336  */
4337 static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
4338 				    char *prefix, char *suffix)
4339 {
4340 	if (prefix && suffix)
4341 		dev_info(&oct->pci_dev->dev, "%u: %s%s\n", console_num, prefix,
4342 			 suffix);
4343 	else if (prefix)
4344 		dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, prefix);
4345 	else if (suffix)
4346 		dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, suffix);
4347 
4348 	return 0;
4349 }
4350 
4351 /**
4352  * liquidio_exit - Exits the module
4353  */
4354 static void __exit liquidio_exit(void)
4355 {
4356 	liquidio_deinit_pci();
4357 
4358 	pr_info("LiquidIO network module is now unloaded\n");
4359 }
4360 
4361 module_init(liquidio_init);
4362 module_exit(liquidio_exit);
4363