1 /**********************************************************************
2 * Author: Cavium, Inc.
3 *
4 * Contact: support@cavium.com
5 *          Please include "LiquidIO" in the subject.
6 *
7 * Copyright (c) 2003-2015 Cavium, Inc.
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT.  See the GNU General Public License for more
17 * details.
18 *
19 * This file may also be available under a different license from Cavium.
20 * Contact Cavium, Inc. for more information
21 **********************************************************************/
22 #include <linux/version.h>
23 #include <linux/pci.h>
24 #include <linux/firmware.h>
25 #include <linux/ptp_clock_kernel.h>
26 #include <net/vxlan.h>
27 #include <linux/kthread.h>
28 #include "liquidio_common.h"
29 #include "octeon_droq.h"
30 #include "octeon_iq.h"
31 #include "response_manager.h"
32 #include "octeon_device.h"
33 #include "octeon_nic.h"
34 #include "octeon_main.h"
35 #include "octeon_network.h"
36 #include "cn66xx_regs.h"
37 #include "cn66xx_device.h"
38 #include "cn68xx_device.h"
39 #include "cn23xx_pf_device.h"
40 #include "liquidio_image.h"
41 
42 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
43 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
44 MODULE_LICENSE("GPL");
45 MODULE_VERSION(LIQUIDIO_VERSION);
46 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME LIO_FW_NAME_SUFFIX);
47 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME LIO_FW_NAME_SUFFIX);
48 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME LIO_FW_NAME_SUFFIX);
49 
50 static int ddr_timeout = 10000;
51 module_param(ddr_timeout, int, 0644);
52 MODULE_PARM_DESC(ddr_timeout,
53 		 "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
54 
55 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
56 
57 #define INCR_INSTRQUEUE_PKT_COUNT(octeon_dev_ptr, iq_no, field, count)  \
58 	(octeon_dev_ptr->instr_queue[iq_no]->stats.field += count)
59 
60 static int debug = -1;
61 module_param(debug, int, 0644);
62 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
63 
64 static char fw_type[LIO_MAX_FW_TYPE_LEN];
65 module_param_string(fw_type, fw_type, sizeof(fw_type), 0000);
66 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\"");
67 
68 static int conf_type;
69 module_param(conf_type, int, 0);
70 MODULE_PARM_DESC(conf_type, "select octeon configuration 0 default 1 ovs");
71 
72 static int ptp_enable = 1;
73 
74 /* Bit mask values for lio->ifstate */
75 #define   LIO_IFSTATE_DROQ_OPS             0x01
76 #define   LIO_IFSTATE_REGISTERED           0x02
77 #define   LIO_IFSTATE_RUNNING              0x04
78 #define   LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08
79 
80 /* Polling interval for determining when NIC application is alive */
81 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
82 
83 /* runtime link query interval */
84 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS         1000
85 
86 struct liquidio_if_cfg_context {
87 	int octeon_id;
88 
89 	wait_queue_head_t wc;
90 
91 	int cond;
92 };
93 
94 struct liquidio_if_cfg_resp {
95 	u64 rh;
96 	struct liquidio_if_cfg_info cfg_info;
97 	u64 status;
98 };
99 
100 struct liquidio_rx_ctl_context {
101 	int octeon_id;
102 
103 	wait_queue_head_t wc;
104 
105 	int cond;
106 };
107 
108 struct oct_link_status_resp {
109 	u64 rh;
110 	struct oct_link_info link_info;
111 	u64 status;
112 };
113 
114 struct oct_timestamp_resp {
115 	u64 rh;
116 	u64 timestamp;
117 	u64 status;
118 };
119 
120 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
121 
122 union tx_info {
123 	u64 u64;
124 	struct {
125 #ifdef __BIG_ENDIAN_BITFIELD
126 		u16 gso_size;
127 		u16 gso_segs;
128 		u32 reserved;
129 #else
130 		u32 reserved;
131 		u16 gso_segs;
132 		u16 gso_size;
133 #endif
134 	} s;
135 };
136 
137 /** Octeon device properties to be used by the NIC module.
138  * Each octeon device in the system will be represented
139  * by this structure in the NIC module.
140  */
141 
142 #define OCTNIC_MAX_SG  (MAX_SKB_FRAGS)
143 
144 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
145 #define OCTNIC_GSO_MAX_SIZE                                                    \
146 	(CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
147 
148 /** Structure of a node in list of gather components maintained by
149  * NIC driver for each network device.
150  */
151 struct octnic_gather {
152 	/** List manipulation. Next and prev pointers. */
153 	struct list_head list;
154 
155 	/** Size of the gather component at sg in bytes. */
156 	int sg_size;
157 
158 	/** Number of bytes that sg was adjusted to make it 8B-aligned. */
159 	int adjust;
160 
161 	/** Gather component that can accommodate max sized fragment list
162 	 *  received from the IP layer.
163 	 */
164 	struct octeon_sg_entry *sg;
165 
166 	u64 sg_dma_ptr;
167 };
168 
169 struct handshake {
170 	struct completion init;
171 	struct completion started;
172 	struct pci_dev *pci_dev;
173 	int init_ok;
174 	int started_ok;
175 };
176 
177 struct octeon_device_priv {
178 	/** Tasklet structures for this device. */
179 	struct tasklet_struct droq_tasklet;
180 	unsigned long napi_mask;
181 };
182 
183 static int octeon_device_init(struct octeon_device *);
184 static int liquidio_stop(struct net_device *netdev);
185 static void liquidio_remove(struct pci_dev *pdev);
186 static int liquidio_probe(struct pci_dev *pdev,
187 			  const struct pci_device_id *ent);
188 
189 static struct handshake handshake[MAX_OCTEON_DEVICES];
190 static struct completion first_stage;
191 
192 static void octeon_droq_bh(unsigned long pdev)
193 {
194 	int q_no;
195 	int reschedule = 0;
196 	struct octeon_device *oct = (struct octeon_device *)pdev;
197 	struct octeon_device_priv *oct_priv =
198 		(struct octeon_device_priv *)oct->priv;
199 
200 	/* for (q_no = 0; q_no < oct->num_oqs; q_no++) { */
201 	for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
202 		if (!(oct->io_qmask.oq & (1ULL << q_no)))
203 			continue;
204 		reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
205 							  MAX_PACKET_BUDGET);
206 		lio_enable_irq(oct->droq[q_no], NULL);
207 
208 		if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
209 			/* set time and cnt interrupt thresholds for this DROQ
210 			 * for NAPI
211 			 */
212 			int adjusted_q_no = q_no + oct->sriov_info.pf_srn;
213 
214 			octeon_write_csr64(
215 			    oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(adjusted_q_no),
216 			    0x5700000040ULL);
217 			octeon_write_csr64(
218 			    oct, CN23XX_SLI_OQ_PKTS_SENT(adjusted_q_no), 0);
219 		}
220 	}
221 
222 	if (reschedule)
223 		tasklet_schedule(&oct_priv->droq_tasklet);
224 }
225 
226 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
227 {
228 	struct octeon_device_priv *oct_priv =
229 		(struct octeon_device_priv *)oct->priv;
230 	int retry = 100, pkt_cnt = 0, pending_pkts = 0;
231 	int i;
232 
233 	do {
234 		pending_pkts = 0;
235 
236 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
237 			if (!(oct->io_qmask.oq & (1ULL << i)))
238 				continue;
239 			pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
240 		}
241 		if (pkt_cnt > 0) {
242 			pending_pkts += pkt_cnt;
243 			tasklet_schedule(&oct_priv->droq_tasklet);
244 		}
245 		pkt_cnt = 0;
246 		schedule_timeout_uninterruptible(1);
247 
248 	} while (retry-- && pending_pkts);
249 
250 	return pkt_cnt;
251 }
252 
253 /**
254  * \brief Forces all IO queues off on a given device
255  * @param oct Pointer to Octeon device
256  */
257 static void force_io_queues_off(struct octeon_device *oct)
258 {
259 	if ((oct->chip_id == OCTEON_CN66XX) ||
260 	    (oct->chip_id == OCTEON_CN68XX)) {
261 		/* Reset the Enable bits for Input Queues. */
262 		octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
263 
264 		/* Reset the Enable bits for Output Queues. */
265 		octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
266 	}
267 }
268 
269 /**
270  * \brief wait for all pending requests to complete
271  * @param oct Pointer to Octeon device
272  *
273  * Called during shutdown sequence
274  */
275 static int wait_for_pending_requests(struct octeon_device *oct)
276 {
277 	int i, pcount = 0;
278 
279 	for (i = 0; i < 100; i++) {
280 		pcount =
281 			atomic_read(&oct->response_list
282 				[OCTEON_ORDERED_SC_LIST].pending_req_count);
283 		if (pcount)
284 			schedule_timeout_uninterruptible(HZ / 10);
285 		else
286 			break;
287 	}
288 
289 	if (pcount)
290 		return 1;
291 
292 	return 0;
293 }
294 
295 /**
296  * \brief Cause device to go quiet so it can be safely removed/reset/etc
297  * @param oct Pointer to Octeon device
298  */
299 static inline void pcierror_quiesce_device(struct octeon_device *oct)
300 {
301 	int i;
302 
303 	/* Disable the input and output queues now. No more packets will
304 	 * arrive from Octeon, but we should wait for all packet processing
305 	 * to finish.
306 	 */
307 	force_io_queues_off(oct);
308 
309 	/* To allow for in-flight requests */
310 	schedule_timeout_uninterruptible(100);
311 
312 	if (wait_for_pending_requests(oct))
313 		dev_err(&oct->pci_dev->dev, "There were pending requests\n");
314 
315 	/* Force all requests waiting to be fetched by OCTEON to complete. */
316 	for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
317 		struct octeon_instr_queue *iq;
318 
319 		if (!(oct->io_qmask.iq & (1ULL << i)))
320 			continue;
321 		iq = oct->instr_queue[i];
322 
323 		if (atomic_read(&iq->instr_pending)) {
324 			spin_lock_bh(&iq->lock);
325 			iq->fill_cnt = 0;
326 			iq->octeon_read_index = iq->host_write_index;
327 			iq->stats.instr_processed +=
328 				atomic_read(&iq->instr_pending);
329 			lio_process_iq_request_list(oct, iq, 0);
330 			spin_unlock_bh(&iq->lock);
331 		}
332 	}
333 
334 	/* Force all pending ordered list requests to time out. */
335 	lio_process_ordered_list(oct, 1);
336 
337 	/* We do not need to wait for output queue packets to be processed. */
338 }
339 
340 /**
341  * \brief Cleanup PCI AER uncorrectable error status
342  * @param dev Pointer to PCI device
343  */
344 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
345 {
346 	int pos = 0x100;
347 	u32 status, mask;
348 
349 	pr_info("%s :\n", __func__);
350 
351 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
352 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
353 	if (dev->error_state == pci_channel_io_normal)
354 		status &= ~mask;        /* Clear corresponding nonfatal bits */
355 	else
356 		status &= mask;         /* Clear corresponding fatal bits */
357 	pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
358 }
359 
360 /**
361  * \brief Stop all PCI IO to a given device
362  * @param dev Pointer to Octeon device
363  */
364 static void stop_pci_io(struct octeon_device *oct)
365 {
366 	/* No more instructions will be forwarded. */
367 	atomic_set(&oct->status, OCT_DEV_IN_RESET);
368 
369 	pci_disable_device(oct->pci_dev);
370 
371 	/* Disable interrupts  */
372 	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
373 
374 	pcierror_quiesce_device(oct);
375 
376 	/* Release the interrupt line */
377 	free_irq(oct->pci_dev->irq, oct);
378 
379 	if (oct->flags & LIO_FLAG_MSI_ENABLED)
380 		pci_disable_msi(oct->pci_dev);
381 
382 	dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
383 		lio_get_state_string(&oct->status));
384 
385 	/* cn63xx_cleanup_aer_uncorrect_error_status(oct->pci_dev); */
386 	/* making it a common function for all OCTEON models */
387 	cleanup_aer_uncorrect_error_status(oct->pci_dev);
388 }
389 
390 /**
391  * \brief called when PCI error is detected
392  * @param pdev Pointer to PCI device
393  * @param state The current pci connection state
394  *
395  * This function is called after a PCI bus error affecting
396  * this device has been detected.
397  */
398 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
399 						     pci_channel_state_t state)
400 {
401 	struct octeon_device *oct = pci_get_drvdata(pdev);
402 
403 	/* Non-correctable Non-fatal errors */
404 	if (state == pci_channel_io_normal) {
405 		dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
406 		cleanup_aer_uncorrect_error_status(oct->pci_dev);
407 		return PCI_ERS_RESULT_CAN_RECOVER;
408 	}
409 
410 	/* Non-correctable Fatal errors */
411 	dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
412 	stop_pci_io(oct);
413 
414 	/* Always return a DISCONNECT. There is no support for recovery but only
415 	 * for a clean shutdown.
416 	 */
417 	return PCI_ERS_RESULT_DISCONNECT;
418 }
419 
420 /**
421  * \brief mmio handler
422  * @param pdev Pointer to PCI device
423  */
424 static pci_ers_result_t liquidio_pcie_mmio_enabled(
425 				struct pci_dev *pdev __attribute__((unused)))
426 {
427 	/* We should never hit this since we never ask for a reset for a Fatal
428 	 * Error. We always return DISCONNECT in io_error above.
429 	 * But play safe and return RECOVERED for now.
430 	 */
431 	return PCI_ERS_RESULT_RECOVERED;
432 }
433 
434 /**
435  * \brief called after the pci bus has been reset.
436  * @param pdev Pointer to PCI device
437  *
438  * Restart the card from scratch, as if from a cold-boot. Implementation
439  * resembles the first-half of the octeon_resume routine.
440  */
441 static pci_ers_result_t liquidio_pcie_slot_reset(
442 				struct pci_dev *pdev __attribute__((unused)))
443 {
444 	/* We should never hit this since we never ask for a reset for a Fatal
445 	 * Error. We always return DISCONNECT in io_error above.
446 	 * But play safe and return RECOVERED for now.
447 	 */
448 	return PCI_ERS_RESULT_RECOVERED;
449 }
450 
451 /**
452  * \brief called when traffic can start flowing again.
453  * @param pdev Pointer to PCI device
454  *
455  * This callback is called when the error recovery driver tells us that
456  * its OK to resume normal operation. Implementation resembles the
457  * second-half of the octeon_resume routine.
458  */
459 static void liquidio_pcie_resume(struct pci_dev *pdev __attribute__((unused)))
460 {
461 	/* Nothing to be done here. */
462 }
463 
464 #ifdef CONFIG_PM
465 /**
466  * \brief called when suspending
467  * @param pdev Pointer to PCI device
468  * @param state state to suspend to
469  */
470 static int liquidio_suspend(struct pci_dev *pdev __attribute__((unused)),
471 			    pm_message_t state __attribute__((unused)))
472 {
473 	return 0;
474 }
475 
476 /**
477  * \brief called when resuming
478  * @param pdev Pointer to PCI device
479  */
480 static int liquidio_resume(struct pci_dev *pdev __attribute__((unused)))
481 {
482 	return 0;
483 }
484 #endif
485 
486 /* For PCI-E Advanced Error Recovery (AER) Interface */
487 static const struct pci_error_handlers liquidio_err_handler = {
488 	.error_detected = liquidio_pcie_error_detected,
489 	.mmio_enabled	= liquidio_pcie_mmio_enabled,
490 	.slot_reset	= liquidio_pcie_slot_reset,
491 	.resume		= liquidio_pcie_resume,
492 };
493 
494 static const struct pci_device_id liquidio_pci_tbl[] = {
495 	{       /* 68xx */
496 		PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
497 	},
498 	{       /* 66xx */
499 		PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
500 	},
501 	{       /* 23xx pf */
502 		PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
503 	},
504 	{
505 		0, 0, 0, 0, 0, 0, 0
506 	}
507 };
508 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
509 
510 static struct pci_driver liquidio_pci_driver = {
511 	.name		= "LiquidIO",
512 	.id_table	= liquidio_pci_tbl,
513 	.probe		= liquidio_probe,
514 	.remove		= liquidio_remove,
515 	.err_handler	= &liquidio_err_handler,    /* For AER */
516 
517 #ifdef CONFIG_PM
518 	.suspend	= liquidio_suspend,
519 	.resume		= liquidio_resume,
520 #endif
521 };
522 
523 /**
524  * \brief register PCI driver
525  */
526 static int liquidio_init_pci(void)
527 {
528 	return pci_register_driver(&liquidio_pci_driver);
529 }
530 
531 /**
532  * \brief unregister PCI driver
533  */
534 static void liquidio_deinit_pci(void)
535 {
536 	pci_unregister_driver(&liquidio_pci_driver);
537 }
538 
539 /**
540  * \brief check interface state
541  * @param lio per-network private data
542  * @param state_flag flag state to check
543  */
544 static inline int ifstate_check(struct lio *lio, int state_flag)
545 {
546 	return atomic_read(&lio->ifstate) & state_flag;
547 }
548 
549 /**
550  * \brief set interface state
551  * @param lio per-network private data
552  * @param state_flag flag state to set
553  */
554 static inline void ifstate_set(struct lio *lio, int state_flag)
555 {
556 	atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) | state_flag));
557 }
558 
559 /**
560  * \brief clear interface state
561  * @param lio per-network private data
562  * @param state_flag flag state to clear
563  */
564 static inline void ifstate_reset(struct lio *lio, int state_flag)
565 {
566 	atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag)));
567 }
568 
569 /**
570  * \brief Stop Tx queues
571  * @param netdev network device
572  */
573 static inline void txqs_stop(struct net_device *netdev)
574 {
575 	if (netif_is_multiqueue(netdev)) {
576 		int i;
577 
578 		for (i = 0; i < netdev->num_tx_queues; i++)
579 			netif_stop_subqueue(netdev, i);
580 	} else {
581 		netif_stop_queue(netdev);
582 	}
583 }
584 
585 /**
586  * \brief Start Tx queues
587  * @param netdev network device
588  */
589 static inline void txqs_start(struct net_device *netdev)
590 {
591 	if (netif_is_multiqueue(netdev)) {
592 		int i;
593 
594 		for (i = 0; i < netdev->num_tx_queues; i++)
595 			netif_start_subqueue(netdev, i);
596 	} else {
597 		netif_start_queue(netdev);
598 	}
599 }
600 
601 /**
602  * \brief Wake Tx queues
603  * @param netdev network device
604  */
605 static inline void txqs_wake(struct net_device *netdev)
606 {
607 	struct lio *lio = GET_LIO(netdev);
608 
609 	if (netif_is_multiqueue(netdev)) {
610 		int i;
611 
612 		for (i = 0; i < netdev->num_tx_queues; i++) {
613 			int qno = lio->linfo.txpciq[i %
614 				(lio->linfo.num_txpciq)].s.q_no;
615 
616 			if (__netif_subqueue_stopped(netdev, i)) {
617 				INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno,
618 							  tx_restart, 1);
619 				netif_wake_subqueue(netdev, i);
620 			}
621 		}
622 	} else {
623 		INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
624 					  tx_restart, 1);
625 		netif_wake_queue(netdev);
626 	}
627 }
628 
629 /**
630  * \brief Stop Tx queue
631  * @param netdev network device
632  */
633 static void stop_txq(struct net_device *netdev)
634 {
635 	txqs_stop(netdev);
636 }
637 
638 /**
639  * \brief Start Tx queue
640  * @param netdev network device
641  */
642 static void start_txq(struct net_device *netdev)
643 {
644 	struct lio *lio = GET_LIO(netdev);
645 
646 	if (lio->linfo.link.s.link_up) {
647 		txqs_start(netdev);
648 		return;
649 	}
650 }
651 
652 /**
653  * \brief Wake a queue
654  * @param netdev network device
655  * @param q which queue to wake
656  */
657 static inline void wake_q(struct net_device *netdev, int q)
658 {
659 	if (netif_is_multiqueue(netdev))
660 		netif_wake_subqueue(netdev, q);
661 	else
662 		netif_wake_queue(netdev);
663 }
664 
665 /**
666  * \brief Stop a queue
667  * @param netdev network device
668  * @param q which queue to stop
669  */
670 static inline void stop_q(struct net_device *netdev, int q)
671 {
672 	if (netif_is_multiqueue(netdev))
673 		netif_stop_subqueue(netdev, q);
674 	else
675 		netif_stop_queue(netdev);
676 }
677 
678 /**
679  * \brief Check Tx queue status, and take appropriate action
680  * @param lio per-network private data
681  * @returns 0 if full, number of queues woken up otherwise
682  */
683 static inline int check_txq_status(struct lio *lio)
684 {
685 	int ret_val = 0;
686 
687 	if (netif_is_multiqueue(lio->netdev)) {
688 		int numqs = lio->netdev->num_tx_queues;
689 		int q, iq = 0;
690 
691 		/* check each sub-queue state */
692 		for (q = 0; q < numqs; q++) {
693 			iq = lio->linfo.txpciq[q %
694 				(lio->linfo.num_txpciq)].s.q_no;
695 			if (octnet_iq_is_full(lio->oct_dev, iq))
696 				continue;
697 			if (__netif_subqueue_stopped(lio->netdev, q)) {
698 				wake_q(lio->netdev, q);
699 				INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
700 							  tx_restart, 1);
701 				ret_val++;
702 			}
703 		}
704 	} else {
705 		if (octnet_iq_is_full(lio->oct_dev, lio->txq))
706 			return 0;
707 		wake_q(lio->netdev, lio->txq);
708 		INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
709 					  tx_restart, 1);
710 		ret_val = 1;
711 	}
712 	return ret_val;
713 }
714 
715 /**
716  * Remove the node at the head of the list. The list would be empty at
717  * the end of this call if there are no more nodes in the list.
718  */
719 static inline struct list_head *list_delete_head(struct list_head *root)
720 {
721 	struct list_head *node;
722 
723 	if ((root->prev == root) && (root->next == root))
724 		node = NULL;
725 	else
726 		node = root->next;
727 
728 	if (node)
729 		list_del(node);
730 
731 	return node;
732 }
733 
734 /**
735  * \brief Delete gather lists
736  * @param lio per-network private data
737  */
738 static void delete_glists(struct lio *lio)
739 {
740 	struct octnic_gather *g;
741 	int i;
742 
743 	if (!lio->glist)
744 		return;
745 
746 	for (i = 0; i < lio->linfo.num_txpciq; i++) {
747 		do {
748 			g = (struct octnic_gather *)
749 				list_delete_head(&lio->glist[i]);
750 			if (g) {
751 				if (g->sg) {
752 					dma_unmap_single(&lio->oct_dev->
753 							 pci_dev->dev,
754 							 g->sg_dma_ptr,
755 							 g->sg_size,
756 							 DMA_TO_DEVICE);
757 					kfree((void *)((unsigned long)g->sg -
758 						       g->adjust));
759 				}
760 				kfree(g);
761 			}
762 		} while (g);
763 	}
764 
765 	kfree((void *)lio->glist);
766 }
767 
768 /**
769  * \brief Setup gather lists
770  * @param lio per-network private data
771  */
772 static int setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)
773 {
774 	int i, j;
775 	struct octnic_gather *g;
776 
777 	lio->glist_lock = kcalloc(num_iqs, sizeof(*lio->glist_lock),
778 				  GFP_KERNEL);
779 	if (!lio->glist_lock)
780 		return 1;
781 
782 	lio->glist = kcalloc(num_iqs, sizeof(*lio->glist),
783 			     GFP_KERNEL);
784 	if (!lio->glist) {
785 		kfree((void *)lio->glist_lock);
786 		return 1;
787 	}
788 
789 	for (i = 0; i < num_iqs; i++) {
790 		int numa_node = cpu_to_node(i % num_online_cpus());
791 
792 		spin_lock_init(&lio->glist_lock[i]);
793 
794 		INIT_LIST_HEAD(&lio->glist[i]);
795 
796 		for (j = 0; j < lio->tx_qsize; j++) {
797 			g = kzalloc_node(sizeof(*g), GFP_KERNEL,
798 					 numa_node);
799 			if (!g)
800 				g = kzalloc(sizeof(*g), GFP_KERNEL);
801 			if (!g)
802 				break;
803 
804 			g->sg_size = ((ROUNDUP4(OCTNIC_MAX_SG) >> 2) *
805 				      OCT_SG_ENTRY_SIZE);
806 
807 			g->sg = kmalloc_node(g->sg_size + 8,
808 					     GFP_KERNEL, numa_node);
809 			if (!g->sg)
810 				g->sg = kmalloc(g->sg_size + 8, GFP_KERNEL);
811 			if (!g->sg) {
812 				kfree(g);
813 				break;
814 			}
815 
816 			/* The gather component should be aligned on 64-bit
817 			 * boundary
818 			 */
819 			if (((unsigned long)g->sg) & 7) {
820 				g->adjust = 8 - (((unsigned long)g->sg) & 7);
821 				g->sg = (struct octeon_sg_entry *)
822 					((unsigned long)g->sg + g->adjust);
823 			}
824 			g->sg_dma_ptr = dma_map_single(&oct->pci_dev->dev,
825 						       g->sg, g->sg_size,
826 						       DMA_TO_DEVICE);
827 			if (dma_mapping_error(&oct->pci_dev->dev,
828 					      g->sg_dma_ptr)) {
829 				kfree((void *)((unsigned long)g->sg -
830 					       g->adjust));
831 				kfree(g);
832 				break;
833 			}
834 
835 			list_add_tail(&g->list, &lio->glist[i]);
836 		}
837 
838 		if (j != lio->tx_qsize) {
839 			delete_glists(lio);
840 			return 1;
841 		}
842 	}
843 
844 	return 0;
845 }
846 
847 /**
848  * \brief Print link information
849  * @param netdev network device
850  */
851 static void print_link_info(struct net_device *netdev)
852 {
853 	struct lio *lio = GET_LIO(netdev);
854 
855 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) {
856 		struct oct_link_info *linfo = &lio->linfo;
857 
858 		if (linfo->link.s.link_up) {
859 			netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
860 				   linfo->link.s.speed,
861 				   (linfo->link.s.duplex) ? "Full" : "Half");
862 		} else {
863 			netif_info(lio, link, lio->netdev, "Link Down\n");
864 		}
865 	}
866 }
867 
868 /**
869  * \brief Routine to notify MTU change
870  * @param work work_struct data structure
871  */
872 static void octnet_link_status_change(struct work_struct *work)
873 {
874 	struct cavium_wk *wk = (struct cavium_wk *)work;
875 	struct lio *lio = (struct lio *)wk->ctxptr;
876 
877 	rtnl_lock();
878 	call_netdevice_notifiers(NETDEV_CHANGEMTU, lio->netdev);
879 	rtnl_unlock();
880 }
881 
882 /**
883  * \brief Sets up the mtu status change work
884  * @param netdev network device
885  */
886 static inline int setup_link_status_change_wq(struct net_device *netdev)
887 {
888 	struct lio *lio = GET_LIO(netdev);
889 	struct octeon_device *oct = lio->oct_dev;
890 
891 	lio->link_status_wq.wq = alloc_workqueue("link-status",
892 						 WQ_MEM_RECLAIM, 0);
893 	if (!lio->link_status_wq.wq) {
894 		dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");
895 		return -1;
896 	}
897 	INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,
898 			  octnet_link_status_change);
899 	lio->link_status_wq.wk.ctxptr = lio;
900 
901 	return 0;
902 }
903 
904 static inline void cleanup_link_status_change_wq(struct net_device *netdev)
905 {
906 	struct lio *lio = GET_LIO(netdev);
907 
908 	if (lio->link_status_wq.wq) {
909 		cancel_delayed_work_sync(&lio->link_status_wq.wk.work);
910 		destroy_workqueue(lio->link_status_wq.wq);
911 	}
912 }
913 
914 /**
915  * \brief Update link status
916  * @param netdev network device
917  * @param ls link status structure
918  *
919  * Called on receipt of a link status response from the core application to
920  * update each interface's link status.
921  */
922 static inline void update_link_status(struct net_device *netdev,
923 				      union oct_link_status *ls)
924 {
925 	struct lio *lio = GET_LIO(netdev);
926 	int changed = (lio->linfo.link.u64 != ls->u64);
927 
928 	lio->linfo.link.u64 = ls->u64;
929 
930 	if ((lio->intf_open) && (changed)) {
931 		print_link_info(netdev);
932 		lio->link_changes++;
933 
934 		if (lio->linfo.link.s.link_up) {
935 			netif_carrier_on(netdev);
936 			/* start_txq(netdev); */
937 			txqs_wake(netdev);
938 		} else {
939 			netif_carrier_off(netdev);
940 			stop_txq(netdev);
941 		}
942 	}
943 }
944 
945 /* Runs in interrupt context. */
946 static void update_txq_status(struct octeon_device *oct, int iq_num)
947 {
948 	struct net_device *netdev;
949 	struct lio *lio;
950 	struct octeon_instr_queue *iq = oct->instr_queue[iq_num];
951 
952 	netdev = oct->props[iq->ifidx].netdev;
953 
954 	/* This is needed because the first IQ does not have
955 	 * a netdev associated with it.
956 	 */
957 	if (!netdev)
958 		return;
959 
960 	lio = GET_LIO(netdev);
961 	if (netif_is_multiqueue(netdev)) {
962 		if (__netif_subqueue_stopped(netdev, iq->q_index) &&
963 		    lio->linfo.link.s.link_up &&
964 		    (!octnet_iq_is_full(oct, iq_num))) {
965 			INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq_num,
966 						  tx_restart, 1);
967 			netif_wake_subqueue(netdev, iq->q_index);
968 		} else {
969 			if (!octnet_iq_is_full(oct, lio->txq)) {
970 				INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev,
971 							  lio->txq,
972 							  tx_restart, 1);
973 				wake_q(netdev, lio->txq);
974 			}
975 		}
976 	}
977 }
978 
979 static
980 int liquidio_schedule_msix_droq_pkt_handler(struct octeon_droq *droq, u64 ret)
981 {
982 	struct octeon_device *oct = droq->oct_dev;
983 	struct octeon_device_priv *oct_priv =
984 	    (struct octeon_device_priv *)oct->priv;
985 
986 	if (droq->ops.poll_mode) {
987 		droq->ops.napi_fn(droq);
988 	} else {
989 		if (ret & MSIX_PO_INT) {
990 			tasklet_schedule(&oct_priv->droq_tasklet);
991 			return 1;
992 		}
993 		/* this will be flushed periodically by check iq db */
994 		if (ret & MSIX_PI_INT)
995 			return 0;
996 	}
997 	return 0;
998 }
999 
1000 /**
1001  * \brief Droq packet processor sceduler
1002  * @param oct octeon device
1003  */
1004 static void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
1005 {
1006 	struct octeon_device_priv *oct_priv =
1007 		(struct octeon_device_priv *)oct->priv;
1008 	u64 oq_no;
1009 	struct octeon_droq *droq;
1010 
1011 	if (oct->int_status & OCT_DEV_INTR_PKT_DATA) {
1012 		for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct);
1013 		     oq_no++) {
1014 			if (!(oct->droq_intr & (1ULL << oq_no)))
1015 				continue;
1016 
1017 			droq = oct->droq[oq_no];
1018 
1019 			if (droq->ops.poll_mode) {
1020 				droq->ops.napi_fn(droq);
1021 				oct_priv->napi_mask |= (1 << oq_no);
1022 			} else {
1023 				tasklet_schedule(&oct_priv->droq_tasklet);
1024 			}
1025 		}
1026 	}
1027 }
1028 
1029 static irqreturn_t
1030 liquidio_msix_intr_handler(int irq __attribute__((unused)), void *dev)
1031 {
1032 	u64 ret;
1033 	struct octeon_ioq_vector *ioq_vector = (struct octeon_ioq_vector *)dev;
1034 	struct octeon_device *oct = ioq_vector->oct_dev;
1035 	struct octeon_droq *droq = oct->droq[ioq_vector->droq_index];
1036 
1037 	ret = oct->fn_list.msix_interrupt_handler(ioq_vector);
1038 
1039 	if ((ret & MSIX_PO_INT) || (ret & MSIX_PI_INT))
1040 		liquidio_schedule_msix_droq_pkt_handler(droq, ret);
1041 
1042 	return IRQ_HANDLED;
1043 }
1044 
1045 /**
1046  * \brief Interrupt handler for octeon
1047  * @param irq unused
1048  * @param dev octeon device
1049  */
1050 static
1051 irqreturn_t liquidio_legacy_intr_handler(int irq __attribute__((unused)),
1052 					 void *dev)
1053 {
1054 	struct octeon_device *oct = (struct octeon_device *)dev;
1055 	irqreturn_t ret;
1056 
1057 	/* Disable our interrupts for the duration of ISR */
1058 	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
1059 
1060 	ret = oct->fn_list.process_interrupt_regs(oct);
1061 
1062 	if (ret == IRQ_HANDLED)
1063 		liquidio_schedule_droq_pkt_handlers(oct);
1064 
1065 	/* Re-enable our interrupts  */
1066 	if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET))
1067 		oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);
1068 
1069 	return ret;
1070 }
1071 
1072 /**
1073  * \brief Setup interrupt for octeon device
1074  * @param oct octeon device
1075  *
1076  *  Enable interrupt in Octeon device as given in the PCI interrupt mask.
1077  */
1078 static int octeon_setup_interrupt(struct octeon_device *oct)
1079 {
1080 	int irqret, err;
1081 	struct msix_entry *msix_entries;
1082 	int i;
1083 	int num_ioq_vectors;
1084 	int num_alloc_ioq_vectors;
1085 
1086 	if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
1087 		oct->num_msix_irqs = oct->sriov_info.num_pf_rings;
1088 		/* one non ioq interrupt for handling sli_mac_pf_int_sum */
1089 		oct->num_msix_irqs += 1;
1090 
1091 		oct->msix_entries = kcalloc(
1092 		    oct->num_msix_irqs, sizeof(struct msix_entry), GFP_KERNEL);
1093 		if (!oct->msix_entries)
1094 			return 1;
1095 
1096 		msix_entries = (struct msix_entry *)oct->msix_entries;
1097 		/*Assumption is that pf msix vectors start from pf srn to pf to
1098 		 * trs and not from 0. if not change this code
1099 		 */
1100 		for (i = 0; i < oct->num_msix_irqs - 1; i++)
1101 			msix_entries[i].entry = oct->sriov_info.pf_srn + i;
1102 		msix_entries[oct->num_msix_irqs - 1].entry =
1103 		    oct->sriov_info.trs;
1104 		num_alloc_ioq_vectors = pci_enable_msix_range(
1105 						oct->pci_dev, msix_entries,
1106 						oct->num_msix_irqs,
1107 						oct->num_msix_irqs);
1108 		if (num_alloc_ioq_vectors < 0) {
1109 			dev_err(&oct->pci_dev->dev, "unable to Allocate MSI-X interrupts\n");
1110 			kfree(oct->msix_entries);
1111 			oct->msix_entries = NULL;
1112 			return 1;
1113 		}
1114 		dev_dbg(&oct->pci_dev->dev, "OCTEON: Enough MSI-X interrupts are allocated...\n");
1115 
1116 		num_ioq_vectors = oct->num_msix_irqs;
1117 
1118 		/** For PF, there is one non-ioq interrupt handler */
1119 		num_ioq_vectors -= 1;
1120 		irqret = request_irq(msix_entries[num_ioq_vectors].vector,
1121 				     liquidio_legacy_intr_handler, 0, "octeon",
1122 				     oct);
1123 		if (irqret) {
1124 			dev_err(&oct->pci_dev->dev,
1125 				"OCTEON: Request_irq failed for MSIX interrupt Error: %d\n",
1126 				irqret);
1127 			pci_disable_msix(oct->pci_dev);
1128 			kfree(oct->msix_entries);
1129 			oct->msix_entries = NULL;
1130 			return 1;
1131 		}
1132 
1133 		for (i = 0; i < num_ioq_vectors; i++) {
1134 			irqret = request_irq(msix_entries[i].vector,
1135 					     liquidio_msix_intr_handler, 0,
1136 					     "octeon", &oct->ioq_vector[i]);
1137 			if (irqret) {
1138 				dev_err(&oct->pci_dev->dev,
1139 					"OCTEON: Request_irq failed for MSIX interrupt Error: %d\n",
1140 					irqret);
1141 				/** Freeing the non-ioq irq vector here . */
1142 				free_irq(msix_entries[num_ioq_vectors].vector,
1143 					 oct);
1144 
1145 				while (i) {
1146 					i--;
1147 					/** clearing affinity mask. */
1148 					irq_set_affinity_hint(
1149 						msix_entries[i].vector, NULL);
1150 					free_irq(msix_entries[i].vector,
1151 						 &oct->ioq_vector[i]);
1152 				}
1153 				pci_disable_msix(oct->pci_dev);
1154 				kfree(oct->msix_entries);
1155 				oct->msix_entries = NULL;
1156 				return 1;
1157 			}
1158 			oct->ioq_vector[i].vector = msix_entries[i].vector;
1159 			/* assign the cpu mask for this msix interrupt vector */
1160 			irq_set_affinity_hint(
1161 					msix_entries[i].vector,
1162 					(&oct->ioq_vector[i].affinity_mask));
1163 		}
1164 		dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: MSI-X enabled\n",
1165 			oct->octeon_id);
1166 	} else {
1167 		err = pci_enable_msi(oct->pci_dev);
1168 		if (err)
1169 			dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n",
1170 				 err);
1171 		else
1172 			oct->flags |= LIO_FLAG_MSI_ENABLED;
1173 
1174 		irqret = request_irq(oct->pci_dev->irq,
1175 				     liquidio_legacy_intr_handler, IRQF_SHARED,
1176 				     "octeon", oct);
1177 		if (irqret) {
1178 			if (oct->flags & LIO_FLAG_MSI_ENABLED)
1179 				pci_disable_msi(oct->pci_dev);
1180 			dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n",
1181 				irqret);
1182 			return 1;
1183 		}
1184 	}
1185 	return 0;
1186 }
1187 
1188 static int liquidio_watchdog(void *param)
1189 {
1190 	u64 wdog;
1191 	u16 mask_of_stuck_cores = 0;
1192 	u16 mask_of_crashed_cores = 0;
1193 	int core_num;
1194 	u8 core_is_stuck[LIO_MAX_CORES];
1195 	u8 core_crashed[LIO_MAX_CORES];
1196 	struct octeon_device *oct = param;
1197 
1198 	memset(core_is_stuck, 0, sizeof(core_is_stuck));
1199 	memset(core_crashed, 0, sizeof(core_crashed));
1200 
1201 	while (!kthread_should_stop()) {
1202 		mask_of_crashed_cores =
1203 		    (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);
1204 
1205 		for (core_num = 0; core_num < LIO_MAX_CORES; core_num++) {
1206 			if (!core_is_stuck[core_num]) {
1207 				wdog = lio_pci_readq(oct, CIU3_WDOG(core_num));
1208 
1209 				/* look at watchdog state field */
1210 				wdog &= CIU3_WDOG_MASK;
1211 				if (wdog) {
1212 					/* this watchdog timer has expired */
1213 					core_is_stuck[core_num] =
1214 						LIO_MONITOR_WDOG_EXPIRE;
1215 					mask_of_stuck_cores |= (1 << core_num);
1216 				}
1217 			}
1218 
1219 			if (!core_crashed[core_num])
1220 				core_crashed[core_num] =
1221 				    (mask_of_crashed_cores >> core_num) & 1;
1222 		}
1223 
1224 		if (mask_of_stuck_cores) {
1225 			for (core_num = 0; core_num < LIO_MAX_CORES;
1226 			     core_num++) {
1227 				if (core_is_stuck[core_num] == 1) {
1228 					dev_err(&oct->pci_dev->dev,
1229 						"ERROR: Octeon core %d is stuck!\n",
1230 						core_num);
1231 					/* 2 means we have printk'd  an error
1232 					 * so no need to repeat the same printk
1233 					 */
1234 					core_is_stuck[core_num] =
1235 						LIO_MONITOR_CORE_STUCK_MSGD;
1236 				}
1237 			}
1238 		}
1239 
1240 		if (mask_of_crashed_cores) {
1241 			for (core_num = 0; core_num < LIO_MAX_CORES;
1242 			     core_num++) {
1243 				if (core_crashed[core_num] == 1) {
1244 					dev_err(&oct->pci_dev->dev,
1245 						"ERROR: Octeon core %d crashed!  See oct-fwdump for details.\n",
1246 						core_num);
1247 					/* 2 means we have printk'd  an error
1248 					 * so no need to repeat the same printk
1249 					 */
1250 					core_crashed[core_num] =
1251 						LIO_MONITOR_CORE_STUCK_MSGD;
1252 				}
1253 			}
1254 		}
1255 #ifdef CONFIG_MODULE_UNLOAD
1256 		if (mask_of_stuck_cores || mask_of_crashed_cores) {
1257 			/* make module refcount=0 so that rmmod will work */
1258 			long refcount;
1259 
1260 			refcount = module_refcount(THIS_MODULE);
1261 
1262 			while (refcount > 0) {
1263 				module_put(THIS_MODULE);
1264 				refcount = module_refcount(THIS_MODULE);
1265 			}
1266 
1267 			/* compensate for and withstand an unlikely (but still
1268 			 * possible) race condition
1269 			 */
1270 			while (refcount < 0) {
1271 				try_module_get(THIS_MODULE);
1272 				refcount = module_refcount(THIS_MODULE);
1273 			}
1274 		}
1275 #endif
1276 		/* sleep for two seconds */
1277 		set_current_state(TASK_INTERRUPTIBLE);
1278 		schedule_timeout(2 * HZ);
1279 	}
1280 
1281 	return 0;
1282 }
1283 
1284 /**
1285  * \brief PCI probe handler
1286  * @param pdev PCI device structure
1287  * @param ent unused
1288  */
1289 static int
1290 liquidio_probe(struct pci_dev *pdev,
1291 	       const struct pci_device_id *ent __attribute__((unused)))
1292 {
1293 	struct octeon_device *oct_dev = NULL;
1294 	struct handshake *hs;
1295 
1296 	oct_dev = octeon_allocate_device(pdev->device,
1297 					 sizeof(struct octeon_device_priv));
1298 	if (!oct_dev) {
1299 		dev_err(&pdev->dev, "Unable to allocate device\n");
1300 		return -ENOMEM;
1301 	}
1302 
1303 	if (pdev->device == OCTEON_CN23XX_PF_VID)
1304 		oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
1305 
1306 	dev_info(&pdev->dev, "Initializing device %x:%x.\n",
1307 		 (u32)pdev->vendor, (u32)pdev->device);
1308 
1309 	/* Assign octeon_device for this device to the private data area. */
1310 	pci_set_drvdata(pdev, oct_dev);
1311 
1312 	/* set linux specific device pointer */
1313 	oct_dev->pci_dev = (void *)pdev;
1314 
1315 	hs = &handshake[oct_dev->octeon_id];
1316 	init_completion(&hs->init);
1317 	init_completion(&hs->started);
1318 	hs->pci_dev = pdev;
1319 
1320 	if (oct_dev->octeon_id == 0)
1321 		/* first LiquidIO NIC is detected */
1322 		complete(&first_stage);
1323 
1324 	if (octeon_device_init(oct_dev)) {
1325 		liquidio_remove(pdev);
1326 		return -ENOMEM;
1327 	}
1328 
1329 	if (OCTEON_CN23XX_PF(oct_dev)) {
1330 		u64 scratch1;
1331 		u8 bus, device, function;
1332 
1333 		scratch1 = octeon_read_csr64(oct_dev, CN23XX_SLI_SCRATCH1);
1334 		if (!(scratch1 & 4ULL)) {
1335 			/* Bit 2 of SLI_SCRATCH_1 is a flag that indicates that
1336 			 * the lio watchdog kernel thread is running for this
1337 			 * NIC.  Each NIC gets one watchdog kernel thread.
1338 			 */
1339 			scratch1 |= 4ULL;
1340 			octeon_write_csr64(oct_dev, CN23XX_SLI_SCRATCH1,
1341 					   scratch1);
1342 
1343 			bus = pdev->bus->number;
1344 			device = PCI_SLOT(pdev->devfn);
1345 			function = PCI_FUNC(pdev->devfn);
1346 			oct_dev->watchdog_task = kthread_create(
1347 			    liquidio_watchdog, oct_dev,
1348 			    "liowd/%02hhx:%02hhx.%hhx", bus, device, function);
1349 			wake_up_process(oct_dev->watchdog_task);
1350 		}
1351 	}
1352 
1353 	oct_dev->rx_pause = 1;
1354 	oct_dev->tx_pause = 1;
1355 
1356 	dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
1357 
1358 	return 0;
1359 }
1360 
1361 /**
1362  *\brief Destroy resources associated with octeon device
1363  * @param pdev PCI device structure
1364  * @param ent unused
1365  */
1366 static void octeon_destroy_resources(struct octeon_device *oct)
1367 {
1368 	int i;
1369 	struct msix_entry *msix_entries;
1370 	struct octeon_device_priv *oct_priv =
1371 		(struct octeon_device_priv *)oct->priv;
1372 
1373 	struct handshake *hs;
1374 
1375 	switch (atomic_read(&oct->status)) {
1376 	case OCT_DEV_RUNNING:
1377 	case OCT_DEV_CORE_OK:
1378 
1379 		/* No more instructions will be forwarded. */
1380 		atomic_set(&oct->status, OCT_DEV_IN_RESET);
1381 
1382 		oct->app_mode = CVM_DRV_INVALID_APP;
1383 		dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
1384 			lio_get_state_string(&oct->status));
1385 
1386 		schedule_timeout_uninterruptible(HZ / 10);
1387 
1388 		/* fallthrough */
1389 	case OCT_DEV_HOST_OK:
1390 
1391 		/* fallthrough */
1392 	case OCT_DEV_CONSOLE_INIT_DONE:
1393 		/* Remove any consoles */
1394 		octeon_remove_consoles(oct);
1395 
1396 		/* fallthrough */
1397 	case OCT_DEV_IO_QUEUES_DONE:
1398 		if (wait_for_pending_requests(oct))
1399 			dev_err(&oct->pci_dev->dev, "There were pending requests\n");
1400 
1401 		if (lio_wait_for_instr_fetch(oct))
1402 			dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
1403 
1404 		/* Disable the input and output queues now. No more packets will
1405 		 * arrive from Octeon, but we should wait for all packet
1406 		 * processing to finish.
1407 		 */
1408 		oct->fn_list.disable_io_queues(oct);
1409 
1410 		if (lio_wait_for_oq_pkts(oct))
1411 			dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
1412 
1413 		/* Disable interrupts  */
1414 		oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
1415 
1416 		if (oct->msix_on) {
1417 			msix_entries = (struct msix_entry *)oct->msix_entries;
1418 			for (i = 0; i < oct->num_msix_irqs - 1; i++) {
1419 				/* clear the affinity_cpumask */
1420 				irq_set_affinity_hint(msix_entries[i].vector,
1421 						      NULL);
1422 				free_irq(msix_entries[i].vector,
1423 					 &oct->ioq_vector[i]);
1424 			}
1425 			/* non-iov vector's argument is oct struct */
1426 			free_irq(msix_entries[i].vector, oct);
1427 
1428 			pci_disable_msix(oct->pci_dev);
1429 			kfree(oct->msix_entries);
1430 			oct->msix_entries = NULL;
1431 		} else {
1432 			/* Release the interrupt line */
1433 			free_irq(oct->pci_dev->irq, oct);
1434 
1435 			if (oct->flags & LIO_FLAG_MSI_ENABLED)
1436 				pci_disable_msi(oct->pci_dev);
1437 		}
1438 
1439 		if (OCTEON_CN23XX_PF(oct))
1440 			octeon_free_ioq_vector(oct);
1441 	/* fallthrough */
1442 	case OCT_DEV_IN_RESET:
1443 	case OCT_DEV_DROQ_INIT_DONE:
1444 		/*atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);*/
1445 		mdelay(100);
1446 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
1447 			if (!(oct->io_qmask.oq & BIT_ULL(i)))
1448 				continue;
1449 			octeon_delete_droq(oct, i);
1450 		}
1451 
1452 		/* Force any pending handshakes to complete */
1453 		for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1454 			hs = &handshake[i];
1455 
1456 			if (hs->pci_dev) {
1457 				handshake[oct->octeon_id].init_ok = 0;
1458 				complete(&handshake[oct->octeon_id].init);
1459 				handshake[oct->octeon_id].started_ok = 0;
1460 				complete(&handshake[oct->octeon_id].started);
1461 			}
1462 		}
1463 
1464 		/* fallthrough */
1465 	case OCT_DEV_RESP_LIST_INIT_DONE:
1466 		octeon_delete_response_list(oct);
1467 
1468 		/* fallthrough */
1469 	case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1470 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1471 			if (!(oct->io_qmask.iq & BIT_ULL(i)))
1472 				continue;
1473 			octeon_delete_instr_queue(oct, i);
1474 		}
1475 		/* fallthrough */
1476 	case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1477 		octeon_free_sc_buffer_pool(oct);
1478 
1479 		/* fallthrough */
1480 	case OCT_DEV_DISPATCH_INIT_DONE:
1481 		octeon_delete_dispatch_list(oct);
1482 		cancel_delayed_work_sync(&oct->nic_poll_work.work);
1483 
1484 		/* fallthrough */
1485 	case OCT_DEV_PCI_MAP_DONE:
1486 		/* Soft reset the octeon device before exiting */
1487 		if ((!OCTEON_CN23XX_PF(oct)) || !oct->octeon_id)
1488 			oct->fn_list.soft_reset(oct);
1489 
1490 		octeon_unmap_pci_barx(oct, 0);
1491 		octeon_unmap_pci_barx(oct, 1);
1492 
1493 		/* fallthrough */
1494 	case OCT_DEV_BEGIN_STATE:
1495 		/* Disable the device, releasing the PCI INT */
1496 		pci_disable_device(oct->pci_dev);
1497 
1498 		/* Nothing to be done here either */
1499 		break;
1500 	}                       /* end switch (oct->status) */
1501 
1502 	tasklet_kill(&oct_priv->droq_tasklet);
1503 }
1504 
1505 /**
1506  * \brief Callback for rx ctrl
1507  * @param status status of request
1508  * @param buf pointer to resp structure
1509  */
1510 static void rx_ctl_callback(struct octeon_device *oct,
1511 			    u32 status,
1512 			    void *buf)
1513 {
1514 	struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
1515 	struct liquidio_rx_ctl_context *ctx;
1516 
1517 	ctx  = (struct liquidio_rx_ctl_context *)sc->ctxptr;
1518 
1519 	oct = lio_get_device(ctx->octeon_id);
1520 	if (status)
1521 		dev_err(&oct->pci_dev->dev, "rx ctl instruction failed. Status: %llx\n",
1522 			CVM_CAST64(status));
1523 	WRITE_ONCE(ctx->cond, 1);
1524 
1525 	/* This barrier is required to be sure that the response has been
1526 	 * written fully before waking up the handler
1527 	 */
1528 	wmb();
1529 
1530 	wake_up_interruptible(&ctx->wc);
1531 }
1532 
1533 /**
1534  * \brief Send Rx control command
1535  * @param lio per-network private data
1536  * @param start_stop whether to start or stop
1537  */
1538 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1539 {
1540 	struct octeon_soft_command *sc;
1541 	struct liquidio_rx_ctl_context *ctx;
1542 	union octnet_cmd *ncmd;
1543 	int ctx_size = sizeof(struct liquidio_rx_ctl_context);
1544 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1545 	int retval;
1546 
1547 	if (oct->props[lio->ifidx].rx_on == start_stop)
1548 		return;
1549 
1550 	sc = (struct octeon_soft_command *)
1551 		octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
1552 					  16, ctx_size);
1553 
1554 	ncmd = (union octnet_cmd *)sc->virtdptr;
1555 	ctx  = (struct liquidio_rx_ctl_context *)sc->ctxptr;
1556 
1557 	WRITE_ONCE(ctx->cond, 0);
1558 	ctx->octeon_id = lio_get_device_id(oct);
1559 	init_waitqueue_head(&ctx->wc);
1560 
1561 	ncmd->u64 = 0;
1562 	ncmd->s.cmd = OCTNET_CMD_RX_CTL;
1563 	ncmd->s.param1 = start_stop;
1564 
1565 	octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
1566 
1567 	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1568 
1569 	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
1570 				    OPCODE_NIC_CMD, 0, 0, 0);
1571 
1572 	sc->callback = rx_ctl_callback;
1573 	sc->callback_arg = sc;
1574 	sc->wait_time = 5000;
1575 
1576 	retval = octeon_send_soft_command(oct, sc);
1577 	if (retval == IQ_SEND_FAILED) {
1578 		netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1579 	} else {
1580 		/* Sleep on a wait queue till the cond flag indicates that the
1581 		 * response arrived or timed-out.
1582 		 */
1583 		if (sleep_cond(&ctx->wc, &ctx->cond) == -EINTR)
1584 			return;
1585 		oct->props[lio->ifidx].rx_on = start_stop;
1586 	}
1587 
1588 	octeon_free_soft_command(oct, sc);
1589 }
1590 
1591 /**
1592  * \brief Destroy NIC device interface
1593  * @param oct octeon device
1594  * @param ifidx which interface to destroy
1595  *
1596  * Cleanup associated with each interface for an Octeon device  when NIC
1597  * module is being unloaded or if initialization fails during load.
1598  */
1599 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1600 {
1601 	struct net_device *netdev = oct->props[ifidx].netdev;
1602 	struct lio *lio;
1603 	struct napi_struct *napi, *n;
1604 
1605 	if (!netdev) {
1606 		dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1607 			__func__, ifidx);
1608 		return;
1609 	}
1610 
1611 	lio = GET_LIO(netdev);
1612 
1613 	dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1614 
1615 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1616 		liquidio_stop(netdev);
1617 
1618 	if (oct->props[lio->ifidx].napi_enabled == 1) {
1619 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1620 			napi_disable(napi);
1621 
1622 		oct->props[lio->ifidx].napi_enabled = 0;
1623 
1624 		if (OCTEON_CN23XX_PF(oct))
1625 			oct->droq[0]->ops.poll_mode = 0;
1626 	}
1627 
1628 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1629 		unregister_netdev(netdev);
1630 
1631 	cleanup_link_status_change_wq(netdev);
1632 
1633 	delete_glists(lio);
1634 
1635 	free_netdev(netdev);
1636 
1637 	oct->props[ifidx].gmxport = -1;
1638 
1639 	oct->props[ifidx].netdev = NULL;
1640 }
1641 
1642 /**
1643  * \brief Stop complete NIC functionality
1644  * @param oct octeon device
1645  */
1646 static int liquidio_stop_nic_module(struct octeon_device *oct)
1647 {
1648 	int i, j;
1649 	struct lio *lio;
1650 
1651 	dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1652 	if (!oct->ifcount) {
1653 		dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1654 		return 1;
1655 	}
1656 
1657 	spin_lock_bh(&oct->cmd_resp_wqlock);
1658 	oct->cmd_resp_state = OCT_DRV_OFFLINE;
1659 	spin_unlock_bh(&oct->cmd_resp_wqlock);
1660 
1661 	for (i = 0; i < oct->ifcount; i++) {
1662 		lio = GET_LIO(oct->props[i].netdev);
1663 		for (j = 0; j < lio->linfo.num_rxpciq; j++)
1664 			octeon_unregister_droq_ops(oct,
1665 						   lio->linfo.rxpciq[j].s.q_no);
1666 	}
1667 
1668 	for (i = 0; i < oct->ifcount; i++)
1669 		liquidio_destroy_nic_device(oct, i);
1670 
1671 	dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1672 	return 0;
1673 }
1674 
1675 /**
1676  * \brief Cleans up resources at unload time
1677  * @param pdev PCI device structure
1678  */
1679 static void liquidio_remove(struct pci_dev *pdev)
1680 {
1681 	struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1682 
1683 	dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1684 
1685 	if (oct_dev->watchdog_task)
1686 		kthread_stop(oct_dev->watchdog_task);
1687 
1688 	if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1689 		liquidio_stop_nic_module(oct_dev);
1690 
1691 	/* Reset the octeon device and cleanup all memory allocated for
1692 	 * the octeon device by driver.
1693 	 */
1694 	octeon_destroy_resources(oct_dev);
1695 
1696 	dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1697 
1698 	/* This octeon device has been removed. Update the global
1699 	 * data structure to reflect this. Free the device structure.
1700 	 */
1701 	octeon_free_device_mem(oct_dev);
1702 }
1703 
1704 /**
1705  * \brief Identify the Octeon device and to map the BAR address space
1706  * @param oct octeon device
1707  */
1708 static int octeon_chip_specific_setup(struct octeon_device *oct)
1709 {
1710 	u32 dev_id, rev_id;
1711 	int ret = 1;
1712 	char *s;
1713 
1714 	pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1715 	pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1716 	oct->rev_id = rev_id & 0xff;
1717 
1718 	switch (dev_id) {
1719 	case OCTEON_CN68XX_PCIID:
1720 		oct->chip_id = OCTEON_CN68XX;
1721 		ret = lio_setup_cn68xx_octeon_device(oct);
1722 		s = "CN68XX";
1723 		break;
1724 
1725 	case OCTEON_CN66XX_PCIID:
1726 		oct->chip_id = OCTEON_CN66XX;
1727 		ret = lio_setup_cn66xx_octeon_device(oct);
1728 		s = "CN66XX";
1729 		break;
1730 
1731 	case OCTEON_CN23XX_PCIID_PF:
1732 		oct->chip_id = OCTEON_CN23XX_PF_VID;
1733 		ret = setup_cn23xx_octeon_pf_device(oct);
1734 		s = "CN23XX";
1735 		break;
1736 
1737 	default:
1738 		s = "?";
1739 		dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1740 			dev_id);
1741 	}
1742 
1743 	if (!ret)
1744 		dev_info(&oct->pci_dev->dev, "%s PASS%d.%d %s Version: %s\n", s,
1745 			 OCTEON_MAJOR_REV(oct),
1746 			 OCTEON_MINOR_REV(oct),
1747 			 octeon_get_conf(oct)->card_name,
1748 			 LIQUIDIO_VERSION);
1749 
1750 	return ret;
1751 }
1752 
1753 /**
1754  * \brief PCI initialization for each Octeon device.
1755  * @param oct octeon device
1756  */
1757 static int octeon_pci_os_setup(struct octeon_device *oct)
1758 {
1759 	/* setup PCI stuff first */
1760 	if (pci_enable_device(oct->pci_dev)) {
1761 		dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1762 		return 1;
1763 	}
1764 
1765 	if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1766 		dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1767 		return 1;
1768 	}
1769 
1770 	/* Enable PCI DMA Master. */
1771 	pci_set_master(oct->pci_dev);
1772 
1773 	return 0;
1774 }
1775 
1776 static inline int skb_iq(struct lio *lio, struct sk_buff *skb)
1777 {
1778 	int q = 0;
1779 
1780 	if (netif_is_multiqueue(lio->netdev))
1781 		q = skb->queue_mapping % lio->linfo.num_txpciq;
1782 
1783 	return q;
1784 }
1785 
1786 /**
1787  * \brief Check Tx queue state for a given network buffer
1788  * @param lio per-network private data
1789  * @param skb network buffer
1790  */
1791 static inline int check_txq_state(struct lio *lio, struct sk_buff *skb)
1792 {
1793 	int q = 0, iq = 0;
1794 
1795 	if (netif_is_multiqueue(lio->netdev)) {
1796 		q = skb->queue_mapping;
1797 		iq = lio->linfo.txpciq[(q % (lio->linfo.num_txpciq))].s.q_no;
1798 	} else {
1799 		iq = lio->txq;
1800 		q = iq;
1801 	}
1802 
1803 	if (octnet_iq_is_full(lio->oct_dev, iq))
1804 		return 0;
1805 
1806 	if (__netif_subqueue_stopped(lio->netdev, q)) {
1807 		INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq, tx_restart, 1);
1808 		wake_q(lio->netdev, q);
1809 	}
1810 	return 1;
1811 }
1812 
1813 /**
1814  * \brief Unmap and free network buffer
1815  * @param buf buffer
1816  */
1817 static void free_netbuf(void *buf)
1818 {
1819 	struct sk_buff *skb;
1820 	struct octnet_buf_free_info *finfo;
1821 	struct lio *lio;
1822 
1823 	finfo = (struct octnet_buf_free_info *)buf;
1824 	skb = finfo->skb;
1825 	lio = finfo->lio;
1826 
1827 	dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1828 			 DMA_TO_DEVICE);
1829 
1830 	check_txq_state(lio, skb);
1831 
1832 	tx_buffer_free(skb);
1833 }
1834 
1835 /**
1836  * \brief Unmap and free gather buffer
1837  * @param buf buffer
1838  */
1839 static void free_netsgbuf(void *buf)
1840 {
1841 	struct octnet_buf_free_info *finfo;
1842 	struct sk_buff *skb;
1843 	struct lio *lio;
1844 	struct octnic_gather *g;
1845 	int i, frags, iq;
1846 
1847 	finfo = (struct octnet_buf_free_info *)buf;
1848 	skb = finfo->skb;
1849 	lio = finfo->lio;
1850 	g = finfo->g;
1851 	frags = skb_shinfo(skb)->nr_frags;
1852 
1853 	dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1854 			 g->sg[0].ptr[0], (skb->len - skb->data_len),
1855 			 DMA_TO_DEVICE);
1856 
1857 	i = 1;
1858 	while (frags--) {
1859 		struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1860 
1861 		pci_unmap_page((lio->oct_dev)->pci_dev,
1862 			       g->sg[(i >> 2)].ptr[(i & 3)],
1863 			       frag->size, DMA_TO_DEVICE);
1864 		i++;
1865 	}
1866 
1867 	dma_sync_single_for_cpu(&lio->oct_dev->pci_dev->dev,
1868 				g->sg_dma_ptr, g->sg_size, DMA_TO_DEVICE);
1869 
1870 	iq = skb_iq(lio, skb);
1871 	spin_lock(&lio->glist_lock[iq]);
1872 	list_add_tail(&g->list, &lio->glist[iq]);
1873 	spin_unlock(&lio->glist_lock[iq]);
1874 
1875 	check_txq_state(lio, skb);     /* mq support: sub-queue state check */
1876 
1877 	tx_buffer_free(skb);
1878 }
1879 
1880 /**
1881  * \brief Unmap and free gather buffer with response
1882  * @param buf buffer
1883  */
1884 static void free_netsgbuf_with_resp(void *buf)
1885 {
1886 	struct octeon_soft_command *sc;
1887 	struct octnet_buf_free_info *finfo;
1888 	struct sk_buff *skb;
1889 	struct lio *lio;
1890 	struct octnic_gather *g;
1891 	int i, frags, iq;
1892 
1893 	sc = (struct octeon_soft_command *)buf;
1894 	skb = (struct sk_buff *)sc->callback_arg;
1895 	finfo = (struct octnet_buf_free_info *)&skb->cb;
1896 
1897 	lio = finfo->lio;
1898 	g = finfo->g;
1899 	frags = skb_shinfo(skb)->nr_frags;
1900 
1901 	dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1902 			 g->sg[0].ptr[0], (skb->len - skb->data_len),
1903 			 DMA_TO_DEVICE);
1904 
1905 	i = 1;
1906 	while (frags--) {
1907 		struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1908 
1909 		pci_unmap_page((lio->oct_dev)->pci_dev,
1910 			       g->sg[(i >> 2)].ptr[(i & 3)],
1911 			       frag->size, DMA_TO_DEVICE);
1912 		i++;
1913 	}
1914 
1915 	dma_sync_single_for_cpu(&lio->oct_dev->pci_dev->dev,
1916 				g->sg_dma_ptr, g->sg_size, DMA_TO_DEVICE);
1917 
1918 	iq = skb_iq(lio, skb);
1919 
1920 	spin_lock(&lio->glist_lock[iq]);
1921 	list_add_tail(&g->list, &lio->glist[iq]);
1922 	spin_unlock(&lio->glist_lock[iq]);
1923 
1924 	/* Don't free the skb yet */
1925 
1926 	check_txq_state(lio, skb);
1927 }
1928 
1929 /**
1930  * \brief Adjust ptp frequency
1931  * @param ptp PTP clock info
1932  * @param ppb how much to adjust by, in parts-per-billion
1933  */
1934 static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
1935 {
1936 	struct lio *lio = container_of(ptp, struct lio, ptp_info);
1937 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1938 	u64 comp, delta;
1939 	unsigned long flags;
1940 	bool neg_adj = false;
1941 
1942 	if (ppb < 0) {
1943 		neg_adj = true;
1944 		ppb = -ppb;
1945 	}
1946 
1947 	/* The hardware adds the clock compensation value to the
1948 	 * PTP clock on every coprocessor clock cycle, so we
1949 	 * compute the delta in terms of coprocessor clocks.
1950 	 */
1951 	delta = (u64)ppb << 32;
1952 	do_div(delta, oct->coproc_clock_rate);
1953 
1954 	spin_lock_irqsave(&lio->ptp_lock, flags);
1955 	comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1956 	if (neg_adj)
1957 		comp -= delta;
1958 	else
1959 		comp += delta;
1960 	lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1961 	spin_unlock_irqrestore(&lio->ptp_lock, flags);
1962 
1963 	return 0;
1964 }
1965 
1966 /**
1967  * \brief Adjust ptp time
1968  * @param ptp PTP clock info
1969  * @param delta how much to adjust by, in nanosecs
1970  */
1971 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1972 {
1973 	unsigned long flags;
1974 	struct lio *lio = container_of(ptp, struct lio, ptp_info);
1975 
1976 	spin_lock_irqsave(&lio->ptp_lock, flags);
1977 	lio->ptp_adjust += delta;
1978 	spin_unlock_irqrestore(&lio->ptp_lock, flags);
1979 
1980 	return 0;
1981 }
1982 
1983 /**
1984  * \brief Get hardware clock time, including any adjustment
1985  * @param ptp PTP clock info
1986  * @param ts timespec
1987  */
1988 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1989 				struct timespec64 *ts)
1990 {
1991 	u64 ns;
1992 	unsigned long flags;
1993 	struct lio *lio = container_of(ptp, struct lio, ptp_info);
1994 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1995 
1996 	spin_lock_irqsave(&lio->ptp_lock, flags);
1997 	ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1998 	ns += lio->ptp_adjust;
1999 	spin_unlock_irqrestore(&lio->ptp_lock, flags);
2000 
2001 	*ts = ns_to_timespec64(ns);
2002 
2003 	return 0;
2004 }
2005 
2006 /**
2007  * \brief Set hardware clock time. Reset adjustment
2008  * @param ptp PTP clock info
2009  * @param ts timespec
2010  */
2011 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
2012 				const struct timespec64 *ts)
2013 {
2014 	u64 ns;
2015 	unsigned long flags;
2016 	struct lio *lio = container_of(ptp, struct lio, ptp_info);
2017 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
2018 
2019 	ns = timespec_to_ns(ts);
2020 
2021 	spin_lock_irqsave(&lio->ptp_lock, flags);
2022 	lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
2023 	lio->ptp_adjust = 0;
2024 	spin_unlock_irqrestore(&lio->ptp_lock, flags);
2025 
2026 	return 0;
2027 }
2028 
2029 /**
2030  * \brief Check if PTP is enabled
2031  * @param ptp PTP clock info
2032  * @param rq request
2033  * @param on is it on
2034  */
2035 static int
2036 liquidio_ptp_enable(struct ptp_clock_info *ptp __attribute__((unused)),
2037 		    struct ptp_clock_request *rq __attribute__((unused)),
2038 		    int on __attribute__((unused)))
2039 {
2040 	return -EOPNOTSUPP;
2041 }
2042 
2043 /**
2044  * \brief Open PTP clock source
2045  * @param netdev network device
2046  */
2047 static void oct_ptp_open(struct net_device *netdev)
2048 {
2049 	struct lio *lio = GET_LIO(netdev);
2050 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
2051 
2052 	spin_lock_init(&lio->ptp_lock);
2053 
2054 	snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
2055 	lio->ptp_info.owner = THIS_MODULE;
2056 	lio->ptp_info.max_adj = 250000000;
2057 	lio->ptp_info.n_alarm = 0;
2058 	lio->ptp_info.n_ext_ts = 0;
2059 	lio->ptp_info.n_per_out = 0;
2060 	lio->ptp_info.pps = 0;
2061 	lio->ptp_info.adjfreq = liquidio_ptp_adjfreq;
2062 	lio->ptp_info.adjtime = liquidio_ptp_adjtime;
2063 	lio->ptp_info.gettime64 = liquidio_ptp_gettime;
2064 	lio->ptp_info.settime64 = liquidio_ptp_settime;
2065 	lio->ptp_info.enable = liquidio_ptp_enable;
2066 
2067 	lio->ptp_adjust = 0;
2068 
2069 	lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
2070 					     &oct->pci_dev->dev);
2071 
2072 	if (IS_ERR(lio->ptp_clock))
2073 		lio->ptp_clock = NULL;
2074 }
2075 
2076 /**
2077  * \brief Init PTP clock
2078  * @param oct octeon device
2079  */
2080 static void liquidio_ptp_init(struct octeon_device *oct)
2081 {
2082 	u64 clock_comp, cfg;
2083 
2084 	clock_comp = (u64)NSEC_PER_SEC << 32;
2085 	do_div(clock_comp, oct->coproc_clock_rate);
2086 	lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
2087 
2088 	/* Enable */
2089 	cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
2090 	lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
2091 }
2092 
2093 /**
2094  * \brief Load firmware to device
2095  * @param oct octeon device
2096  *
2097  * Maps device to firmware filename, requests firmware, and downloads it
2098  */
2099 static int load_firmware(struct octeon_device *oct)
2100 {
2101 	int ret = 0;
2102 	const struct firmware *fw;
2103 	char fw_name[LIO_MAX_FW_FILENAME_LEN];
2104 	char *tmp_fw_type;
2105 
2106 	if (strncmp(fw_type, LIO_FW_NAME_TYPE_NONE,
2107 		    sizeof(LIO_FW_NAME_TYPE_NONE)) == 0) {
2108 		dev_info(&oct->pci_dev->dev, "Skipping firmware load\n");
2109 		return ret;
2110 	}
2111 
2112 	if (fw_type[0] == '\0')
2113 		tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
2114 	else
2115 		tmp_fw_type = fw_type;
2116 
2117 	sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,
2118 		octeon_get_conf(oct)->card_name, tmp_fw_type,
2119 		LIO_FW_NAME_SUFFIX);
2120 
2121 	ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);
2122 	if (ret) {
2123 		dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n.",
2124 			fw_name);
2125 		release_firmware(fw);
2126 		return ret;
2127 	}
2128 
2129 	ret = octeon_download_firmware(oct, fw->data, fw->size);
2130 
2131 	release_firmware(fw);
2132 
2133 	return ret;
2134 }
2135 
2136 /**
2137  * \brief Setup output queue
2138  * @param oct octeon device
2139  * @param q_no which queue
2140  * @param num_descs how many descriptors
2141  * @param desc_size size of each descriptor
2142  * @param app_ctx application context
2143  */
2144 static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
2145 			     int desc_size, void *app_ctx)
2146 {
2147 	int ret_val = 0;
2148 
2149 	dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no);
2150 	/* droq creation and local register settings. */
2151 	ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx);
2152 	if (ret_val < 0)
2153 		return ret_val;
2154 
2155 	if (ret_val == 1) {
2156 		dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no);
2157 		return 0;
2158 	}
2159 	/* tasklet creation for the droq */
2160 
2161 	/* Enable the droq queues */
2162 	octeon_set_droq_pkt_op(oct, q_no, 1);
2163 
2164 	/* Send Credit for Octeon Output queues. Credits are always
2165 	 * sent after the output queue is enabled.
2166 	 */
2167 	writel(oct->droq[q_no]->max_count,
2168 	       oct->droq[q_no]->pkts_credit_reg);
2169 
2170 	return ret_val;
2171 }
2172 
2173 /**
2174  * \brief Callback for getting interface configuration
2175  * @param status status of request
2176  * @param buf pointer to resp structure
2177  */
2178 static void if_cfg_callback(struct octeon_device *oct,
2179 			    u32 status __attribute__((unused)),
2180 			    void *buf)
2181 {
2182 	struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
2183 	struct liquidio_if_cfg_resp *resp;
2184 	struct liquidio_if_cfg_context *ctx;
2185 
2186 	resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
2187 	ctx = (struct liquidio_if_cfg_context *)sc->ctxptr;
2188 
2189 	oct = lio_get_device(ctx->octeon_id);
2190 	if (resp->status)
2191 		dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: %llx\n",
2192 			CVM_CAST64(resp->status));
2193 	WRITE_ONCE(ctx->cond, 1);
2194 
2195 	snprintf(oct->fw_info.liquidio_firmware_version, 32, "%s",
2196 		 resp->cfg_info.liquidio_firmware_version);
2197 
2198 	/* This barrier is required to be sure that the response has been
2199 	 * written fully before waking up the handler
2200 	 */
2201 	wmb();
2202 
2203 	wake_up_interruptible(&ctx->wc);
2204 }
2205 
2206 /**
2207  * \brief Select queue based on hash
2208  * @param dev Net device
2209  * @param skb sk_buff structure
2210  * @returns selected queue number
2211  */
2212 static u16 select_q(struct net_device *dev, struct sk_buff *skb,
2213 		    void *accel_priv __attribute__((unused)),
2214 		    select_queue_fallback_t fallback __attribute__((unused)))
2215 {
2216 	u32 qindex = 0;
2217 	struct lio *lio;
2218 
2219 	lio = GET_LIO(dev);
2220 	qindex = skb_tx_hash(dev, skb);
2221 
2222 	return (u16)(qindex % (lio->linfo.num_txpciq));
2223 }
2224 
2225 /** Routine to push packets arriving on Octeon interface upto network layer.
2226  * @param oct_id   - octeon device id.
2227  * @param skbuff   - skbuff struct to be passed to network layer.
2228  * @param len      - size of total data received.
2229  * @param rh       - Control header associated with the packet
2230  * @param param    - additional control data with the packet
2231  * @param arg	   - farg registered in droq_ops
2232  */
2233 static void
2234 liquidio_push_packet(u32 octeon_id __attribute__((unused)),
2235 		     void *skbuff,
2236 		     u32 len,
2237 		     union octeon_rh *rh,
2238 		     void *param,
2239 		     void *arg)
2240 {
2241 	struct napi_struct *napi = param;
2242 	struct sk_buff *skb = (struct sk_buff *)skbuff;
2243 	struct skb_shared_hwtstamps *shhwtstamps;
2244 	u64 ns;
2245 	u16 vtag = 0;
2246 	struct net_device *netdev = (struct net_device *)arg;
2247 	struct octeon_droq *droq = container_of(param, struct octeon_droq,
2248 						napi);
2249 	if (netdev) {
2250 		int packet_was_received;
2251 		struct lio *lio = GET_LIO(netdev);
2252 		struct octeon_device *oct = lio->oct_dev;
2253 
2254 		/* Do not proceed if the interface is not in RUNNING state. */
2255 		if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
2256 			recv_buffer_free(skb);
2257 			droq->stats.rx_dropped++;
2258 			return;
2259 		}
2260 
2261 		skb->dev = netdev;
2262 
2263 		skb_record_rx_queue(skb, droq->q_no);
2264 		if (likely(len > MIN_SKB_SIZE)) {
2265 			struct octeon_skb_page_info *pg_info;
2266 			unsigned char *va;
2267 
2268 			pg_info = ((struct octeon_skb_page_info *)(skb->cb));
2269 			if (pg_info->page) {
2270 				/* For Paged allocation use the frags */
2271 				va = page_address(pg_info->page) +
2272 					pg_info->page_offset;
2273 				memcpy(skb->data, va, MIN_SKB_SIZE);
2274 				skb_put(skb, MIN_SKB_SIZE);
2275 				skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
2276 						pg_info->page,
2277 						pg_info->page_offset +
2278 						MIN_SKB_SIZE,
2279 						len - MIN_SKB_SIZE,
2280 						LIO_RXBUFFER_SZ);
2281 			}
2282 		} else {
2283 			struct octeon_skb_page_info *pg_info =
2284 				((struct octeon_skb_page_info *)(skb->cb));
2285 			skb_copy_to_linear_data(skb, page_address(pg_info->page)
2286 						+ pg_info->page_offset, len);
2287 			skb_put(skb, len);
2288 			put_page(pg_info->page);
2289 		}
2290 
2291 		if (((oct->chip_id == OCTEON_CN66XX) ||
2292 		     (oct->chip_id == OCTEON_CN68XX)) &&
2293 		    ptp_enable) {
2294 			if (rh->r_dh.has_hwtstamp) {
2295 				/* timestamp is included from the hardware at
2296 				 * the beginning of the packet.
2297 				 */
2298 				if (ifstate_check
2299 				    (lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
2300 					/* Nanoseconds are in the first 64-bits
2301 					 * of the packet.
2302 					 */
2303 					memcpy(&ns, (skb->data), sizeof(ns));
2304 					shhwtstamps = skb_hwtstamps(skb);
2305 					shhwtstamps->hwtstamp =
2306 						ns_to_ktime(ns +
2307 							    lio->ptp_adjust);
2308 				}
2309 				skb_pull(skb, sizeof(ns));
2310 			}
2311 		}
2312 
2313 		skb->protocol = eth_type_trans(skb, skb->dev);
2314 		if ((netdev->features & NETIF_F_RXCSUM) &&
2315 		    (((rh->r_dh.encap_on) &&
2316 		      (rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) ||
2317 		     (!(rh->r_dh.encap_on) &&
2318 		      (rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED))))
2319 			/* checksum has already been verified */
2320 			skb->ip_summed = CHECKSUM_UNNECESSARY;
2321 		else
2322 			skb->ip_summed = CHECKSUM_NONE;
2323 
2324 		/* Setting Encapsulation field on basis of status received
2325 		 * from the firmware
2326 		 */
2327 		if (rh->r_dh.encap_on) {
2328 			skb->encapsulation = 1;
2329 			skb->csum_level = 1;
2330 			droq->stats.rx_vxlan++;
2331 		}
2332 
2333 		/* inbound VLAN tag */
2334 		if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
2335 		    (rh->r_dh.vlan != 0)) {
2336 			u16 vid = rh->r_dh.vlan;
2337 			u16 priority = rh->r_dh.priority;
2338 
2339 			vtag = priority << 13 | vid;
2340 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
2341 		}
2342 
2343 		packet_was_received = napi_gro_receive(napi, skb) != GRO_DROP;
2344 
2345 		if (packet_was_received) {
2346 			droq->stats.rx_bytes_received += len;
2347 			droq->stats.rx_pkts_received++;
2348 			netdev->last_rx = jiffies;
2349 		} else {
2350 			droq->stats.rx_dropped++;
2351 			netif_info(lio, rx_err, lio->netdev,
2352 				   "droq:%d  error rx_dropped:%llu\n",
2353 				   droq->q_no, droq->stats.rx_dropped);
2354 		}
2355 
2356 	} else {
2357 		recv_buffer_free(skb);
2358 	}
2359 }
2360 
2361 /**
2362  * \brief wrapper for calling napi_schedule
2363  * @param param parameters to pass to napi_schedule
2364  *
2365  * Used when scheduling on different CPUs
2366  */
2367 static void napi_schedule_wrapper(void *param)
2368 {
2369 	struct napi_struct *napi = param;
2370 
2371 	napi_schedule(napi);
2372 }
2373 
2374 /**
2375  * \brief callback when receive interrupt occurs and we are in NAPI mode
2376  * @param arg pointer to octeon output queue
2377  */
2378 static void liquidio_napi_drv_callback(void *arg)
2379 {
2380 	struct octeon_device *oct;
2381 	struct octeon_droq *droq = arg;
2382 	int this_cpu = smp_processor_id();
2383 
2384 	oct = droq->oct_dev;
2385 
2386 	if (OCTEON_CN23XX_PF(oct) || droq->cpu_id == this_cpu) {
2387 		napi_schedule_irqoff(&droq->napi);
2388 	} else {
2389 		struct call_single_data *csd = &droq->csd;
2390 
2391 		csd->func = napi_schedule_wrapper;
2392 		csd->info = &droq->napi;
2393 		csd->flags = 0;
2394 
2395 		smp_call_function_single_async(droq->cpu_id, csd);
2396 	}
2397 }
2398 
2399 /**
2400  * \brief Entry point for NAPI polling
2401  * @param napi NAPI structure
2402  * @param budget maximum number of items to process
2403  */
2404 static int liquidio_napi_poll(struct napi_struct *napi, int budget)
2405 {
2406 	struct octeon_droq *droq;
2407 	int work_done;
2408 	int tx_done = 0, iq_no;
2409 	struct octeon_instr_queue *iq;
2410 	struct octeon_device *oct;
2411 
2412 	droq = container_of(napi, struct octeon_droq, napi);
2413 	oct = droq->oct_dev;
2414 	iq_no = droq->q_no;
2415 	/* Handle Droq descriptors */
2416 	work_done = octeon_process_droq_poll_cmd(oct, droq->q_no,
2417 						 POLL_EVENT_PROCESS_PKTS,
2418 						 budget);
2419 
2420 	/* Flush the instruction queue */
2421 	iq = oct->instr_queue[iq_no];
2422 	if (iq) {
2423 		/* Process iq buffers with in the budget limits */
2424 		tx_done = octeon_flush_iq(oct, iq, 1, budget);
2425 		/* Update iq read-index rather than waiting for next interrupt.
2426 		 * Return back if tx_done is false.
2427 		 */
2428 		update_txq_status(oct, iq_no);
2429 		/*tx_done = (iq->flush_index == iq->octeon_read_index);*/
2430 	} else {
2431 		dev_err(&oct->pci_dev->dev, "%s:  iq (%d) num invalid\n",
2432 			__func__, iq_no);
2433 	}
2434 
2435 	if ((work_done < budget) && (tx_done)) {
2436 		napi_complete(napi);
2437 		octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no,
2438 					     POLL_EVENT_ENABLE_INTR, 0);
2439 		return 0;
2440 	}
2441 
2442 	return (!tx_done) ? (budget) : (work_done);
2443 }
2444 
2445 /**
2446  * \brief Setup input and output queues
2447  * @param octeon_dev octeon device
2448  * @param ifidx  Interface Index
2449  *
2450  * Note: Queues are with respect to the octeon device. Thus
2451  * an input queue is for egress packets, and output queues
2452  * are for ingress packets.
2453  */
2454 static inline int setup_io_queues(struct octeon_device *octeon_dev,
2455 				  int ifidx)
2456 {
2457 	struct octeon_droq_ops droq_ops;
2458 	struct net_device *netdev;
2459 	static int cpu_id;
2460 	static int cpu_id_modulus;
2461 	struct octeon_droq *droq;
2462 	struct napi_struct *napi;
2463 	int q, q_no, retval = 0;
2464 	struct lio *lio;
2465 	int num_tx_descs;
2466 
2467 	netdev = octeon_dev->props[ifidx].netdev;
2468 
2469 	lio = GET_LIO(netdev);
2470 
2471 	memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
2472 
2473 	droq_ops.fptr = liquidio_push_packet;
2474 	droq_ops.farg = (void *)netdev;
2475 
2476 	droq_ops.poll_mode = 1;
2477 	droq_ops.napi_fn = liquidio_napi_drv_callback;
2478 	cpu_id = 0;
2479 	cpu_id_modulus = num_present_cpus();
2480 
2481 	/* set up DROQs. */
2482 	for (q = 0; q < lio->linfo.num_rxpciq; q++) {
2483 		q_no = lio->linfo.rxpciq[q].s.q_no;
2484 		dev_dbg(&octeon_dev->pci_dev->dev,
2485 			"setup_io_queues index:%d linfo.rxpciq.s.q_no:%d\n",
2486 			q, q_no);
2487 		retval = octeon_setup_droq(octeon_dev, q_no,
2488 					   CFG_GET_NUM_RX_DESCS_NIC_IF
2489 						   (octeon_get_conf(octeon_dev),
2490 						   lio->ifidx),
2491 					   CFG_GET_NUM_RX_BUF_SIZE_NIC_IF
2492 						   (octeon_get_conf(octeon_dev),
2493 						   lio->ifidx), NULL);
2494 		if (retval) {
2495 			dev_err(&octeon_dev->pci_dev->dev,
2496 				"%s : Runtime DROQ(RxQ) creation failed.\n",
2497 				__func__);
2498 			return 1;
2499 		}
2500 
2501 		droq = octeon_dev->droq[q_no];
2502 		napi = &droq->napi;
2503 		dev_dbg(&octeon_dev->pci_dev->dev, "netif_napi_add netdev:%llx oct:%llx pf_num:%d\n",
2504 			(u64)netdev, (u64)octeon_dev, octeon_dev->pf_num);
2505 		netif_napi_add(netdev, napi, liquidio_napi_poll, 64);
2506 
2507 		/* designate a CPU for this droq */
2508 		droq->cpu_id = cpu_id;
2509 		cpu_id++;
2510 		if (cpu_id >= cpu_id_modulus)
2511 			cpu_id = 0;
2512 
2513 		octeon_register_droq_ops(octeon_dev, q_no, &droq_ops);
2514 	}
2515 
2516 	if (OCTEON_CN23XX_PF(octeon_dev)) {
2517 		/* 23XX PF can receive control messages (via the first PF-owned
2518 		 * droq) from the firmware even if the ethX interface is down,
2519 		 * so that's why poll_mode must be off for the first droq.
2520 		 */
2521 		octeon_dev->droq[0]->ops.poll_mode = 0;
2522 	}
2523 
2524 	/* set up IQs. */
2525 	for (q = 0; q < lio->linfo.num_txpciq; q++) {
2526 		num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf
2527 							   (octeon_dev),
2528 							   lio->ifidx);
2529 		retval = octeon_setup_iq(octeon_dev, ifidx, q,
2530 					 lio->linfo.txpciq[q], num_tx_descs,
2531 					 netdev_get_tx_queue(netdev, q));
2532 		if (retval) {
2533 			dev_err(&octeon_dev->pci_dev->dev,
2534 				" %s : Runtime IQ(TxQ) creation failed.\n",
2535 				__func__);
2536 			return 1;
2537 		}
2538 	}
2539 
2540 	return 0;
2541 }
2542 
2543 /**
2544  * \brief Poll routine for checking transmit queue status
2545  * @param work work_struct data structure
2546  */
2547 static void octnet_poll_check_txq_status(struct work_struct *work)
2548 {
2549 	struct cavium_wk *wk = (struct cavium_wk *)work;
2550 	struct lio *lio = (struct lio *)wk->ctxptr;
2551 
2552 	if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
2553 		return;
2554 
2555 	check_txq_status(lio);
2556 	queue_delayed_work(lio->txq_status_wq.wq,
2557 			   &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2558 }
2559 
2560 /**
2561  * \brief Sets up the txq poll check
2562  * @param netdev network device
2563  */
2564 static inline int setup_tx_poll_fn(struct net_device *netdev)
2565 {
2566 	struct lio *lio = GET_LIO(netdev);
2567 	struct octeon_device *oct = lio->oct_dev;
2568 
2569 	lio->txq_status_wq.wq = alloc_workqueue("txq-status",
2570 						WQ_MEM_RECLAIM, 0);
2571 	if (!lio->txq_status_wq.wq) {
2572 		dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
2573 		return -1;
2574 	}
2575 	INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
2576 			  octnet_poll_check_txq_status);
2577 	lio->txq_status_wq.wk.ctxptr = lio;
2578 	queue_delayed_work(lio->txq_status_wq.wq,
2579 			   &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2580 	return 0;
2581 }
2582 
2583 static inline void cleanup_tx_poll_fn(struct net_device *netdev)
2584 {
2585 	struct lio *lio = GET_LIO(netdev);
2586 
2587 	if (lio->txq_status_wq.wq) {
2588 		cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
2589 		destroy_workqueue(lio->txq_status_wq.wq);
2590 	}
2591 }
2592 
2593 /**
2594  * \brief Net device open for LiquidIO
2595  * @param netdev network device
2596  */
2597 static int liquidio_open(struct net_device *netdev)
2598 {
2599 	struct lio *lio = GET_LIO(netdev);
2600 	struct octeon_device *oct = lio->oct_dev;
2601 	struct napi_struct *napi, *n;
2602 
2603 	if (oct->props[lio->ifidx].napi_enabled == 0) {
2604 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
2605 			napi_enable(napi);
2606 
2607 		oct->props[lio->ifidx].napi_enabled = 1;
2608 
2609 		if (OCTEON_CN23XX_PF(oct))
2610 			oct->droq[0]->ops.poll_mode = 1;
2611 	}
2612 
2613 	oct_ptp_open(netdev);
2614 
2615 	ifstate_set(lio, LIO_IFSTATE_RUNNING);
2616 
2617 	/* Ready for link status updates */
2618 	lio->intf_open = 1;
2619 
2620 	netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
2621 
2622 	if (OCTEON_CN23XX_PF(oct)) {
2623 		if (!oct->msix_on)
2624 			if (setup_tx_poll_fn(netdev))
2625 				return -1;
2626 	} else {
2627 		if (setup_tx_poll_fn(netdev))
2628 			return -1;
2629 	}
2630 
2631 	start_txq(netdev);
2632 
2633 	/* tell Octeon to start forwarding packets to host */
2634 	send_rx_ctrl_cmd(lio, 1);
2635 
2636 	dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
2637 		 netdev->name);
2638 
2639 	return 0;
2640 }
2641 
2642 /**
2643  * \brief Net device stop for LiquidIO
2644  * @param netdev network device
2645  */
2646 static int liquidio_stop(struct net_device *netdev)
2647 {
2648 	struct lio *lio = GET_LIO(netdev);
2649 	struct octeon_device *oct = lio->oct_dev;
2650 
2651 	ifstate_reset(lio, LIO_IFSTATE_RUNNING);
2652 
2653 	netif_tx_disable(netdev);
2654 
2655 	/* Inform that netif carrier is down */
2656 	netif_carrier_off(netdev);
2657 	lio->intf_open = 0;
2658 	lio->linfo.link.s.link_up = 0;
2659 	lio->link_changes++;
2660 
2661 	/* Pause for a moment and wait for Octeon to flush out (to the wire) any
2662 	 * egress packets that are in-flight.
2663 	 */
2664 	set_current_state(TASK_INTERRUPTIBLE);
2665 	schedule_timeout(msecs_to_jiffies(100));
2666 
2667 	/* Now it should be safe to tell Octeon that nic interface is down. */
2668 	send_rx_ctrl_cmd(lio, 0);
2669 
2670 	if (OCTEON_CN23XX_PF(oct)) {
2671 		if (!oct->msix_on)
2672 			cleanup_tx_poll_fn(netdev);
2673 	} else {
2674 		cleanup_tx_poll_fn(netdev);
2675 	}
2676 
2677 	if (lio->ptp_clock) {
2678 		ptp_clock_unregister(lio->ptp_clock);
2679 		lio->ptp_clock = NULL;
2680 	}
2681 
2682 	dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
2683 
2684 	return 0;
2685 }
2686 
2687 /**
2688  * \brief Converts a mask based on net device flags
2689  * @param netdev network device
2690  *
2691  * This routine generates a octnet_ifflags mask from the net device flags
2692  * received from the OS.
2693  */
2694 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
2695 {
2696 	enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
2697 
2698 	if (netdev->flags & IFF_PROMISC)
2699 		f |= OCTNET_IFFLAG_PROMISC;
2700 
2701 	if (netdev->flags & IFF_ALLMULTI)
2702 		f |= OCTNET_IFFLAG_ALLMULTI;
2703 
2704 	if (netdev->flags & IFF_MULTICAST) {
2705 		f |= OCTNET_IFFLAG_MULTICAST;
2706 
2707 		/* Accept all multicast addresses if there are more than we
2708 		 * can handle
2709 		 */
2710 		if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
2711 			f |= OCTNET_IFFLAG_ALLMULTI;
2712 	}
2713 
2714 	if (netdev->flags & IFF_BROADCAST)
2715 		f |= OCTNET_IFFLAG_BROADCAST;
2716 
2717 	return f;
2718 }
2719 
2720 /**
2721  * \brief Net device set_multicast_list
2722  * @param netdev network device
2723  */
2724 static void liquidio_set_mcast_list(struct net_device *netdev)
2725 {
2726 	struct lio *lio = GET_LIO(netdev);
2727 	struct octeon_device *oct = lio->oct_dev;
2728 	struct octnic_ctrl_pkt nctrl;
2729 	struct netdev_hw_addr *ha;
2730 	u64 *mc;
2731 	int ret;
2732 	int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
2733 
2734 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2735 
2736 	/* Create a ctrl pkt command to be sent to core app. */
2737 	nctrl.ncmd.u64 = 0;
2738 	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
2739 	nctrl.ncmd.s.param1 = get_new_flags(netdev);
2740 	nctrl.ncmd.s.param2 = mc_count;
2741 	nctrl.ncmd.s.more = mc_count;
2742 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2743 	nctrl.netpndev = (u64)netdev;
2744 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2745 
2746 	/* copy all the addresses into the udd */
2747 	mc = &nctrl.udd[0];
2748 	netdev_for_each_mc_addr(ha, netdev) {
2749 		*mc = 0;
2750 		memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
2751 		/* no need to swap bytes */
2752 
2753 		if (++mc > &nctrl.udd[mc_count])
2754 			break;
2755 	}
2756 
2757 	/* Apparently, any activity in this call from the kernel has to
2758 	 * be atomic. So we won't wait for response.
2759 	 */
2760 	nctrl.wait_time = 0;
2761 
2762 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2763 	if (ret < 0) {
2764 		dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2765 			ret);
2766 	}
2767 }
2768 
2769 /**
2770  * \brief Net device set_mac_address
2771  * @param netdev network device
2772  */
2773 static int liquidio_set_mac(struct net_device *netdev, void *p)
2774 {
2775 	int ret = 0;
2776 	struct lio *lio = GET_LIO(netdev);
2777 	struct octeon_device *oct = lio->oct_dev;
2778 	struct sockaddr *addr = (struct sockaddr *)p;
2779 	struct octnic_ctrl_pkt nctrl;
2780 
2781 	if (!is_valid_ether_addr(addr->sa_data))
2782 		return -EADDRNOTAVAIL;
2783 
2784 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2785 
2786 	nctrl.ncmd.u64 = 0;
2787 	nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2788 	nctrl.ncmd.s.param1 = 0;
2789 	nctrl.ncmd.s.more = 1;
2790 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2791 	nctrl.netpndev = (u64)netdev;
2792 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2793 	nctrl.wait_time = 100;
2794 
2795 	nctrl.udd[0] = 0;
2796 	/* The MAC Address is presented in network byte order. */
2797 	memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2798 
2799 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2800 	if (ret < 0) {
2801 		dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2802 		return -ENOMEM;
2803 	}
2804 	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2805 	memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2806 
2807 	return 0;
2808 }
2809 
2810 /**
2811  * \brief Net device get_stats
2812  * @param netdev network device
2813  */
2814 static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
2815 {
2816 	struct lio *lio = GET_LIO(netdev);
2817 	struct net_device_stats *stats = &netdev->stats;
2818 	struct octeon_device *oct;
2819 	u64 pkts = 0, drop = 0, bytes = 0;
2820 	struct oct_droq_stats *oq_stats;
2821 	struct oct_iq_stats *iq_stats;
2822 	int i, iq_no, oq_no;
2823 
2824 	oct = lio->oct_dev;
2825 
2826 	for (i = 0; i < lio->linfo.num_txpciq; i++) {
2827 		iq_no = lio->linfo.txpciq[i].s.q_no;
2828 		iq_stats = &oct->instr_queue[iq_no]->stats;
2829 		pkts += iq_stats->tx_done;
2830 		drop += iq_stats->tx_dropped;
2831 		bytes += iq_stats->tx_tot_bytes;
2832 	}
2833 
2834 	stats->tx_packets = pkts;
2835 	stats->tx_bytes = bytes;
2836 	stats->tx_dropped = drop;
2837 
2838 	pkts = 0;
2839 	drop = 0;
2840 	bytes = 0;
2841 
2842 	for (i = 0; i < lio->linfo.num_rxpciq; i++) {
2843 		oq_no = lio->linfo.rxpciq[i].s.q_no;
2844 		oq_stats = &oct->droq[oq_no]->stats;
2845 		pkts += oq_stats->rx_pkts_received;
2846 		drop += (oq_stats->rx_dropped +
2847 			 oq_stats->dropped_nodispatch +
2848 			 oq_stats->dropped_toomany +
2849 			 oq_stats->dropped_nomem);
2850 		bytes += oq_stats->rx_bytes_received;
2851 	}
2852 
2853 	stats->rx_bytes = bytes;
2854 	stats->rx_packets = pkts;
2855 	stats->rx_dropped = drop;
2856 
2857 	return stats;
2858 }
2859 
2860 /**
2861  * \brief Net device change_mtu
2862  * @param netdev network device
2863  */
2864 static int liquidio_change_mtu(struct net_device *netdev, int new_mtu)
2865 {
2866 	struct lio *lio = GET_LIO(netdev);
2867 	struct octeon_device *oct = lio->oct_dev;
2868 	struct octnic_ctrl_pkt nctrl;
2869 	int ret = 0;
2870 
2871 	/* Limit the MTU to make sure the ethernet packets are between 68 bytes
2872 	 * and 16000 bytes
2873 	 */
2874 	if ((new_mtu < LIO_MIN_MTU_SIZE) ||
2875 	    (new_mtu > LIO_MAX_MTU_SIZE)) {
2876 		dev_err(&oct->pci_dev->dev, "Invalid MTU: %d\n", new_mtu);
2877 		dev_err(&oct->pci_dev->dev, "Valid range %d and %d\n",
2878 			LIO_MIN_MTU_SIZE, LIO_MAX_MTU_SIZE);
2879 		return -EINVAL;
2880 	}
2881 
2882 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2883 
2884 	nctrl.ncmd.u64 = 0;
2885 	nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MTU;
2886 	nctrl.ncmd.s.param1 = new_mtu;
2887 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2888 	nctrl.wait_time = 100;
2889 	nctrl.netpndev = (u64)netdev;
2890 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2891 
2892 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2893 	if (ret < 0) {
2894 		dev_err(&oct->pci_dev->dev, "Failed to set MTU\n");
2895 		return -1;
2896 	}
2897 
2898 	lio->mtu = new_mtu;
2899 
2900 	return 0;
2901 }
2902 
2903 /**
2904  * \brief Handler for SIOCSHWTSTAMP ioctl
2905  * @param netdev network device
2906  * @param ifr interface request
2907  * @param cmd command
2908  */
2909 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
2910 {
2911 	struct hwtstamp_config conf;
2912 	struct lio *lio = GET_LIO(netdev);
2913 
2914 	if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2915 		return -EFAULT;
2916 
2917 	if (conf.flags)
2918 		return -EINVAL;
2919 
2920 	switch (conf.tx_type) {
2921 	case HWTSTAMP_TX_ON:
2922 	case HWTSTAMP_TX_OFF:
2923 		break;
2924 	default:
2925 		return -ERANGE;
2926 	}
2927 
2928 	switch (conf.rx_filter) {
2929 	case HWTSTAMP_FILTER_NONE:
2930 		break;
2931 	case HWTSTAMP_FILTER_ALL:
2932 	case HWTSTAMP_FILTER_SOME:
2933 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2934 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2935 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2936 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2937 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2938 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2939 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2940 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2941 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2942 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
2943 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
2944 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2945 		conf.rx_filter = HWTSTAMP_FILTER_ALL;
2946 		break;
2947 	default:
2948 		return -ERANGE;
2949 	}
2950 
2951 	if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2952 		ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2953 
2954 	else
2955 		ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2956 
2957 	return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2958 }
2959 
2960 /**
2961  * \brief ioctl handler
2962  * @param netdev network device
2963  * @param ifr interface request
2964  * @param cmd command
2965  */
2966 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2967 {
2968 	switch (cmd) {
2969 	case SIOCSHWTSTAMP:
2970 		return hwtstamp_ioctl(netdev, ifr);
2971 	default:
2972 		return -EOPNOTSUPP;
2973 	}
2974 }
2975 
2976 /**
2977  * \brief handle a Tx timestamp response
2978  * @param status response status
2979  * @param buf pointer to skb
2980  */
2981 static void handle_timestamp(struct octeon_device *oct,
2982 			     u32 status,
2983 			     void *buf)
2984 {
2985 	struct octnet_buf_free_info *finfo;
2986 	struct octeon_soft_command *sc;
2987 	struct oct_timestamp_resp *resp;
2988 	struct lio *lio;
2989 	struct sk_buff *skb = (struct sk_buff *)buf;
2990 
2991 	finfo = (struct octnet_buf_free_info *)skb->cb;
2992 	lio = finfo->lio;
2993 	sc = finfo->sc;
2994 	oct = lio->oct_dev;
2995 	resp = (struct oct_timestamp_resp *)sc->virtrptr;
2996 
2997 	if (status != OCTEON_REQUEST_DONE) {
2998 		dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2999 			CVM_CAST64(status));
3000 		resp->timestamp = 0;
3001 	}
3002 
3003 	octeon_swap_8B_data(&resp->timestamp, 1);
3004 
3005 	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
3006 		struct skb_shared_hwtstamps ts;
3007 		u64 ns = resp->timestamp;
3008 
3009 		netif_info(lio, tx_done, lio->netdev,
3010 			   "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
3011 			   skb, (unsigned long long)ns);
3012 		ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
3013 		skb_tstamp_tx(skb, &ts);
3014 	}
3015 
3016 	octeon_free_soft_command(oct, sc);
3017 	tx_buffer_free(skb);
3018 }
3019 
3020 /* \brief Send a data packet that will be timestamped
3021  * @param oct octeon device
3022  * @param ndata pointer to network data
3023  * @param finfo pointer to private network data
3024  */
3025 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
3026 					 struct octnic_data_pkt *ndata,
3027 					 struct octnet_buf_free_info *finfo)
3028 {
3029 	int retval;
3030 	struct octeon_soft_command *sc;
3031 	struct lio *lio;
3032 	int ring_doorbell;
3033 	u32 len;
3034 
3035 	lio = finfo->lio;
3036 
3037 	sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
3038 					    sizeof(struct oct_timestamp_resp));
3039 	finfo->sc = sc;
3040 
3041 	if (!sc) {
3042 		dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
3043 		return IQ_SEND_FAILED;
3044 	}
3045 
3046 	if (ndata->reqtype == REQTYPE_NORESP_NET)
3047 		ndata->reqtype = REQTYPE_RESP_NET;
3048 	else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
3049 		ndata->reqtype = REQTYPE_RESP_NET_SG;
3050 
3051 	sc->callback = handle_timestamp;
3052 	sc->callback_arg = finfo->skb;
3053 	sc->iq_no = ndata->q_no;
3054 
3055 	if (OCTEON_CN23XX_PF(oct))
3056 		len = (u32)((struct octeon_instr_ih3 *)
3057 			    (&sc->cmd.cmd3.ih3))->dlengsz;
3058 	else
3059 		len = (u32)((struct octeon_instr_ih2 *)
3060 			    (&sc->cmd.cmd2.ih2))->dlengsz;
3061 
3062 	ring_doorbell = 1;
3063 
3064 	retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
3065 				     sc, len, ndata->reqtype);
3066 
3067 	if (retval == IQ_SEND_FAILED) {
3068 		dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
3069 			retval);
3070 		octeon_free_soft_command(oct, sc);
3071 	} else {
3072 		netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
3073 	}
3074 
3075 	return retval;
3076 }
3077 
3078 /** \brief Transmit networks packets to the Octeon interface
3079  * @param skbuff   skbuff struct to be passed to network layer.
3080  * @param netdev    pointer to network device
3081  * @returns whether the packet was transmitted to the device okay or not
3082  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
3083  */
3084 static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
3085 {
3086 	struct lio *lio;
3087 	struct octnet_buf_free_info *finfo;
3088 	union octnic_cmd_setup cmdsetup;
3089 	struct octnic_data_pkt ndata;
3090 	struct octeon_device *oct;
3091 	struct oct_iq_stats *stats;
3092 	struct octeon_instr_irh *irh;
3093 	union tx_info *tx_info;
3094 	int status = 0;
3095 	int q_idx = 0, iq_no = 0;
3096 	int j;
3097 	u64 dptr = 0;
3098 	u32 tag = 0;
3099 
3100 	lio = GET_LIO(netdev);
3101 	oct = lio->oct_dev;
3102 
3103 	if (netif_is_multiqueue(netdev)) {
3104 		q_idx = skb->queue_mapping;
3105 		q_idx = (q_idx % (lio->linfo.num_txpciq));
3106 		tag = q_idx;
3107 		iq_no = lio->linfo.txpciq[q_idx].s.q_no;
3108 	} else {
3109 		iq_no = lio->txq;
3110 	}
3111 
3112 	stats = &oct->instr_queue[iq_no]->stats;
3113 
3114 	/* Check for all conditions in which the current packet cannot be
3115 	 * transmitted.
3116 	 */
3117 	if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
3118 	    (!lio->linfo.link.s.link_up) ||
3119 	    (skb->len <= 0)) {
3120 		netif_info(lio, tx_err, lio->netdev,
3121 			   "Transmit failed link_status : %d\n",
3122 			   lio->linfo.link.s.link_up);
3123 		goto lio_xmit_failed;
3124 	}
3125 
3126 	/* Use space in skb->cb to store info used to unmap and
3127 	 * free the buffers.
3128 	 */
3129 	finfo = (struct octnet_buf_free_info *)skb->cb;
3130 	finfo->lio = lio;
3131 	finfo->skb = skb;
3132 	finfo->sc = NULL;
3133 
3134 	/* Prepare the attributes for the data to be passed to OSI. */
3135 	memset(&ndata, 0, sizeof(struct octnic_data_pkt));
3136 
3137 	ndata.buf = (void *)finfo;
3138 
3139 	ndata.q_no = iq_no;
3140 
3141 	if (netif_is_multiqueue(netdev)) {
3142 		if (octnet_iq_is_full(oct, ndata.q_no)) {
3143 			/* defer sending if queue is full */
3144 			netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
3145 				   ndata.q_no);
3146 			stats->tx_iq_busy++;
3147 			return NETDEV_TX_BUSY;
3148 		}
3149 	} else {
3150 		if (octnet_iq_is_full(oct, lio->txq)) {
3151 			/* defer sending if queue is full */
3152 			stats->tx_iq_busy++;
3153 			netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
3154 				   lio->txq);
3155 			return NETDEV_TX_BUSY;
3156 		}
3157 	}
3158 	/* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",
3159 	 *	lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
3160 	 */
3161 
3162 	ndata.datasize = skb->len;
3163 
3164 	cmdsetup.u64 = 0;
3165 	cmdsetup.s.iq_no = iq_no;
3166 
3167 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
3168 		if (skb->encapsulation) {
3169 			cmdsetup.s.tnl_csum = 1;
3170 			stats->tx_vxlan++;
3171 		} else {
3172 			cmdsetup.s.transport_csum = 1;
3173 		}
3174 	}
3175 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
3176 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3177 		cmdsetup.s.timestamp = 1;
3178 	}
3179 
3180 	if (skb_shinfo(skb)->nr_frags == 0) {
3181 		cmdsetup.s.u.datasize = skb->len;
3182 		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
3183 
3184 		/* Offload checksum calculation for TCP/UDP packets */
3185 		dptr = dma_map_single(&oct->pci_dev->dev,
3186 				      skb->data,
3187 				      skb->len,
3188 				      DMA_TO_DEVICE);
3189 		if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
3190 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
3191 				__func__);
3192 			return NETDEV_TX_BUSY;
3193 		}
3194 
3195 		if (OCTEON_CN23XX_PF(oct))
3196 			ndata.cmd.cmd3.dptr = dptr;
3197 		else
3198 			ndata.cmd.cmd2.dptr = dptr;
3199 		finfo->dptr = dptr;
3200 		ndata.reqtype = REQTYPE_NORESP_NET;
3201 
3202 	} else {
3203 		int i, frags;
3204 		struct skb_frag_struct *frag;
3205 		struct octnic_gather *g;
3206 
3207 		spin_lock(&lio->glist_lock[q_idx]);
3208 		g = (struct octnic_gather *)
3209 			list_delete_head(&lio->glist[q_idx]);
3210 		spin_unlock(&lio->glist_lock[q_idx]);
3211 
3212 		if (!g) {
3213 			netif_info(lio, tx_err, lio->netdev,
3214 				   "Transmit scatter gather: glist null!\n");
3215 			goto lio_xmit_failed;
3216 		}
3217 
3218 		cmdsetup.s.gather = 1;
3219 		cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
3220 		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
3221 
3222 		memset(g->sg, 0, g->sg_size);
3223 
3224 		g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
3225 						 skb->data,
3226 						 (skb->len - skb->data_len),
3227 						 DMA_TO_DEVICE);
3228 		if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
3229 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
3230 				__func__);
3231 			return NETDEV_TX_BUSY;
3232 		}
3233 		add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
3234 
3235 		frags = skb_shinfo(skb)->nr_frags;
3236 		i = 1;
3237 		while (frags--) {
3238 			frag = &skb_shinfo(skb)->frags[i - 1];
3239 
3240 			g->sg[(i >> 2)].ptr[(i & 3)] =
3241 				dma_map_page(&oct->pci_dev->dev,
3242 					     frag->page.p,
3243 					     frag->page_offset,
3244 					     frag->size,
3245 					     DMA_TO_DEVICE);
3246 
3247 			if (dma_mapping_error(&oct->pci_dev->dev,
3248 					      g->sg[i >> 2].ptr[i & 3])) {
3249 				dma_unmap_single(&oct->pci_dev->dev,
3250 						 g->sg[0].ptr[0],
3251 						 skb->len - skb->data_len,
3252 						 DMA_TO_DEVICE);
3253 				for (j = 1; j < i; j++) {
3254 					frag = &skb_shinfo(skb)->frags[j - 1];
3255 					dma_unmap_page(&oct->pci_dev->dev,
3256 						       g->sg[j >> 2].ptr[j & 3],
3257 						       frag->size,
3258 						       DMA_TO_DEVICE);
3259 				}
3260 				dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
3261 					__func__);
3262 				return NETDEV_TX_BUSY;
3263 			}
3264 
3265 			add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3));
3266 			i++;
3267 		}
3268 
3269 		dma_sync_single_for_device(&oct->pci_dev->dev, g->sg_dma_ptr,
3270 					   g->sg_size, DMA_TO_DEVICE);
3271 		dptr = g->sg_dma_ptr;
3272 
3273 		if (OCTEON_CN23XX_PF(oct))
3274 			ndata.cmd.cmd3.dptr = dptr;
3275 		else
3276 			ndata.cmd.cmd2.dptr = dptr;
3277 		finfo->dptr = dptr;
3278 		finfo->g = g;
3279 
3280 		ndata.reqtype = REQTYPE_NORESP_NET_SG;
3281 	}
3282 
3283 	if (OCTEON_CN23XX_PF(oct)) {
3284 		irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
3285 		tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];
3286 	} else {
3287 		irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
3288 		tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
3289 	}
3290 
3291 	if (skb_shinfo(skb)->gso_size) {
3292 		tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
3293 		tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
3294 		stats->tx_gso++;
3295 	}
3296 
3297 	/* HW insert VLAN tag */
3298 	if (skb_vlan_tag_present(skb)) {
3299 		irh->priority = skb_vlan_tag_get(skb) >> 13;
3300 		irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
3301 	}
3302 
3303 	if (unlikely(cmdsetup.s.timestamp))
3304 		status = send_nic_timestamp_pkt(oct, &ndata, finfo);
3305 	else
3306 		status = octnet_send_nic_data_pkt(oct, &ndata);
3307 	if (status == IQ_SEND_FAILED)
3308 		goto lio_xmit_failed;
3309 
3310 	netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
3311 
3312 	if (status == IQ_SEND_STOP)
3313 		stop_q(lio->netdev, q_idx);
3314 
3315 	netif_trans_update(netdev);
3316 
3317 	if (skb_shinfo(skb)->gso_size)
3318 		stats->tx_done += skb_shinfo(skb)->gso_segs;
3319 	else
3320 		stats->tx_done++;
3321 	stats->tx_tot_bytes += skb->len;
3322 
3323 	return NETDEV_TX_OK;
3324 
3325 lio_xmit_failed:
3326 	stats->tx_dropped++;
3327 	netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
3328 		   iq_no, stats->tx_dropped);
3329 	if (dptr)
3330 		dma_unmap_single(&oct->pci_dev->dev, dptr,
3331 				 ndata.datasize, DMA_TO_DEVICE);
3332 	tx_buffer_free(skb);
3333 	return NETDEV_TX_OK;
3334 }
3335 
3336 /** \brief Network device Tx timeout
3337  * @param netdev    pointer to network device
3338  */
3339 static void liquidio_tx_timeout(struct net_device *netdev)
3340 {
3341 	struct lio *lio;
3342 
3343 	lio = GET_LIO(netdev);
3344 
3345 	netif_info(lio, tx_err, lio->netdev,
3346 		   "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
3347 		   netdev->stats.tx_dropped);
3348 	netif_trans_update(netdev);
3349 	txqs_wake(netdev);
3350 }
3351 
3352 static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
3353 				    __be16 proto __attribute__((unused)),
3354 				    u16 vid)
3355 {
3356 	struct lio *lio = GET_LIO(netdev);
3357 	struct octeon_device *oct = lio->oct_dev;
3358 	struct octnic_ctrl_pkt nctrl;
3359 	int ret = 0;
3360 
3361 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3362 
3363 	nctrl.ncmd.u64 = 0;
3364 	nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
3365 	nctrl.ncmd.s.param1 = vid;
3366 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3367 	nctrl.wait_time = 100;
3368 	nctrl.netpndev = (u64)netdev;
3369 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3370 
3371 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3372 	if (ret < 0) {
3373 		dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
3374 			ret);
3375 	}
3376 
3377 	return ret;
3378 }
3379 
3380 static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
3381 				     __be16 proto __attribute__((unused)),
3382 				     u16 vid)
3383 {
3384 	struct lio *lio = GET_LIO(netdev);
3385 	struct octeon_device *oct = lio->oct_dev;
3386 	struct octnic_ctrl_pkt nctrl;
3387 	int ret = 0;
3388 
3389 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3390 
3391 	nctrl.ncmd.u64 = 0;
3392 	nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
3393 	nctrl.ncmd.s.param1 = vid;
3394 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3395 	nctrl.wait_time = 100;
3396 	nctrl.netpndev = (u64)netdev;
3397 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3398 
3399 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3400 	if (ret < 0) {
3401 		dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
3402 			ret);
3403 	}
3404 	return ret;
3405 }
3406 
3407 /** Sending command to enable/disable RX checksum offload
3408  * @param netdev                pointer to network device
3409  * @param command               OCTNET_CMD_TNL_RX_CSUM_CTL
3410  * @param rx_cmd_bit            OCTNET_CMD_RXCSUM_ENABLE/
3411  *                              OCTNET_CMD_RXCSUM_DISABLE
3412  * @returns                     SUCCESS or FAILURE
3413  */
3414 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
3415 				       u8 rx_cmd)
3416 {
3417 	struct lio *lio = GET_LIO(netdev);
3418 	struct octeon_device *oct = lio->oct_dev;
3419 	struct octnic_ctrl_pkt nctrl;
3420 	int ret = 0;
3421 
3422 	nctrl.ncmd.u64 = 0;
3423 	nctrl.ncmd.s.cmd = command;
3424 	nctrl.ncmd.s.param1 = rx_cmd;
3425 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3426 	nctrl.wait_time = 100;
3427 	nctrl.netpndev = (u64)netdev;
3428 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3429 
3430 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3431 	if (ret < 0) {
3432 		dev_err(&oct->pci_dev->dev,
3433 			"DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
3434 			ret);
3435 	}
3436 	return ret;
3437 }
3438 
3439 /** Sending command to add/delete VxLAN UDP port to firmware
3440  * @param netdev                pointer to network device
3441  * @param command               OCTNET_CMD_VXLAN_PORT_CONFIG
3442  * @param vxlan_port            VxLAN port to be added or deleted
3443  * @param vxlan_cmd_bit         OCTNET_CMD_VXLAN_PORT_ADD,
3444  *                              OCTNET_CMD_VXLAN_PORT_DEL
3445  * @returns                     SUCCESS or FAILURE
3446  */
3447 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
3448 				       u16 vxlan_port, u8 vxlan_cmd_bit)
3449 {
3450 	struct lio *lio = GET_LIO(netdev);
3451 	struct octeon_device *oct = lio->oct_dev;
3452 	struct octnic_ctrl_pkt nctrl;
3453 	int ret = 0;
3454 
3455 	nctrl.ncmd.u64 = 0;
3456 	nctrl.ncmd.s.cmd = command;
3457 	nctrl.ncmd.s.more = vxlan_cmd_bit;
3458 	nctrl.ncmd.s.param1 = vxlan_port;
3459 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3460 	nctrl.wait_time = 100;
3461 	nctrl.netpndev = (u64)netdev;
3462 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3463 
3464 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3465 	if (ret < 0) {
3466 		dev_err(&oct->pci_dev->dev,
3467 			"VxLAN port add/delete failed in core (ret:0x%x)\n",
3468 			ret);
3469 	}
3470 	return ret;
3471 }
3472 
3473 /** \brief Net device fix features
3474  * @param netdev  pointer to network device
3475  * @param request features requested
3476  * @returns updated features list
3477  */
3478 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
3479 					       netdev_features_t request)
3480 {
3481 	struct lio *lio = netdev_priv(netdev);
3482 
3483 	if ((request & NETIF_F_RXCSUM) &&
3484 	    !(lio->dev_capability & NETIF_F_RXCSUM))
3485 		request &= ~NETIF_F_RXCSUM;
3486 
3487 	if ((request & NETIF_F_HW_CSUM) &&
3488 	    !(lio->dev_capability & NETIF_F_HW_CSUM))
3489 		request &= ~NETIF_F_HW_CSUM;
3490 
3491 	if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
3492 		request &= ~NETIF_F_TSO;
3493 
3494 	if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
3495 		request &= ~NETIF_F_TSO6;
3496 
3497 	if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
3498 		request &= ~NETIF_F_LRO;
3499 
3500 	/*Disable LRO if RXCSUM is off */
3501 	if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
3502 	    (lio->dev_capability & NETIF_F_LRO))
3503 		request &= ~NETIF_F_LRO;
3504 
3505 	return request;
3506 }
3507 
3508 /** \brief Net device set features
3509  * @param netdev  pointer to network device
3510  * @param features features to enable/disable
3511  */
3512 static int liquidio_set_features(struct net_device *netdev,
3513 				 netdev_features_t features)
3514 {
3515 	struct lio *lio = netdev_priv(netdev);
3516 
3517 	if (!((netdev->features ^ features) & NETIF_F_LRO))
3518 		return 0;
3519 
3520 	if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO))
3521 		liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3522 				     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3523 	else if (!(features & NETIF_F_LRO) &&
3524 		 (lio->dev_capability & NETIF_F_LRO))
3525 		liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
3526 				     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3527 
3528 	/* Sending command to firmware to enable/disable RX checksum
3529 	 * offload settings using ethtool
3530 	 */
3531 	if (!(netdev->features & NETIF_F_RXCSUM) &&
3532 	    (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
3533 	    (features & NETIF_F_RXCSUM))
3534 		liquidio_set_rxcsum_command(netdev,
3535 					    OCTNET_CMD_TNL_RX_CSUM_CTL,
3536 					    OCTNET_CMD_RXCSUM_ENABLE);
3537 	else if ((netdev->features & NETIF_F_RXCSUM) &&
3538 		 (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
3539 		 !(features & NETIF_F_RXCSUM))
3540 		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3541 					    OCTNET_CMD_RXCSUM_DISABLE);
3542 
3543 	return 0;
3544 }
3545 
3546 static void liquidio_add_vxlan_port(struct net_device *netdev,
3547 				    struct udp_tunnel_info *ti)
3548 {
3549 	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3550 		return;
3551 
3552 	liquidio_vxlan_port_command(netdev,
3553 				    OCTNET_CMD_VXLAN_PORT_CONFIG,
3554 				    htons(ti->port),
3555 				    OCTNET_CMD_VXLAN_PORT_ADD);
3556 }
3557 
3558 static void liquidio_del_vxlan_port(struct net_device *netdev,
3559 				    struct udp_tunnel_info *ti)
3560 {
3561 	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3562 		return;
3563 
3564 	liquidio_vxlan_port_command(netdev,
3565 				    OCTNET_CMD_VXLAN_PORT_CONFIG,
3566 				    htons(ti->port),
3567 				    OCTNET_CMD_VXLAN_PORT_DEL);
3568 }
3569 
3570 static struct net_device_ops lionetdevops = {
3571 	.ndo_open		= liquidio_open,
3572 	.ndo_stop		= liquidio_stop,
3573 	.ndo_start_xmit		= liquidio_xmit,
3574 	.ndo_get_stats		= liquidio_get_stats,
3575 	.ndo_set_mac_address	= liquidio_set_mac,
3576 	.ndo_set_rx_mode	= liquidio_set_mcast_list,
3577 	.ndo_tx_timeout		= liquidio_tx_timeout,
3578 
3579 	.ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
3580 	.ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
3581 	.ndo_change_mtu		= liquidio_change_mtu,
3582 	.ndo_do_ioctl		= liquidio_ioctl,
3583 	.ndo_fix_features	= liquidio_fix_features,
3584 	.ndo_set_features	= liquidio_set_features,
3585 	.ndo_udp_tunnel_add	= liquidio_add_vxlan_port,
3586 	.ndo_udp_tunnel_del	= liquidio_del_vxlan_port,
3587 };
3588 
3589 /** \brief Entry point for the liquidio module
3590  */
3591 static int __init liquidio_init(void)
3592 {
3593 	int i;
3594 	struct handshake *hs;
3595 
3596 	init_completion(&first_stage);
3597 
3598 	octeon_init_device_list(conf_type);
3599 
3600 	if (liquidio_init_pci())
3601 		return -EINVAL;
3602 
3603 	wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3604 
3605 	for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3606 		hs = &handshake[i];
3607 		if (hs->pci_dev) {
3608 			wait_for_completion(&hs->init);
3609 			if (!hs->init_ok) {
3610 				/* init handshake failed */
3611 				dev_err(&hs->pci_dev->dev,
3612 					"Failed to init device\n");
3613 				liquidio_deinit_pci();
3614 				return -EIO;
3615 			}
3616 		}
3617 	}
3618 
3619 	for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3620 		hs = &handshake[i];
3621 		if (hs->pci_dev) {
3622 			wait_for_completion_timeout(&hs->started,
3623 						    msecs_to_jiffies(30000));
3624 			if (!hs->started_ok) {
3625 				/* starter handshake failed */
3626 				dev_err(&hs->pci_dev->dev,
3627 					"Firmware failed to start\n");
3628 				liquidio_deinit_pci();
3629 				return -EIO;
3630 			}
3631 		}
3632 	}
3633 
3634 	return 0;
3635 }
3636 
3637 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3638 {
3639 	struct octeon_device *oct = (struct octeon_device *)buf;
3640 	struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3641 	int gmxport = 0;
3642 	union oct_link_status *ls;
3643 	int i;
3644 
3645 	if (recv_pkt->buffer_size[0] != sizeof(*ls)) {
3646 		dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3647 			recv_pkt->buffer_size[0],
3648 			recv_pkt->rh.r_nic_info.gmxport);
3649 		goto nic_info_err;
3650 	}
3651 
3652 	gmxport = recv_pkt->rh.r_nic_info.gmxport;
3653 	ls = (union oct_link_status *)get_rbd(recv_pkt->buffer_ptr[0]);
3654 
3655 	octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3656 	for (i = 0; i < oct->ifcount; i++) {
3657 		if (oct->props[i].gmxport == gmxport) {
3658 			update_link_status(oct->props[i].netdev, ls);
3659 			break;
3660 		}
3661 	}
3662 
3663 nic_info_err:
3664 	for (i = 0; i < recv_pkt->buffer_count; i++)
3665 		recv_buffer_free(recv_pkt->buffer_ptr[i]);
3666 	octeon_free_recv_info(recv_info);
3667 	return 0;
3668 }
3669 
3670 /**
3671  * \brief Setup network interfaces
3672  * @param octeon_dev  octeon device
3673  *
3674  * Called during init time for each device. It assumes the NIC
3675  * is already up and running.  The link information for each
3676  * interface is passed in link_info.
3677  */
3678 static int setup_nic_devices(struct octeon_device *octeon_dev)
3679 {
3680 	struct lio *lio = NULL;
3681 	struct net_device *netdev;
3682 	u8 mac[6], i, j;
3683 	struct octeon_soft_command *sc;
3684 	struct liquidio_if_cfg_context *ctx;
3685 	struct liquidio_if_cfg_resp *resp;
3686 	struct octdev_props *props;
3687 	int retval, num_iqueues, num_oqueues;
3688 	union oct_nic_if_cfg if_cfg;
3689 	unsigned int base_queue;
3690 	unsigned int gmx_port_id;
3691 	u32 resp_size, ctx_size, data_size;
3692 	u32 ifidx_or_pfnum;
3693 	struct lio_version *vdata;
3694 
3695 	/* This is to handle link status changes */
3696 	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3697 				    OPCODE_NIC_INFO,
3698 				    lio_nic_info, octeon_dev);
3699 
3700 	/* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3701 	 * They are handled directly.
3702 	 */
3703 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3704 					free_netbuf);
3705 
3706 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3707 					free_netsgbuf);
3708 
3709 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3710 					free_netsgbuf_with_resp);
3711 
3712 	for (i = 0; i < octeon_dev->ifcount; i++) {
3713 		resp_size = sizeof(struct liquidio_if_cfg_resp);
3714 		ctx_size = sizeof(struct liquidio_if_cfg_context);
3715 		data_size = sizeof(struct lio_version);
3716 		sc = (struct octeon_soft_command *)
3717 			octeon_alloc_soft_command(octeon_dev, data_size,
3718 						  resp_size, ctx_size);
3719 		resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3720 		ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
3721 		vdata = (struct lio_version *)sc->virtdptr;
3722 
3723 		*((u64 *)vdata) = 0;
3724 		vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
3725 		vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
3726 		vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
3727 
3728 		if (OCTEON_CN23XX_PF(octeon_dev)) {
3729 			num_iqueues = octeon_dev->sriov_info.num_pf_rings;
3730 			num_oqueues = octeon_dev->sriov_info.num_pf_rings;
3731 			base_queue = octeon_dev->sriov_info.pf_srn;
3732 
3733 			gmx_port_id = octeon_dev->pf_num;
3734 			ifidx_or_pfnum = octeon_dev->pf_num;
3735 		} else {
3736 			num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(
3737 						octeon_get_conf(octeon_dev), i);
3738 			num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(
3739 						octeon_get_conf(octeon_dev), i);
3740 			base_queue = CFG_GET_BASE_QUE_NIC_IF(
3741 						octeon_get_conf(octeon_dev), i);
3742 			gmx_port_id = CFG_GET_GMXID_NIC_IF(
3743 						octeon_get_conf(octeon_dev), i);
3744 			ifidx_or_pfnum = i;
3745 		}
3746 
3747 		dev_dbg(&octeon_dev->pci_dev->dev,
3748 			"requesting config for interface %d, iqs %d, oqs %d\n",
3749 			ifidx_or_pfnum, num_iqueues, num_oqueues);
3750 		WRITE_ONCE(ctx->cond, 0);
3751 		ctx->octeon_id = lio_get_device_id(octeon_dev);
3752 		init_waitqueue_head(&ctx->wc);
3753 
3754 		if_cfg.u64 = 0;
3755 		if_cfg.s.num_iqueues = num_iqueues;
3756 		if_cfg.s.num_oqueues = num_oqueues;
3757 		if_cfg.s.base_queue = base_queue;
3758 		if_cfg.s.gmx_port_id = gmx_port_id;
3759 
3760 		sc->iq_no = 0;
3761 
3762 		octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
3763 					    OPCODE_NIC_IF_CFG, 0,
3764 					    if_cfg.u64, 0);
3765 
3766 		sc->callback = if_cfg_callback;
3767 		sc->callback_arg = sc;
3768 		sc->wait_time = 3000;
3769 
3770 		retval = octeon_send_soft_command(octeon_dev, sc);
3771 		if (retval == IQ_SEND_FAILED) {
3772 			dev_err(&octeon_dev->pci_dev->dev,
3773 				"iq/oq config failed status: %x\n",
3774 				retval);
3775 			/* Soft instr is freed by driver in case of failure. */
3776 			goto setup_nic_dev_fail;
3777 		}
3778 
3779 		/* Sleep on a wait queue till the cond flag indicates that the
3780 		 * response arrived or timed-out.
3781 		 */
3782 		if (sleep_cond(&ctx->wc, &ctx->cond) == -EINTR) {
3783 			dev_err(&octeon_dev->pci_dev->dev, "Wait interrupted\n");
3784 			goto setup_nic_wait_intr;
3785 		}
3786 
3787 		retval = resp->status;
3788 		if (retval) {
3789 			dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3790 			goto setup_nic_dev_fail;
3791 		}
3792 
3793 		octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3794 				    (sizeof(struct liquidio_if_cfg_info)) >> 3);
3795 
3796 		num_iqueues = hweight64(resp->cfg_info.iqmask);
3797 		num_oqueues = hweight64(resp->cfg_info.oqmask);
3798 
3799 		if (!(num_iqueues) || !(num_oqueues)) {
3800 			dev_err(&octeon_dev->pci_dev->dev,
3801 				"Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3802 				resp->cfg_info.iqmask,
3803 				resp->cfg_info.oqmask);
3804 			goto setup_nic_dev_fail;
3805 		}
3806 		dev_dbg(&octeon_dev->pci_dev->dev,
3807 			"interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
3808 			i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3809 			num_iqueues, num_oqueues);
3810 		netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues);
3811 
3812 		if (!netdev) {
3813 			dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3814 			goto setup_nic_dev_fail;
3815 		}
3816 
3817 		SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
3818 
3819 		if (num_iqueues > 1)
3820 			lionetdevops.ndo_select_queue = select_q;
3821 
3822 		/* Associate the routines that will handle different
3823 		 * netdev tasks.
3824 		 */
3825 		netdev->netdev_ops = &lionetdevops;
3826 
3827 		lio = GET_LIO(netdev);
3828 
3829 		memset(lio, 0, sizeof(struct lio));
3830 
3831 		lio->ifidx = ifidx_or_pfnum;
3832 
3833 		props = &octeon_dev->props[i];
3834 		props->gmxport = resp->cfg_info.linfo.gmxport;
3835 		props->netdev = netdev;
3836 
3837 		lio->linfo.num_rxpciq = num_oqueues;
3838 		lio->linfo.num_txpciq = num_iqueues;
3839 		for (j = 0; j < num_oqueues; j++) {
3840 			lio->linfo.rxpciq[j].u64 =
3841 				resp->cfg_info.linfo.rxpciq[j].u64;
3842 		}
3843 		for (j = 0; j < num_iqueues; j++) {
3844 			lio->linfo.txpciq[j].u64 =
3845 				resp->cfg_info.linfo.txpciq[j].u64;
3846 		}
3847 		lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3848 		lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3849 		lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3850 
3851 		lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3852 
3853 		if (OCTEON_CN23XX_PF(octeon_dev) ||
3854 		    OCTEON_CN6XXX(octeon_dev)) {
3855 			lio->dev_capability = NETIF_F_HIGHDMA
3856 					      | NETIF_F_IP_CSUM
3857 					      | NETIF_F_IPV6_CSUM
3858 					      | NETIF_F_SG | NETIF_F_RXCSUM
3859 					      | NETIF_F_GRO
3860 					      | NETIF_F_TSO | NETIF_F_TSO6
3861 					      | NETIF_F_LRO;
3862 		}
3863 		netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3864 
3865 		/*  Copy of transmit encapsulation capabilities:
3866 		 *  TSO, TSO6, Checksums for this device
3867 		 */
3868 		lio->enc_dev_capability = NETIF_F_IP_CSUM
3869 					  | NETIF_F_IPV6_CSUM
3870 					  | NETIF_F_GSO_UDP_TUNNEL
3871 					  | NETIF_F_HW_CSUM | NETIF_F_SG
3872 					  | NETIF_F_RXCSUM
3873 					  | NETIF_F_TSO | NETIF_F_TSO6
3874 					  | NETIF_F_LRO;
3875 
3876 		netdev->hw_enc_features = (lio->enc_dev_capability &
3877 					   ~NETIF_F_LRO);
3878 
3879 		lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;
3880 
3881 		netdev->vlan_features = lio->dev_capability;
3882 		/* Add any unchangeable hw features */
3883 		lio->dev_capability |=  NETIF_F_HW_VLAN_CTAG_FILTER |
3884 					NETIF_F_HW_VLAN_CTAG_RX |
3885 					NETIF_F_HW_VLAN_CTAG_TX;
3886 
3887 		netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
3888 
3889 		netdev->hw_features = lio->dev_capability;
3890 		/*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
3891 		netdev->hw_features = netdev->hw_features &
3892 			~NETIF_F_HW_VLAN_CTAG_RX;
3893 
3894 		/* Point to the  properties for octeon device to which this
3895 		 * interface belongs.
3896 		 */
3897 		lio->oct_dev = octeon_dev;
3898 		lio->octprops = props;
3899 		lio->netdev = netdev;
3900 
3901 		dev_dbg(&octeon_dev->pci_dev->dev,
3902 			"if%d gmx: %d hw_addr: 0x%llx\n", i,
3903 			lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3904 
3905 		/* 64-bit swap required on LE machines */
3906 		octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3907 		for (j = 0; j < 6; j++)
3908 			mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3909 
3910 		/* Copy MAC Address to OS network device structure */
3911 
3912 		ether_addr_copy(netdev->dev_addr, mac);
3913 
3914 		/* By default all interfaces on a single Octeon uses the same
3915 		 * tx and rx queues
3916 		 */
3917 		lio->txq = lio->linfo.txpciq[0].s.q_no;
3918 		lio->rxq = lio->linfo.rxpciq[0].s.q_no;
3919 		if (setup_io_queues(octeon_dev, i)) {
3920 			dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3921 			goto setup_nic_dev_fail;
3922 		}
3923 
3924 		ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3925 
3926 		lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3927 		lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3928 
3929 		if (setup_glists(octeon_dev, lio, num_iqueues)) {
3930 			dev_err(&octeon_dev->pci_dev->dev,
3931 				"Gather list allocation failed\n");
3932 			goto setup_nic_dev_fail;
3933 		}
3934 
3935 		/* Register ethtool support */
3936 		liquidio_set_ethtool_ops(netdev);
3937 		if (lio->oct_dev->chip_id == OCTEON_CN23XX_PF_VID)
3938 			octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;
3939 		else
3940 			octeon_dev->priv_flags = 0x0;
3941 
3942 		if (netdev->features & NETIF_F_LRO)
3943 			liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3944 					     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3945 
3946 		liquidio_set_feature(netdev, OCTNET_CMD_ENABLE_VLAN_FILTER, 0);
3947 
3948 		if ((debug != -1) && (debug & NETIF_MSG_HW))
3949 			liquidio_set_feature(netdev,
3950 					     OCTNET_CMD_VERBOSE_ENABLE, 0);
3951 
3952 		if (setup_link_status_change_wq(netdev))
3953 			goto setup_nic_dev_fail;
3954 
3955 		/* Register the network device with the OS */
3956 		if (register_netdev(netdev)) {
3957 			dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3958 			goto setup_nic_dev_fail;
3959 		}
3960 
3961 		dev_dbg(&octeon_dev->pci_dev->dev,
3962 			"Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3963 			i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3964 		netif_carrier_off(netdev);
3965 		lio->link_changes++;
3966 
3967 		ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3968 
3969 		/* Sending command to firmware to enable Rx checksum offload
3970 		 * by default at the time of setup of Liquidio driver for
3971 		 * this device
3972 		 */
3973 		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3974 					    OCTNET_CMD_RXCSUM_ENABLE);
3975 		liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
3976 				     OCTNET_CMD_TXCSUM_ENABLE);
3977 
3978 		dev_dbg(&octeon_dev->pci_dev->dev,
3979 			"NIC ifidx:%d Setup successful\n", i);
3980 
3981 		octeon_free_soft_command(octeon_dev, sc);
3982 	}
3983 
3984 	return 0;
3985 
3986 setup_nic_dev_fail:
3987 
3988 	octeon_free_soft_command(octeon_dev, sc);
3989 
3990 setup_nic_wait_intr:
3991 
3992 	while (i--) {
3993 		dev_err(&octeon_dev->pci_dev->dev,
3994 			"NIC ifidx:%d Setup failed\n", i);
3995 		liquidio_destroy_nic_device(octeon_dev, i);
3996 	}
3997 	return -ENODEV;
3998 }
3999 
4000 /**
4001  * \brief initialize the NIC
4002  * @param oct octeon device
4003  *
4004  * This initialization routine is called once the Octeon device application is
4005  * up and running
4006  */
4007 static int liquidio_init_nic_module(struct octeon_device *oct)
4008 {
4009 	struct oct_intrmod_cfg *intrmod_cfg;
4010 	int i, retval = 0;
4011 	int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
4012 
4013 	dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
4014 
4015 	/* only default iq and oq were initialized
4016 	 * initialize the rest as well
4017 	 */
4018 	/* run port_config command for each port */
4019 	oct->ifcount = num_nic_ports;
4020 
4021 	memset(oct->props, 0, sizeof(struct octdev_props) * num_nic_ports);
4022 
4023 	for (i = 0; i < MAX_OCTEON_LINKS; i++)
4024 		oct->props[i].gmxport = -1;
4025 
4026 	retval = setup_nic_devices(oct);
4027 	if (retval) {
4028 		dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
4029 		goto octnet_init_failure;
4030 	}
4031 
4032 	liquidio_ptp_init(oct);
4033 
4034 	/* Initialize interrupt moderation params */
4035 	intrmod_cfg = &((struct octeon_device *)oct)->intrmod;
4036 	intrmod_cfg->rx_enable = 1;
4037 	intrmod_cfg->check_intrvl = LIO_INTRMOD_CHECK_INTERVAL;
4038 	intrmod_cfg->maxpkt_ratethr = LIO_INTRMOD_MAXPKT_RATETHR;
4039 	intrmod_cfg->minpkt_ratethr = LIO_INTRMOD_MINPKT_RATETHR;
4040 	intrmod_cfg->rx_maxcnt_trigger = LIO_INTRMOD_RXMAXCNT_TRIGGER;
4041 	intrmod_cfg->rx_maxtmr_trigger = LIO_INTRMOD_RXMAXTMR_TRIGGER;
4042 	intrmod_cfg->rx_mintmr_trigger = LIO_INTRMOD_RXMINTMR_TRIGGER;
4043 	intrmod_cfg->rx_mincnt_trigger = LIO_INTRMOD_RXMINCNT_TRIGGER;
4044 	intrmod_cfg->tx_enable = 1;
4045 	intrmod_cfg->tx_maxcnt_trigger = LIO_INTRMOD_TXMAXCNT_TRIGGER;
4046 	intrmod_cfg->tx_mincnt_trigger = LIO_INTRMOD_TXMINCNT_TRIGGER;
4047 	intrmod_cfg->rx_frames = CFG_GET_OQ_INTR_PKT(octeon_get_conf(oct));
4048 	intrmod_cfg->rx_usecs = CFG_GET_OQ_INTR_TIME(octeon_get_conf(oct));
4049 	intrmod_cfg->tx_frames = CFG_GET_IQ_INTR_PKT(octeon_get_conf(oct));
4050 	dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
4051 
4052 	return retval;
4053 
4054 octnet_init_failure:
4055 
4056 	oct->ifcount = 0;
4057 
4058 	return retval;
4059 }
4060 
4061 /**
4062  * \brief starter callback that invokes the remaining initialization work after
4063  * the NIC is up and running.
4064  * @param octptr  work struct work_struct
4065  */
4066 static void nic_starter(struct work_struct *work)
4067 {
4068 	struct octeon_device *oct;
4069 	struct cavium_wk *wk = (struct cavium_wk *)work;
4070 
4071 	oct = (struct octeon_device *)wk->ctxptr;
4072 
4073 	if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
4074 		return;
4075 
4076 	/* If the status of the device is CORE_OK, the core
4077 	 * application has reported its application type. Call
4078 	 * any registered handlers now and move to the RUNNING
4079 	 * state.
4080 	 */
4081 	if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
4082 		schedule_delayed_work(&oct->nic_poll_work.work,
4083 				      LIQUIDIO_STARTER_POLL_INTERVAL_MS);
4084 		return;
4085 	}
4086 
4087 	atomic_set(&oct->status, OCT_DEV_RUNNING);
4088 
4089 	if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
4090 		dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
4091 
4092 		if (liquidio_init_nic_module(oct))
4093 			dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
4094 		else
4095 			handshake[oct->octeon_id].started_ok = 1;
4096 	} else {
4097 		dev_err(&oct->pci_dev->dev,
4098 			"Unexpected application running on NIC (%d). Check firmware.\n",
4099 			oct->app_mode);
4100 	}
4101 
4102 	complete(&handshake[oct->octeon_id].started);
4103 }
4104 
4105 /**
4106  * \brief Device initialization for each Octeon device that is probed
4107  * @param octeon_dev  octeon device
4108  */
4109 static int octeon_device_init(struct octeon_device *octeon_dev)
4110 {
4111 	int j, ret;
4112 	int fw_loaded = 0;
4113 	char bootcmd[] = "\n";
4114 	struct octeon_device_priv *oct_priv =
4115 		(struct octeon_device_priv *)octeon_dev->priv;
4116 	atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
4117 
4118 	/* Enable access to the octeon device and make its DMA capability
4119 	 * known to the OS.
4120 	 */
4121 	if (octeon_pci_os_setup(octeon_dev))
4122 		return 1;
4123 
4124 	/* Identify the Octeon type and map the BAR address space. */
4125 	if (octeon_chip_specific_setup(octeon_dev)) {
4126 		dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
4127 		return 1;
4128 	}
4129 
4130 	atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
4131 
4132 	octeon_dev->app_mode = CVM_DRV_INVALID_APP;
4133 
4134 	if (OCTEON_CN23XX_PF(octeon_dev)) {
4135 		if (!cn23xx_fw_loaded(octeon_dev)) {
4136 			fw_loaded = 0;
4137 			/* Do a soft reset of the Octeon device. */
4138 			if (octeon_dev->fn_list.soft_reset(octeon_dev))
4139 				return 1;
4140 			/* things might have changed */
4141 			if (!cn23xx_fw_loaded(octeon_dev))
4142 				fw_loaded = 0;
4143 			else
4144 				fw_loaded = 1;
4145 		} else {
4146 			fw_loaded = 1;
4147 		}
4148 	} else if (octeon_dev->fn_list.soft_reset(octeon_dev)) {
4149 		return 1;
4150 	}
4151 
4152 	/* Initialize the dispatch mechanism used to push packets arriving on
4153 	 * Octeon Output queues.
4154 	 */
4155 	if (octeon_init_dispatch_list(octeon_dev))
4156 		return 1;
4157 
4158 	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4159 				    OPCODE_NIC_CORE_DRV_ACTIVE,
4160 				    octeon_core_drv_init,
4161 				    octeon_dev);
4162 
4163 	INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
4164 	octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
4165 	schedule_delayed_work(&octeon_dev->nic_poll_work.work,
4166 			      LIQUIDIO_STARTER_POLL_INTERVAL_MS);
4167 
4168 	atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
4169 
4170 	octeon_set_io_queues_off(octeon_dev);
4171 
4172 	if (OCTEON_CN23XX_PF(octeon_dev)) {
4173 		ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4174 		if (ret) {
4175 			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Failed to configure device registers\n");
4176 			return ret;
4177 		}
4178 	}
4179 
4180 	/* Initialize soft command buffer pool
4181 	 */
4182 	if (octeon_setup_sc_buffer_pool(octeon_dev)) {
4183 		dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
4184 		return 1;
4185 	}
4186 	atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
4187 
4188 	/*  Setup the data structures that manage this Octeon's Input queues. */
4189 	if (octeon_setup_instr_queues(octeon_dev)) {
4190 		dev_err(&octeon_dev->pci_dev->dev,
4191 			"instruction queue initialization failed\n");
4192 		/* On error, release any previously allocated queues */
4193 		for (j = 0; j < octeon_dev->num_iqs; j++)
4194 			octeon_delete_instr_queue(octeon_dev, j);
4195 		return 1;
4196 	}
4197 	atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
4198 
4199 	/* Initialize lists to manage the requests of different types that
4200 	 * arrive from user & kernel applications for this octeon device.
4201 	 */
4202 	if (octeon_setup_response_list(octeon_dev)) {
4203 		dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
4204 		return 1;
4205 	}
4206 	atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
4207 
4208 	if (octeon_setup_output_queues(octeon_dev)) {
4209 		dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
4210 		/* Release any previously allocated queues */
4211 		for (j = 0; j < octeon_dev->num_oqs; j++)
4212 			octeon_delete_droq(octeon_dev, j);
4213 		return 1;
4214 	}
4215 
4216 	atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
4217 
4218 	if (OCTEON_CN23XX_PF(octeon_dev)) {
4219 		if (octeon_allocate_ioq_vector(octeon_dev)) {
4220 			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");
4221 			return 1;
4222 		}
4223 
4224 	} else {
4225 		/* The input and output queue registers were setup earlier (the
4226 		 * queues were not enabled). Any additional registers
4227 		 * that need to be programmed should be done now.
4228 		 */
4229 		ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4230 		if (ret) {
4231 			dev_err(&octeon_dev->pci_dev->dev,
4232 				"Failed to configure device registers\n");
4233 			return ret;
4234 		}
4235 	}
4236 
4237 	/* Initialize the tasklet that handles output queue packet processing.*/
4238 	dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
4239 	tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh,
4240 		     (unsigned long)octeon_dev);
4241 
4242 	/* Setup the interrupt handler and record the INT SUM register address
4243 	 */
4244 	if (octeon_setup_interrupt(octeon_dev))
4245 		return 1;
4246 
4247 	/* Enable Octeon device interrupts */
4248 	octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);
4249 
4250 	/* Enable the input and output queues for this Octeon device */
4251 	ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
4252 	if (ret) {
4253 		dev_err(&octeon_dev->pci_dev->dev, "Failed to enable input/output queues");
4254 		return ret;
4255 	}
4256 
4257 	atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
4258 
4259 	if ((!OCTEON_CN23XX_PF(octeon_dev)) || !fw_loaded) {
4260 		dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
4261 		if (!ddr_timeout) {
4262 			dev_info(&octeon_dev->pci_dev->dev,
4263 				 "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
4264 		}
4265 
4266 		schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
4267 
4268 		/* Wait for the octeon to initialize DDR after the soft-reset.*/
4269 		while (!ddr_timeout) {
4270 			set_current_state(TASK_INTERRUPTIBLE);
4271 			if (schedule_timeout(HZ / 10)) {
4272 				/* user probably pressed Control-C */
4273 				return 1;
4274 			}
4275 		}
4276 		ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
4277 		if (ret) {
4278 			dev_err(&octeon_dev->pci_dev->dev,
4279 				"DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
4280 				ret);
4281 			return 1;
4282 		}
4283 
4284 		if (octeon_wait_for_bootloader(octeon_dev, 1000)) {
4285 			dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
4286 			return 1;
4287 		}
4288 
4289 		/* Divert uboot to take commands from host instead. */
4290 		ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
4291 
4292 		dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
4293 		ret = octeon_init_consoles(octeon_dev);
4294 		if (ret) {
4295 			dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
4296 			return 1;
4297 		}
4298 		ret = octeon_add_console(octeon_dev, 0);
4299 		if (ret) {
4300 			dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
4301 			return 1;
4302 		}
4303 
4304 		atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
4305 
4306 		dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
4307 		ret = load_firmware(octeon_dev);
4308 		if (ret) {
4309 			dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
4310 			return 1;
4311 		}
4312 		/* set bit 1 of SLI_SCRATCH_1 to indicate that firmware is
4313 		 * loaded
4314 		 */
4315 		if (OCTEON_CN23XX_PF(octeon_dev))
4316 			octeon_write_csr64(octeon_dev, CN23XX_SLI_SCRATCH1,
4317 					   2ULL);
4318 	}
4319 
4320 	handshake[octeon_dev->octeon_id].init_ok = 1;
4321 	complete(&handshake[octeon_dev->octeon_id].init);
4322 
4323 	atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
4324 
4325 	/* Send Credit for Octeon Output queues. Credits are always sent after
4326 	 * the output queue is enabled.
4327 	 */
4328 	for (j = 0; j < octeon_dev->num_oqs; j++)
4329 		writel(octeon_dev->droq[j]->max_count,
4330 		       octeon_dev->droq[j]->pkts_credit_reg);
4331 
4332 	/* Packets can start arriving on the output queues from this point. */
4333 	return 0;
4334 }
4335 
4336 /**
4337  * \brief Exits the module
4338  */
4339 static void __exit liquidio_exit(void)
4340 {
4341 	liquidio_deinit_pci();
4342 
4343 	pr_info("LiquidIO network module is now unloaded\n");
4344 }
4345 
4346 module_init(liquidio_init);
4347 module_exit(liquidio_exit);
4348