1 /**********************************************************************
2  * Author: Cavium, Inc.
3  *
4  * Contact: support@cavium.com
5  *          Please include "LiquidIO" in the subject.
6  *
7  * Copyright (c) 2003-2016 Cavium, Inc.
8  *
9  * This file is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License, Version 2, as
11  * published by the Free Software Foundation.
12  *
13  * This file is distributed in the hope that it will be useful, but
14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16  * NONINFRINGEMENT.  See the GNU General Public License for more details.
17  ***********************************************************************/
18 #include <linux/module.h>
19 #include <linux/interrupt.h>
20 #include <linux/pci.h>
21 #include <net/vxlan.h>
22 #include "liquidio_common.h"
23 #include "octeon_droq.h"
24 #include "octeon_iq.h"
25 #include "response_manager.h"
26 #include "octeon_device.h"
27 #include "octeon_nic.h"
28 #include "octeon_main.h"
29 #include "octeon_network.h"
30 #include "cn23xx_vf_device.h"
31 
32 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
33 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Virtual Function Driver");
34 MODULE_LICENSE("GPL");
35 
36 static int debug = -1;
37 module_param(debug, int, 0644);
38 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
39 
40 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
41 
42 struct oct_timestamp_resp {
43 	u64 rh;
44 	u64 timestamp;
45 	u64 status;
46 };
47 
48 union tx_info {
49 	u64 u64;
50 	struct {
51 #ifdef __BIG_ENDIAN_BITFIELD
52 		u16 gso_size;
53 		u16 gso_segs;
54 		u32 reserved;
55 #else
56 		u32 reserved;
57 		u16 gso_segs;
58 		u16 gso_size;
59 #endif
60 	} s;
61 };
62 
63 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
64 #define OCTNIC_GSO_MAX_SIZE \
65 		(CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
66 
67 static int
68 liquidio_vf_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
69 static void liquidio_vf_remove(struct pci_dev *pdev);
70 static int octeon_device_init(struct octeon_device *oct);
71 static int liquidio_stop(struct net_device *netdev);
72 
73 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
74 {
75 	struct octeon_device_priv *oct_priv =
76 	    (struct octeon_device_priv *)oct->priv;
77 	int retry = MAX_IO_PENDING_PKT_COUNT;
78 	int pkt_cnt = 0, pending_pkts;
79 	int i;
80 
81 	do {
82 		pending_pkts = 0;
83 
84 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
85 			if (!(oct->io_qmask.oq & BIT_ULL(i)))
86 				continue;
87 			pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
88 		}
89 		if (pkt_cnt > 0) {
90 			pending_pkts += pkt_cnt;
91 			tasklet_schedule(&oct_priv->droq_tasklet);
92 		}
93 		pkt_cnt = 0;
94 		schedule_timeout_uninterruptible(1);
95 
96 	} while (retry-- && pending_pkts);
97 
98 	return pkt_cnt;
99 }
100 
101 /**
102  * pcierror_quiesce_device - Cause device to go quiet so it can be safely removed/reset/etc
103  * @oct: Pointer to Octeon device
104  */
105 static void pcierror_quiesce_device(struct octeon_device *oct)
106 {
107 	int i;
108 
109 	/* Disable the input and output queues now. No more packets will
110 	 * arrive from Octeon, but we should wait for all packet processing
111 	 * to finish.
112 	 */
113 
114 	/* To allow for in-flight requests */
115 	schedule_timeout_uninterruptible(100);
116 
117 	if (wait_for_pending_requests(oct))
118 		dev_err(&oct->pci_dev->dev, "There were pending requests\n");
119 
120 	/* Force all requests waiting to be fetched by OCTEON to complete. */
121 	for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
122 		struct octeon_instr_queue *iq;
123 
124 		if (!(oct->io_qmask.iq & BIT_ULL(i)))
125 			continue;
126 		iq = oct->instr_queue[i];
127 
128 		if (atomic_read(&iq->instr_pending)) {
129 			spin_lock_bh(&iq->lock);
130 			iq->fill_cnt = 0;
131 			iq->octeon_read_index = iq->host_write_index;
132 			iq->stats.instr_processed +=
133 			    atomic_read(&iq->instr_pending);
134 			lio_process_iq_request_list(oct, iq, 0);
135 			spin_unlock_bh(&iq->lock);
136 		}
137 	}
138 
139 	/* Force all pending ordered list requests to time out. */
140 	lio_process_ordered_list(oct, 1);
141 
142 	/* We do not need to wait for output queue packets to be processed. */
143 }
144 
145 /**
146  * cleanup_aer_uncorrect_error_status - Cleanup PCI AER uncorrectable error status
147  * @dev: Pointer to PCI device
148  */
149 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
150 {
151 	u32 status, mask;
152 	int pos = 0x100;
153 
154 	pr_info("%s :\n", __func__);
155 
156 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
157 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
158 	if (dev->error_state == pci_channel_io_normal)
159 		status &= ~mask; /* Clear corresponding nonfatal bits */
160 	else
161 		status &= mask; /* Clear corresponding fatal bits */
162 	pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
163 }
164 
165 /**
166  * stop_pci_io - Stop all PCI IO to a given device
167  * @oct: Pointer to Octeon device
168  */
169 static void stop_pci_io(struct octeon_device *oct)
170 {
171 	struct msix_entry *msix_entries;
172 	int i;
173 
174 	/* No more instructions will be forwarded. */
175 	atomic_set(&oct->status, OCT_DEV_IN_RESET);
176 
177 	for (i = 0; i < oct->ifcount; i++)
178 		netif_device_detach(oct->props[i].netdev);
179 
180 	/* Disable interrupts  */
181 	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
182 
183 	pcierror_quiesce_device(oct);
184 	if (oct->msix_on) {
185 		msix_entries = (struct msix_entry *)oct->msix_entries;
186 		for (i = 0; i < oct->num_msix_irqs; i++) {
187 			/* clear the affinity_cpumask */
188 			irq_set_affinity_hint(msix_entries[i].vector,
189 					      NULL);
190 			free_irq(msix_entries[i].vector,
191 				 &oct->ioq_vector[i]);
192 		}
193 		pci_disable_msix(oct->pci_dev);
194 		kfree(oct->msix_entries);
195 		oct->msix_entries = NULL;
196 		octeon_free_ioq_vector(oct);
197 	}
198 	dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
199 		lio_get_state_string(&oct->status));
200 
201 	/* making it a common function for all OCTEON models */
202 	cleanup_aer_uncorrect_error_status(oct->pci_dev);
203 
204 	pci_disable_device(oct->pci_dev);
205 }
206 
207 /**
208  * liquidio_pcie_error_detected - called when PCI error is detected
209  * @pdev: Pointer to PCI device
210  * @state: The current pci connection state
211  *
212  * This function is called after a PCI bus error affecting
213  * this device has been detected.
214  */
215 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
216 						     pci_channel_state_t state)
217 {
218 	struct octeon_device *oct = pci_get_drvdata(pdev);
219 
220 	/* Non-correctable Non-fatal errors */
221 	if (state == pci_channel_io_normal) {
222 		dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
223 		cleanup_aer_uncorrect_error_status(oct->pci_dev);
224 		return PCI_ERS_RESULT_CAN_RECOVER;
225 	}
226 
227 	/* Non-correctable Fatal errors */
228 	dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
229 	stop_pci_io(oct);
230 
231 	return PCI_ERS_RESULT_DISCONNECT;
232 }
233 
234 /* For PCI-E Advanced Error Recovery (AER) Interface */
235 static const struct pci_error_handlers liquidio_vf_err_handler = {
236 	.error_detected = liquidio_pcie_error_detected,
237 };
238 
239 static const struct pci_device_id liquidio_vf_pci_tbl[] = {
240 	{
241 		PCI_VENDOR_ID_CAVIUM, OCTEON_CN23XX_VF_VID,
242 		PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
243 	},
244 	{
245 		0, 0, 0, 0, 0, 0, 0
246 	}
247 };
248 MODULE_DEVICE_TABLE(pci, liquidio_vf_pci_tbl);
249 
250 static struct pci_driver liquidio_vf_pci_driver = {
251 	.name		= "LiquidIO_VF",
252 	.id_table	= liquidio_vf_pci_tbl,
253 	.probe		= liquidio_vf_probe,
254 	.remove		= liquidio_vf_remove,
255 	.err_handler	= &liquidio_vf_err_handler,    /* For AER */
256 };
257 
258 /**
259  * print_link_info - Print link information
260  * @netdev: network device
261  */
262 static void print_link_info(struct net_device *netdev)
263 {
264 	struct lio *lio = GET_LIO(netdev);
265 
266 	if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
267 	    ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
268 		struct oct_link_info *linfo = &lio->linfo;
269 
270 		if (linfo->link.s.link_up) {
271 			netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
272 				   linfo->link.s.speed,
273 				   (linfo->link.s.duplex) ? "Full" : "Half");
274 		} else {
275 			netif_info(lio, link, lio->netdev, "Link Down\n");
276 		}
277 	}
278 }
279 
280 /**
281  * octnet_link_status_change - Routine to notify MTU change
282  * @work: work_struct data structure
283  */
284 static void octnet_link_status_change(struct work_struct *work)
285 {
286 	struct cavium_wk *wk = (struct cavium_wk *)work;
287 	struct lio *lio = (struct lio *)wk->ctxptr;
288 
289 	/* lio->linfo.link.s.mtu always contains max MTU of the lio interface.
290 	 * this API is invoked only when new max-MTU of the interface is
291 	 * less than current MTU.
292 	 */
293 	rtnl_lock();
294 	dev_set_mtu(lio->netdev, lio->linfo.link.s.mtu);
295 	rtnl_unlock();
296 }
297 
298 /**
299  * setup_link_status_change_wq - Sets up the mtu status change work
300  * @netdev: network device
301  */
302 static int setup_link_status_change_wq(struct net_device *netdev)
303 {
304 	struct lio *lio = GET_LIO(netdev);
305 	struct octeon_device *oct = lio->oct_dev;
306 
307 	lio->link_status_wq.wq = alloc_workqueue("link-status",
308 						 WQ_MEM_RECLAIM, 0);
309 	if (!lio->link_status_wq.wq) {
310 		dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");
311 		return -1;
312 	}
313 	INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,
314 			  octnet_link_status_change);
315 	lio->link_status_wq.wk.ctxptr = lio;
316 
317 	return 0;
318 }
319 
320 static void cleanup_link_status_change_wq(struct net_device *netdev)
321 {
322 	struct lio *lio = GET_LIO(netdev);
323 
324 	if (lio->link_status_wq.wq) {
325 		cancel_delayed_work_sync(&lio->link_status_wq.wk.work);
326 		destroy_workqueue(lio->link_status_wq.wq);
327 	}
328 }
329 
330 /**
331  * update_link_status - Update link status
332  * @netdev: network device
333  * @ls: link status structure
334  *
335  * Called on receipt of a link status response from the core application to
336  * update each interface's link status.
337  */
338 static void update_link_status(struct net_device *netdev,
339 			       union oct_link_status *ls)
340 {
341 	struct lio *lio = GET_LIO(netdev);
342 	int current_max_mtu = lio->linfo.link.s.mtu;
343 	struct octeon_device *oct = lio->oct_dev;
344 
345 	if ((lio->intf_open) && (lio->linfo.link.u64 != ls->u64)) {
346 		lio->linfo.link.u64 = ls->u64;
347 
348 		print_link_info(netdev);
349 		lio->link_changes++;
350 
351 		if (lio->linfo.link.s.link_up) {
352 			netif_carrier_on(netdev);
353 			wake_txqs(netdev);
354 		} else {
355 			netif_carrier_off(netdev);
356 			stop_txqs(netdev);
357 		}
358 
359 		if (lio->linfo.link.s.mtu != current_max_mtu) {
360 			dev_info(&oct->pci_dev->dev,
361 				 "Max MTU Changed from %d to %d\n",
362 				 current_max_mtu, lio->linfo.link.s.mtu);
363 			netdev->max_mtu = lio->linfo.link.s.mtu;
364 		}
365 
366 		if (lio->linfo.link.s.mtu < netdev->mtu) {
367 			dev_warn(&oct->pci_dev->dev,
368 				 "Current MTU is higher than new max MTU; Reducing the current mtu from %d to %d\n",
369 				 netdev->mtu, lio->linfo.link.s.mtu);
370 			queue_delayed_work(lio->link_status_wq.wq,
371 					   &lio->link_status_wq.wk.work, 0);
372 		}
373 	}
374 }
375 
376 /**
377  * liquidio_vf_probe - PCI probe handler
378  * @pdev: PCI device structure
379  * @ent: unused
380  */
381 static int
382 liquidio_vf_probe(struct pci_dev *pdev,
383 		  const struct pci_device_id __maybe_unused *ent)
384 {
385 	struct octeon_device *oct_dev = NULL;
386 
387 	oct_dev = octeon_allocate_device(pdev->device,
388 					 sizeof(struct octeon_device_priv));
389 
390 	if (!oct_dev) {
391 		dev_err(&pdev->dev, "Unable to allocate device\n");
392 		return -ENOMEM;
393 	}
394 	oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
395 
396 	dev_info(&pdev->dev, "Initializing device %x:%x.\n",
397 		 (u32)pdev->vendor, (u32)pdev->device);
398 
399 	/* Assign octeon_device for this device to the private data area. */
400 	pci_set_drvdata(pdev, oct_dev);
401 
402 	/* set linux specific device pointer */
403 	oct_dev->pci_dev = pdev;
404 
405 	oct_dev->subsystem_id = pdev->subsystem_vendor |
406 		(pdev->subsystem_device << 16);
407 
408 	if (octeon_device_init(oct_dev)) {
409 		liquidio_vf_remove(pdev);
410 		return -ENOMEM;
411 	}
412 
413 	dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
414 
415 	return 0;
416 }
417 
418 /**
419  * octeon_pci_flr - PCI FLR for each Octeon device.
420  * @oct: octeon device
421  */
422 static void octeon_pci_flr(struct octeon_device *oct)
423 {
424 	pci_save_state(oct->pci_dev);
425 
426 	pci_cfg_access_lock(oct->pci_dev);
427 
428 	/* Quiesce the device completely */
429 	pci_write_config_word(oct->pci_dev, PCI_COMMAND,
430 			      PCI_COMMAND_INTX_DISABLE);
431 
432 	pcie_flr(oct->pci_dev);
433 
434 	pci_cfg_access_unlock(oct->pci_dev);
435 
436 	pci_restore_state(oct->pci_dev);
437 }
438 
439 /**
440  * octeon_destroy_resources - Destroy resources associated with octeon device
441  * @oct: octeon device
442  */
443 static void octeon_destroy_resources(struct octeon_device *oct)
444 {
445 	struct octeon_device_priv *oct_priv =
446 		(struct octeon_device_priv *)oct->priv;
447 	struct msix_entry *msix_entries;
448 	int i;
449 
450 	switch (atomic_read(&oct->status)) {
451 	case OCT_DEV_RUNNING:
452 	case OCT_DEV_CORE_OK:
453 		/* No more instructions will be forwarded. */
454 		atomic_set(&oct->status, OCT_DEV_IN_RESET);
455 
456 		oct->app_mode = CVM_DRV_INVALID_APP;
457 		dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
458 			lio_get_state_string(&oct->status));
459 
460 		schedule_timeout_uninterruptible(HZ / 10);
461 
462 		fallthrough;
463 	case OCT_DEV_HOST_OK:
464 	case OCT_DEV_IO_QUEUES_DONE:
465 		if (lio_wait_for_instr_fetch(oct))
466 			dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
467 
468 		if (wait_for_pending_requests(oct))
469 			dev_err(&oct->pci_dev->dev, "There were pending requests\n");
470 
471 		/* Disable the input and output queues now. No more packets will
472 		 * arrive from Octeon, but we should wait for all packet
473 		 * processing to finish.
474 		 */
475 		oct->fn_list.disable_io_queues(oct);
476 
477 		if (lio_wait_for_oq_pkts(oct))
478 			dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
479 
480 		/* Force all requests waiting to be fetched by OCTEON to
481 		 * complete.
482 		 */
483 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
484 			struct octeon_instr_queue *iq;
485 
486 			if (!(oct->io_qmask.iq & BIT_ULL(i)))
487 				continue;
488 			iq = oct->instr_queue[i];
489 
490 			if (atomic_read(&iq->instr_pending)) {
491 				spin_lock_bh(&iq->lock);
492 				iq->fill_cnt = 0;
493 				iq->octeon_read_index = iq->host_write_index;
494 				iq->stats.instr_processed +=
495 					atomic_read(&iq->instr_pending);
496 				lio_process_iq_request_list(oct, iq, 0);
497 				spin_unlock_bh(&iq->lock);
498 			}
499 		}
500 
501 		lio_process_ordered_list(oct, 1);
502 		octeon_free_sc_done_list(oct);
503 		octeon_free_sc_zombie_list(oct);
504 
505 		fallthrough;
506 	case OCT_DEV_INTR_SET_DONE:
507 		/* Disable interrupts  */
508 		oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
509 
510 		if (oct->msix_on) {
511 			msix_entries = (struct msix_entry *)oct->msix_entries;
512 			for (i = 0; i < oct->num_msix_irqs; i++) {
513 				if (oct->ioq_vector[i].vector) {
514 					irq_set_affinity_hint(
515 							msix_entries[i].vector,
516 							NULL);
517 					free_irq(msix_entries[i].vector,
518 						 &oct->ioq_vector[i]);
519 					oct->ioq_vector[i].vector = 0;
520 				}
521 			}
522 			pci_disable_msix(oct->pci_dev);
523 			kfree(oct->msix_entries);
524 			oct->msix_entries = NULL;
525 			kfree(oct->irq_name_storage);
526 			oct->irq_name_storage = NULL;
527 		}
528 		/* Soft reset the octeon device before exiting */
529 		if (!pcie_reset_flr(oct->pci_dev, PCI_RESET_PROBE))
530 			octeon_pci_flr(oct);
531 		else
532 			cn23xx_vf_ask_pf_to_do_flr(oct);
533 
534 		fallthrough;
535 	case OCT_DEV_MSIX_ALLOC_VECTOR_DONE:
536 		octeon_free_ioq_vector(oct);
537 
538 		fallthrough;
539 	case OCT_DEV_MBOX_SETUP_DONE:
540 		oct->fn_list.free_mbox(oct);
541 
542 		fallthrough;
543 	case OCT_DEV_IN_RESET:
544 	case OCT_DEV_DROQ_INIT_DONE:
545 		mdelay(100);
546 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
547 			if (!(oct->io_qmask.oq & BIT_ULL(i)))
548 				continue;
549 			octeon_delete_droq(oct, i);
550 		}
551 
552 		fallthrough;
553 	case OCT_DEV_RESP_LIST_INIT_DONE:
554 		octeon_delete_response_list(oct);
555 
556 		fallthrough;
557 	case OCT_DEV_INSTR_QUEUE_INIT_DONE:
558 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
559 			if (!(oct->io_qmask.iq & BIT_ULL(i)))
560 				continue;
561 			octeon_delete_instr_queue(oct, i);
562 		}
563 
564 		fallthrough;
565 	case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
566 		octeon_free_sc_buffer_pool(oct);
567 
568 		fallthrough;
569 	case OCT_DEV_DISPATCH_INIT_DONE:
570 		octeon_delete_dispatch_list(oct);
571 		cancel_delayed_work_sync(&oct->nic_poll_work.work);
572 
573 		fallthrough;
574 	case OCT_DEV_PCI_MAP_DONE:
575 		octeon_unmap_pci_barx(oct, 0);
576 		octeon_unmap_pci_barx(oct, 1);
577 
578 		fallthrough;
579 	case OCT_DEV_PCI_ENABLE_DONE:
580 		pci_clear_master(oct->pci_dev);
581 		/* Disable the device, releasing the PCI INT */
582 		pci_disable_device(oct->pci_dev);
583 
584 		fallthrough;
585 	case OCT_DEV_BEGIN_STATE:
586 		/* Nothing to be done here either */
587 		break;
588 	}
589 
590 	tasklet_kill(&oct_priv->droq_tasklet);
591 }
592 
593 /**
594  * send_rx_ctrl_cmd - Send Rx control command
595  * @lio: per-network private data
596  * @start_stop: whether to start or stop
597  */
598 static int send_rx_ctrl_cmd(struct lio *lio, int start_stop)
599 {
600 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
601 	struct octeon_soft_command *sc;
602 	union octnet_cmd *ncmd;
603 	int retval;
604 
605 	if (oct->props[lio->ifidx].rx_on == start_stop)
606 		return 0;
607 
608 	sc = (struct octeon_soft_command *)
609 		octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
610 					  16, 0);
611 	if (!sc) {
612 		netif_info(lio, rx_err, lio->netdev,
613 			   "Failed to allocate octeon_soft_command struct\n");
614 		return -ENOMEM;
615 	}
616 
617 	ncmd = (union octnet_cmd *)sc->virtdptr;
618 
619 	ncmd->u64 = 0;
620 	ncmd->s.cmd = OCTNET_CMD_RX_CTL;
621 	ncmd->s.param1 = start_stop;
622 
623 	octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
624 
625 	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
626 
627 	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
628 				    OPCODE_NIC_CMD, 0, 0, 0);
629 
630 	init_completion(&sc->complete);
631 	sc->sc_status = OCTEON_REQUEST_PENDING;
632 
633 	retval = octeon_send_soft_command(oct, sc);
634 	if (retval == IQ_SEND_FAILED) {
635 		netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
636 		octeon_free_soft_command(oct, sc);
637 	} else {
638 		/* Sleep on a wait queue till the cond flag indicates that the
639 		 * response arrived or timed-out.
640 		 */
641 		retval = wait_for_sc_completion_timeout(oct, sc, 0);
642 		if (retval)
643 			return retval;
644 
645 		oct->props[lio->ifidx].rx_on = start_stop;
646 		WRITE_ONCE(sc->caller_is_done, true);
647 	}
648 
649 	return retval;
650 }
651 
652 /**
653  * liquidio_destroy_nic_device - Destroy NIC device interface
654  * @oct: octeon device
655  * @ifidx: which interface to destroy
656  *
657  * Cleanup associated with each interface for an Octeon device  when NIC
658  * module is being unloaded or if initialization fails during load.
659  */
660 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
661 {
662 	struct net_device *netdev = oct->props[ifidx].netdev;
663 	struct octeon_device_priv *oct_priv =
664 		(struct octeon_device_priv *)oct->priv;
665 	struct napi_struct *napi, *n;
666 	struct lio *lio;
667 
668 	if (!netdev) {
669 		dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
670 			__func__, ifidx);
671 		return;
672 	}
673 
674 	lio = GET_LIO(netdev);
675 
676 	dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
677 
678 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
679 		liquidio_stop(netdev);
680 
681 	if (oct->props[lio->ifidx].napi_enabled == 1) {
682 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
683 			napi_disable(napi);
684 
685 		oct->props[lio->ifidx].napi_enabled = 0;
686 
687 		oct->droq[0]->ops.poll_mode = 0;
688 	}
689 
690 	/* Delete NAPI */
691 	list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
692 		netif_napi_del(napi);
693 
694 	tasklet_enable(&oct_priv->droq_tasklet);
695 
696 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
697 		unregister_netdev(netdev);
698 
699 	cleanup_rx_oom_poll_fn(netdev);
700 
701 	cleanup_link_status_change_wq(netdev);
702 
703 	lio_delete_glists(lio);
704 
705 	free_netdev(netdev);
706 
707 	oct->props[ifidx].gmxport = -1;
708 
709 	oct->props[ifidx].netdev = NULL;
710 }
711 
712 /**
713  * liquidio_stop_nic_module - Stop complete NIC functionality
714  * @oct: octeon device
715  */
716 static int liquidio_stop_nic_module(struct octeon_device *oct)
717 {
718 	struct lio *lio;
719 	int i, j;
720 
721 	dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
722 	if (!oct->ifcount) {
723 		dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
724 		return 1;
725 	}
726 
727 	spin_lock_bh(&oct->cmd_resp_wqlock);
728 	oct->cmd_resp_state = OCT_DRV_OFFLINE;
729 	spin_unlock_bh(&oct->cmd_resp_wqlock);
730 
731 	for (i = 0; i < oct->ifcount; i++) {
732 		lio = GET_LIO(oct->props[i].netdev);
733 		for (j = 0; j < oct->num_oqs; j++)
734 			octeon_unregister_droq_ops(oct,
735 						   lio->linfo.rxpciq[j].s.q_no);
736 	}
737 
738 	for (i = 0; i < oct->ifcount; i++)
739 		liquidio_destroy_nic_device(oct, i);
740 
741 	dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
742 	return 0;
743 }
744 
745 /**
746  * liquidio_vf_remove - Cleans up resources at unload time
747  * @pdev: PCI device structure
748  */
749 static void liquidio_vf_remove(struct pci_dev *pdev)
750 {
751 	struct octeon_device *oct_dev = pci_get_drvdata(pdev);
752 
753 	dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
754 
755 	if (oct_dev->app_mode == CVM_DRV_NIC_APP)
756 		liquidio_stop_nic_module(oct_dev);
757 
758 	/* Reset the octeon device and cleanup all memory allocated for
759 	 * the octeon device by driver.
760 	 */
761 	octeon_destroy_resources(oct_dev);
762 
763 	dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
764 
765 	/* This octeon device has been removed. Update the global
766 	 * data structure to reflect this. Free the device structure.
767 	 */
768 	octeon_free_device_mem(oct_dev);
769 }
770 
771 /**
772  * octeon_pci_os_setup - PCI initialization for each Octeon device.
773  * @oct: octeon device
774  */
775 static int octeon_pci_os_setup(struct octeon_device *oct)
776 {
777 #ifdef CONFIG_PCI_IOV
778 	/* setup PCI stuff first */
779 	if (!oct->pci_dev->physfn)
780 		octeon_pci_flr(oct);
781 #endif
782 
783 	if (pci_enable_device(oct->pci_dev)) {
784 		dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
785 		return 1;
786 	}
787 
788 	if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
789 		dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
790 		pci_disable_device(oct->pci_dev);
791 		return 1;
792 	}
793 
794 	/* Enable PCI DMA Master. */
795 	pci_set_master(oct->pci_dev);
796 
797 	return 0;
798 }
799 
800 /**
801  * free_netbuf - Unmap and free network buffer
802  * @buf: buffer
803  */
804 static void free_netbuf(void *buf)
805 {
806 	struct octnet_buf_free_info *finfo;
807 	struct sk_buff *skb;
808 	struct lio *lio;
809 
810 	finfo = (struct octnet_buf_free_info *)buf;
811 	skb = finfo->skb;
812 	lio = finfo->lio;
813 
814 	dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
815 			 DMA_TO_DEVICE);
816 
817 	tx_buffer_free(skb);
818 }
819 
820 /**
821  * free_netsgbuf - Unmap and free gather buffer
822  * @buf: buffer
823  */
824 static void free_netsgbuf(void *buf)
825 {
826 	struct octnet_buf_free_info *finfo;
827 	struct octnic_gather *g;
828 	struct sk_buff *skb;
829 	int i, frags, iq;
830 	struct lio *lio;
831 
832 	finfo = (struct octnet_buf_free_info *)buf;
833 	skb = finfo->skb;
834 	lio = finfo->lio;
835 	g = finfo->g;
836 	frags = skb_shinfo(skb)->nr_frags;
837 
838 	dma_unmap_single(&lio->oct_dev->pci_dev->dev,
839 			 g->sg[0].ptr[0], (skb->len - skb->data_len),
840 			 DMA_TO_DEVICE);
841 
842 	i = 1;
843 	while (frags--) {
844 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
845 
846 		dma_unmap_page(&lio->oct_dev->pci_dev->dev,
847 			       g->sg[(i >> 2)].ptr[(i & 3)],
848 			       skb_frag_size(frag), DMA_TO_DEVICE);
849 		i++;
850 	}
851 
852 	iq = skb_iq(lio->oct_dev, skb);
853 
854 	spin_lock(&lio->glist_lock[iq]);
855 	list_add_tail(&g->list, &lio->glist[iq]);
856 	spin_unlock(&lio->glist_lock[iq]);
857 
858 	tx_buffer_free(skb);
859 }
860 
861 /**
862  * free_netsgbuf_with_resp - Unmap and free gather buffer with response
863  * @buf: buffer
864  */
865 static void free_netsgbuf_with_resp(void *buf)
866 {
867 	struct octnet_buf_free_info *finfo;
868 	struct octeon_soft_command *sc;
869 	struct octnic_gather *g;
870 	struct sk_buff *skb;
871 	int i, frags, iq;
872 	struct lio *lio;
873 
874 	sc = (struct octeon_soft_command *)buf;
875 	skb = (struct sk_buff *)sc->callback_arg;
876 	finfo = (struct octnet_buf_free_info *)&skb->cb;
877 
878 	lio = finfo->lio;
879 	g = finfo->g;
880 	frags = skb_shinfo(skb)->nr_frags;
881 
882 	dma_unmap_single(&lio->oct_dev->pci_dev->dev,
883 			 g->sg[0].ptr[0], (skb->len - skb->data_len),
884 			 DMA_TO_DEVICE);
885 
886 	i = 1;
887 	while (frags--) {
888 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
889 
890 		dma_unmap_page(&lio->oct_dev->pci_dev->dev,
891 			       g->sg[(i >> 2)].ptr[(i & 3)],
892 			       skb_frag_size(frag), DMA_TO_DEVICE);
893 		i++;
894 	}
895 
896 	iq = skb_iq(lio->oct_dev, skb);
897 
898 	spin_lock(&lio->glist_lock[iq]);
899 	list_add_tail(&g->list, &lio->glist[iq]);
900 	spin_unlock(&lio->glist_lock[iq]);
901 
902 	/* Don't free the skb yet */
903 }
904 
905 /**
906  * liquidio_open - Net device open for LiquidIO
907  * @netdev: network device
908  */
909 static int liquidio_open(struct net_device *netdev)
910 {
911 	struct lio *lio = GET_LIO(netdev);
912 	struct octeon_device *oct = lio->oct_dev;
913 	struct octeon_device_priv *oct_priv =
914 		(struct octeon_device_priv *)oct->priv;
915 	struct napi_struct *napi, *n;
916 	int ret = 0;
917 
918 	if (!oct->props[lio->ifidx].napi_enabled) {
919 		tasklet_disable(&oct_priv->droq_tasklet);
920 
921 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
922 			napi_enable(napi);
923 
924 		oct->props[lio->ifidx].napi_enabled = 1;
925 
926 		oct->droq[0]->ops.poll_mode = 1;
927 	}
928 
929 	ifstate_set(lio, LIO_IFSTATE_RUNNING);
930 
931 	/* Ready for link status updates */
932 	lio->intf_open = 1;
933 
934 	netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
935 	start_txqs(netdev);
936 
937 	INIT_DELAYED_WORK(&lio->stats_wk.work, lio_fetch_stats);
938 	lio->stats_wk.ctxptr = lio;
939 	schedule_delayed_work(&lio->stats_wk.work, msecs_to_jiffies
940 					(LIQUIDIO_NDEV_STATS_POLL_TIME_MS));
941 
942 	/* tell Octeon to start forwarding packets to host */
943 	ret = send_rx_ctrl_cmd(lio, 1);
944 	if (ret)
945 		return ret;
946 
947 	dev_info(&oct->pci_dev->dev, "%s interface is opened\n", netdev->name);
948 
949 	return ret;
950 }
951 
952 /**
953  * liquidio_stop - jNet device stop for LiquidIO
954  * @netdev: network device
955  */
956 static int liquidio_stop(struct net_device *netdev)
957 {
958 	struct lio *lio = GET_LIO(netdev);
959 	struct octeon_device *oct = lio->oct_dev;
960 	struct octeon_device_priv *oct_priv =
961 		(struct octeon_device_priv *)oct->priv;
962 	struct napi_struct *napi, *n;
963 	int ret = 0;
964 
965 	/* tell Octeon to stop forwarding packets to host */
966 	ret = send_rx_ctrl_cmd(lio, 0);
967 	if (ret)
968 		return ret;
969 
970 	netif_info(lio, ifdown, lio->netdev, "Stopping interface!\n");
971 	/* Inform that netif carrier is down */
972 	lio->intf_open = 0;
973 	lio->linfo.link.s.link_up = 0;
974 
975 	netif_carrier_off(netdev);
976 	lio->link_changes++;
977 
978 	ifstate_reset(lio, LIO_IFSTATE_RUNNING);
979 
980 	stop_txqs(netdev);
981 
982 	/* Wait for any pending Rx descriptors */
983 	if (lio_wait_for_clean_oq(oct))
984 		netif_info(lio, rx_err, lio->netdev,
985 			   "Proceeding with stop interface after partial RX desc processing\n");
986 
987 	if (oct->props[lio->ifidx].napi_enabled == 1) {
988 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
989 			napi_disable(napi);
990 
991 		oct->props[lio->ifidx].napi_enabled = 0;
992 
993 		oct->droq[0]->ops.poll_mode = 0;
994 
995 		tasklet_enable(&oct_priv->droq_tasklet);
996 	}
997 
998 	cancel_delayed_work_sync(&lio->stats_wk.work);
999 
1000 	dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
1001 
1002 	return ret;
1003 }
1004 
1005 /**
1006  * get_new_flags - Converts a mask based on net device flags
1007  * @netdev: network device
1008  *
1009  * This routine generates a octnet_ifflags mask from the net device flags
1010  * received from the OS.
1011  */
1012 static enum octnet_ifflags get_new_flags(struct net_device *netdev)
1013 {
1014 	enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
1015 
1016 	if (netdev->flags & IFF_PROMISC)
1017 		f |= OCTNET_IFFLAG_PROMISC;
1018 
1019 	if (netdev->flags & IFF_ALLMULTI)
1020 		f |= OCTNET_IFFLAG_ALLMULTI;
1021 
1022 	if (netdev->flags & IFF_MULTICAST) {
1023 		f |= OCTNET_IFFLAG_MULTICAST;
1024 
1025 		/* Accept all multicast addresses if there are more than we
1026 		 * can handle
1027 		 */
1028 		if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
1029 			f |= OCTNET_IFFLAG_ALLMULTI;
1030 	}
1031 
1032 	if (netdev->flags & IFF_BROADCAST)
1033 		f |= OCTNET_IFFLAG_BROADCAST;
1034 
1035 	return f;
1036 }
1037 
1038 static void liquidio_set_uc_list(struct net_device *netdev)
1039 {
1040 	struct lio *lio = GET_LIO(netdev);
1041 	struct octeon_device *oct = lio->oct_dev;
1042 	struct octnic_ctrl_pkt nctrl;
1043 	struct netdev_hw_addr *ha;
1044 	u64 *mac;
1045 
1046 	if (lio->netdev_uc_count == netdev_uc_count(netdev))
1047 		return;
1048 
1049 	if (netdev_uc_count(netdev) > MAX_NCTRL_UDD) {
1050 		dev_err(&oct->pci_dev->dev, "too many MAC addresses in netdev uc list\n");
1051 		return;
1052 	}
1053 
1054 	lio->netdev_uc_count = netdev_uc_count(netdev);
1055 
1056 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1057 	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_UC_LIST;
1058 	nctrl.ncmd.s.more = lio->netdev_uc_count;
1059 	nctrl.ncmd.s.param1 = oct->vf_num;
1060 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1061 	nctrl.netpndev = (u64)netdev;
1062 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1063 
1064 	/* copy all the addresses into the udd */
1065 	mac = &nctrl.udd[0];
1066 	netdev_for_each_uc_addr(ha, netdev) {
1067 		ether_addr_copy(((u8 *)mac) + 2, ha->addr);
1068 		mac++;
1069 	}
1070 
1071 	octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1072 }
1073 
1074 /**
1075  * liquidio_set_mcast_list - Net device set_multicast_list
1076  * @netdev: network device
1077  */
1078 static void liquidio_set_mcast_list(struct net_device *netdev)
1079 {
1080 	int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
1081 	struct lio *lio = GET_LIO(netdev);
1082 	struct octeon_device *oct = lio->oct_dev;
1083 	struct octnic_ctrl_pkt nctrl;
1084 	struct netdev_hw_addr *ha;
1085 	u64 *mc;
1086 	int ret;
1087 
1088 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1089 
1090 	/* Create a ctrl pkt command to be sent to core app. */
1091 	nctrl.ncmd.u64 = 0;
1092 	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
1093 	nctrl.ncmd.s.param1 = get_new_flags(netdev);
1094 	nctrl.ncmd.s.param2 = mc_count;
1095 	nctrl.ncmd.s.more = mc_count;
1096 	nctrl.netpndev = (u64)netdev;
1097 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1098 
1099 	/* copy all the addresses into the udd */
1100 	mc = &nctrl.udd[0];
1101 	netdev_for_each_mc_addr(ha, netdev) {
1102 		*mc = 0;
1103 		ether_addr_copy(((u8 *)mc) + 2, ha->addr);
1104 		/* no need to swap bytes */
1105 		if (++mc > &nctrl.udd[mc_count])
1106 			break;
1107 	}
1108 
1109 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1110 
1111 	/* Apparently, any activity in this call from the kernel has to
1112 	 * be atomic. So we won't wait for response.
1113 	 */
1114 
1115 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1116 	if (ret) {
1117 		dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
1118 			ret);
1119 	}
1120 
1121 	liquidio_set_uc_list(netdev);
1122 }
1123 
1124 /**
1125  * liquidio_set_mac - Net device set_mac_address
1126  * @netdev: network device
1127  * @p: opaque pointer to sockaddr
1128  */
1129 static int liquidio_set_mac(struct net_device *netdev, void *p)
1130 {
1131 	struct sockaddr *addr = (struct sockaddr *)p;
1132 	struct lio *lio = GET_LIO(netdev);
1133 	struct octeon_device *oct = lio->oct_dev;
1134 	struct octnic_ctrl_pkt nctrl;
1135 	int ret = 0;
1136 
1137 	if (!is_valid_ether_addr(addr->sa_data))
1138 		return -EADDRNOTAVAIL;
1139 
1140 	if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
1141 		return 0;
1142 
1143 	if (lio->linfo.macaddr_is_admin_asgnd)
1144 		return -EPERM;
1145 
1146 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1147 
1148 	nctrl.ncmd.u64 = 0;
1149 	nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
1150 	nctrl.ncmd.s.param1 = 0;
1151 	nctrl.ncmd.s.more = 1;
1152 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1153 	nctrl.netpndev = (u64)netdev;
1154 
1155 	nctrl.udd[0] = 0;
1156 	/* The MAC Address is presented in network byte order. */
1157 	ether_addr_copy((u8 *)&nctrl.udd[0] + 2, addr->sa_data);
1158 
1159 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1160 	if (ret < 0) {
1161 		dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
1162 		return -ENOMEM;
1163 	}
1164 
1165 	if (nctrl.sc_status ==
1166 	    FIRMWARE_STATUS_CODE(OCTEON_REQUEST_NO_PERMISSION)) {
1167 		dev_err(&oct->pci_dev->dev, "MAC Address change failed: no permission\n");
1168 		return -EPERM;
1169 	}
1170 
1171 	eth_hw_addr_set(netdev, addr->sa_data);
1172 	ether_addr_copy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data);
1173 
1174 	return 0;
1175 }
1176 
1177 static void
1178 liquidio_get_stats64(struct net_device *netdev,
1179 		     struct rtnl_link_stats64 *lstats)
1180 {
1181 	struct lio *lio = GET_LIO(netdev);
1182 	struct octeon_device *oct;
1183 	u64 pkts = 0, drop = 0, bytes = 0;
1184 	struct oct_droq_stats *oq_stats;
1185 	struct oct_iq_stats *iq_stats;
1186 	int i, iq_no, oq_no;
1187 
1188 	oct = lio->oct_dev;
1189 
1190 	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
1191 		return;
1192 
1193 	for (i = 0; i < oct->num_iqs; i++) {
1194 		iq_no = lio->linfo.txpciq[i].s.q_no;
1195 		iq_stats = &oct->instr_queue[iq_no]->stats;
1196 		pkts += iq_stats->tx_done;
1197 		drop += iq_stats->tx_dropped;
1198 		bytes += iq_stats->tx_tot_bytes;
1199 	}
1200 
1201 	lstats->tx_packets = pkts;
1202 	lstats->tx_bytes = bytes;
1203 	lstats->tx_dropped = drop;
1204 
1205 	pkts = 0;
1206 	drop = 0;
1207 	bytes = 0;
1208 
1209 	for (i = 0; i < oct->num_oqs; i++) {
1210 		oq_no = lio->linfo.rxpciq[i].s.q_no;
1211 		oq_stats = &oct->droq[oq_no]->stats;
1212 		pkts += oq_stats->rx_pkts_received;
1213 		drop += (oq_stats->rx_dropped +
1214 			 oq_stats->dropped_nodispatch +
1215 			 oq_stats->dropped_toomany +
1216 			 oq_stats->dropped_nomem);
1217 		bytes += oq_stats->rx_bytes_received;
1218 	}
1219 
1220 	lstats->rx_bytes = bytes;
1221 	lstats->rx_packets = pkts;
1222 	lstats->rx_dropped = drop;
1223 
1224 	lstats->multicast = oct->link_stats.fromwire.fw_total_mcast;
1225 
1226 	/* detailed rx_errors: */
1227 	lstats->rx_length_errors = oct->link_stats.fromwire.l2_err;
1228 	/* recved pkt with crc error */
1229 	lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err;
1230 	/* recv'd frame alignment error */
1231 	lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err;
1232 
1233 	lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors +
1234 			    lstats->rx_frame_errors;
1235 
1236 	/* detailed tx_errors */
1237 	lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko;
1238 	lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link;
1239 
1240 	lstats->tx_errors = lstats->tx_aborted_errors +
1241 		lstats->tx_carrier_errors;
1242 }
1243 
1244 /**
1245  * hwtstamp_ioctl - Handler for SIOCSHWTSTAMP ioctl
1246  * @netdev: network device
1247  * @ifr: interface request
1248  */
1249 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
1250 {
1251 	struct lio *lio = GET_LIO(netdev);
1252 	struct hwtstamp_config conf;
1253 
1254 	if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
1255 		return -EFAULT;
1256 
1257 	switch (conf.tx_type) {
1258 	case HWTSTAMP_TX_ON:
1259 	case HWTSTAMP_TX_OFF:
1260 		break;
1261 	default:
1262 		return -ERANGE;
1263 	}
1264 
1265 	switch (conf.rx_filter) {
1266 	case HWTSTAMP_FILTER_NONE:
1267 		break;
1268 	case HWTSTAMP_FILTER_ALL:
1269 	case HWTSTAMP_FILTER_SOME:
1270 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1271 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1272 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1273 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1274 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1275 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1276 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1277 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1278 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1279 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
1280 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
1281 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1282 	case HWTSTAMP_FILTER_NTP_ALL:
1283 		conf.rx_filter = HWTSTAMP_FILTER_ALL;
1284 		break;
1285 	default:
1286 		return -ERANGE;
1287 	}
1288 
1289 	if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
1290 		ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
1291 
1292 	else
1293 		ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
1294 
1295 	return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
1296 }
1297 
1298 /**
1299  * liquidio_ioctl - ioctl handler
1300  * @netdev: network device
1301  * @ifr: interface request
1302  * @cmd: command
1303  */
1304 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
1305 {
1306 	switch (cmd) {
1307 	case SIOCSHWTSTAMP:
1308 		return hwtstamp_ioctl(netdev, ifr);
1309 	default:
1310 		return -EOPNOTSUPP;
1311 	}
1312 }
1313 
1314 static void handle_timestamp(struct octeon_device *oct, u32 status, void *buf)
1315 {
1316 	struct sk_buff *skb = (struct sk_buff *)buf;
1317 	struct octnet_buf_free_info *finfo;
1318 	struct oct_timestamp_resp *resp;
1319 	struct octeon_soft_command *sc;
1320 	struct lio *lio;
1321 
1322 	finfo = (struct octnet_buf_free_info *)skb->cb;
1323 	lio = finfo->lio;
1324 	sc = finfo->sc;
1325 	oct = lio->oct_dev;
1326 	resp = (struct oct_timestamp_resp *)sc->virtrptr;
1327 
1328 	if (status != OCTEON_REQUEST_DONE) {
1329 		dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
1330 			CVM_CAST64(status));
1331 		resp->timestamp = 0;
1332 	}
1333 
1334 	octeon_swap_8B_data(&resp->timestamp, 1);
1335 
1336 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) {
1337 		struct skb_shared_hwtstamps ts;
1338 		u64 ns = resp->timestamp;
1339 
1340 		netif_info(lio, tx_done, lio->netdev,
1341 			   "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
1342 			   skb, (unsigned long long)ns);
1343 		ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
1344 		skb_tstamp_tx(skb, &ts);
1345 	}
1346 
1347 	octeon_free_soft_command(oct, sc);
1348 	tx_buffer_free(skb);
1349 }
1350 
1351 /* send_nic_timestamp_pkt - Send a data packet that will be timestamped
1352  * @oct: octeon device
1353  * @ndata: pointer to network data
1354  * @finfo: pointer to private network data
1355  */
1356 static int send_nic_timestamp_pkt(struct octeon_device *oct,
1357 				  struct octnic_data_pkt *ndata,
1358 				  struct octnet_buf_free_info *finfo,
1359 				  int xmit_more)
1360 {
1361 	struct octeon_soft_command *sc;
1362 	int ring_doorbell;
1363 	struct lio *lio;
1364 	int retval;
1365 	u32 len;
1366 
1367 	lio = finfo->lio;
1368 
1369 	sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
1370 					    sizeof(struct oct_timestamp_resp));
1371 	finfo->sc = sc;
1372 
1373 	if (!sc) {
1374 		dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
1375 		return IQ_SEND_FAILED;
1376 	}
1377 
1378 	if (ndata->reqtype == REQTYPE_NORESP_NET)
1379 		ndata->reqtype = REQTYPE_RESP_NET;
1380 	else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
1381 		ndata->reqtype = REQTYPE_RESP_NET_SG;
1382 
1383 	sc->callback = handle_timestamp;
1384 	sc->callback_arg = finfo->skb;
1385 	sc->iq_no = ndata->q_no;
1386 
1387 	len = (u32)((struct octeon_instr_ih3 *)(&sc->cmd.cmd3.ih3))->dlengsz;
1388 
1389 	ring_doorbell = !xmit_more;
1390 
1391 	retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
1392 				     sc, len, ndata->reqtype);
1393 
1394 	if (retval == IQ_SEND_FAILED) {
1395 		dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
1396 			retval);
1397 		octeon_free_soft_command(oct, sc);
1398 	} else {
1399 		netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
1400 	}
1401 
1402 	return retval;
1403 }
1404 
1405 /**
1406  * liquidio_xmit - Transmit networks packets to the Octeon interface
1407  * @skb: skbuff struct to be passed to network layer.
1408  * @netdev: pointer to network device
1409  * @returns whether the packet was transmitted to the device okay or not
1410  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
1411  */
1412 static netdev_tx_t liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
1413 {
1414 	struct octnet_buf_free_info *finfo;
1415 	union octnic_cmd_setup cmdsetup;
1416 	struct octnic_data_pkt ndata;
1417 	struct octeon_instr_irh *irh;
1418 	struct oct_iq_stats *stats;
1419 	struct octeon_device *oct;
1420 	int q_idx = 0, iq_no = 0;
1421 	union tx_info *tx_info;
1422 	int xmit_more = 0;
1423 	struct lio *lio;
1424 	int status = 0;
1425 	u64 dptr = 0;
1426 	u32 tag = 0;
1427 	int j;
1428 
1429 	lio = GET_LIO(netdev);
1430 	oct = lio->oct_dev;
1431 
1432 	q_idx = skb_iq(lio->oct_dev, skb);
1433 	tag = q_idx;
1434 	iq_no = lio->linfo.txpciq[q_idx].s.q_no;
1435 
1436 	stats = &oct->instr_queue[iq_no]->stats;
1437 
1438 	/* Check for all conditions in which the current packet cannot be
1439 	 * transmitted.
1440 	 */
1441 	if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
1442 	    (!lio->linfo.link.s.link_up) || (skb->len <= 0)) {
1443 		netif_info(lio, tx_err, lio->netdev, "Transmit failed link_status : %d\n",
1444 			   lio->linfo.link.s.link_up);
1445 		goto lio_xmit_failed;
1446 	}
1447 
1448 	/* Use space in skb->cb to store info used to unmap and
1449 	 * free the buffers.
1450 	 */
1451 	finfo = (struct octnet_buf_free_info *)skb->cb;
1452 	finfo->lio = lio;
1453 	finfo->skb = skb;
1454 	finfo->sc = NULL;
1455 
1456 	/* Prepare the attributes for the data to be passed to OSI. */
1457 	memset(&ndata, 0, sizeof(struct octnic_data_pkt));
1458 
1459 	ndata.buf = finfo;
1460 
1461 	ndata.q_no = iq_no;
1462 
1463 	if (octnet_iq_is_full(oct, ndata.q_no)) {
1464 		/* defer sending if queue is full */
1465 		netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
1466 			   ndata.q_no);
1467 		stats->tx_iq_busy++;
1468 		return NETDEV_TX_BUSY;
1469 	}
1470 
1471 	ndata.datasize = skb->len;
1472 
1473 	cmdsetup.u64 = 0;
1474 	cmdsetup.s.iq_no = iq_no;
1475 
1476 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
1477 		if (skb->encapsulation) {
1478 			cmdsetup.s.tnl_csum = 1;
1479 			stats->tx_vxlan++;
1480 		} else {
1481 			cmdsetup.s.transport_csum = 1;
1482 		}
1483 	}
1484 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
1485 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1486 		cmdsetup.s.timestamp = 1;
1487 	}
1488 
1489 	if (!skb_shinfo(skb)->nr_frags) {
1490 		cmdsetup.s.u.datasize = skb->len;
1491 		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
1492 		/* Offload checksum calculation for TCP/UDP packets */
1493 		dptr = dma_map_single(&oct->pci_dev->dev,
1494 				      skb->data,
1495 				      skb->len,
1496 				      DMA_TO_DEVICE);
1497 		if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
1498 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
1499 				__func__);
1500 			return NETDEV_TX_BUSY;
1501 		}
1502 
1503 		ndata.cmd.cmd3.dptr = dptr;
1504 		finfo->dptr = dptr;
1505 		ndata.reqtype = REQTYPE_NORESP_NET;
1506 
1507 	} else {
1508 		skb_frag_t *frag;
1509 		struct octnic_gather *g;
1510 		int i, frags;
1511 
1512 		spin_lock(&lio->glist_lock[q_idx]);
1513 		g = (struct octnic_gather *)
1514 			lio_list_delete_head(&lio->glist[q_idx]);
1515 		spin_unlock(&lio->glist_lock[q_idx]);
1516 
1517 		if (!g) {
1518 			netif_info(lio, tx_err, lio->netdev,
1519 				   "Transmit scatter gather: glist null!\n");
1520 			goto lio_xmit_failed;
1521 		}
1522 
1523 		cmdsetup.s.gather = 1;
1524 		cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
1525 		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
1526 
1527 		memset(g->sg, 0, g->sg_size);
1528 
1529 		g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
1530 						 skb->data,
1531 						 (skb->len - skb->data_len),
1532 						 DMA_TO_DEVICE);
1533 		if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
1534 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
1535 				__func__);
1536 			return NETDEV_TX_BUSY;
1537 		}
1538 		add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
1539 
1540 		frags = skb_shinfo(skb)->nr_frags;
1541 		i = 1;
1542 		while (frags--) {
1543 			frag = &skb_shinfo(skb)->frags[i - 1];
1544 
1545 			g->sg[(i >> 2)].ptr[(i & 3)] =
1546 				skb_frag_dma_map(&oct->pci_dev->dev,
1547 						 frag, 0, skb_frag_size(frag),
1548 						 DMA_TO_DEVICE);
1549 			if (dma_mapping_error(&oct->pci_dev->dev,
1550 					      g->sg[i >> 2].ptr[i & 3])) {
1551 				dma_unmap_single(&oct->pci_dev->dev,
1552 						 g->sg[0].ptr[0],
1553 						 skb->len - skb->data_len,
1554 						 DMA_TO_DEVICE);
1555 				for (j = 1; j < i; j++) {
1556 					frag = &skb_shinfo(skb)->frags[j - 1];
1557 					dma_unmap_page(&oct->pci_dev->dev,
1558 						       g->sg[j >> 2].ptr[j & 3],
1559 						       skb_frag_size(frag),
1560 						       DMA_TO_DEVICE);
1561 				}
1562 				dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
1563 					__func__);
1564 				return NETDEV_TX_BUSY;
1565 			}
1566 
1567 			add_sg_size(&g->sg[(i >> 2)], skb_frag_size(frag),
1568 				    (i & 3));
1569 			i++;
1570 		}
1571 
1572 		dptr = g->sg_dma_ptr;
1573 
1574 		ndata.cmd.cmd3.dptr = dptr;
1575 		finfo->dptr = dptr;
1576 		finfo->g = g;
1577 
1578 		ndata.reqtype = REQTYPE_NORESP_NET_SG;
1579 	}
1580 
1581 	irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
1582 	tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];
1583 
1584 	if (skb_shinfo(skb)->gso_size) {
1585 		tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
1586 		tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
1587 	}
1588 
1589 	/* HW insert VLAN tag */
1590 	if (skb_vlan_tag_present(skb)) {
1591 		irh->priority = skb_vlan_tag_get(skb) >> VLAN_PRIO_SHIFT;
1592 		irh->vlan = skb_vlan_tag_get(skb) & VLAN_VID_MASK;
1593 	}
1594 
1595 	xmit_more = netdev_xmit_more();
1596 
1597 	if (unlikely(cmdsetup.s.timestamp))
1598 		status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
1599 	else
1600 		status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
1601 	if (status == IQ_SEND_FAILED)
1602 		goto lio_xmit_failed;
1603 
1604 	netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
1605 
1606 	if (status == IQ_SEND_STOP) {
1607 		dev_err(&oct->pci_dev->dev, "Rcvd IQ_SEND_STOP signal; stopping IQ-%d\n",
1608 			iq_no);
1609 		netif_stop_subqueue(netdev, q_idx);
1610 	}
1611 
1612 	netif_trans_update(netdev);
1613 
1614 	if (tx_info->s.gso_segs)
1615 		stats->tx_done += tx_info->s.gso_segs;
1616 	else
1617 		stats->tx_done++;
1618 	stats->tx_tot_bytes += ndata.datasize;
1619 
1620 	return NETDEV_TX_OK;
1621 
1622 lio_xmit_failed:
1623 	stats->tx_dropped++;
1624 	netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
1625 		   iq_no, stats->tx_dropped);
1626 	if (dptr)
1627 		dma_unmap_single(&oct->pci_dev->dev, dptr,
1628 				 ndata.datasize, DMA_TO_DEVICE);
1629 
1630 	octeon_ring_doorbell_locked(oct, iq_no);
1631 
1632 	tx_buffer_free(skb);
1633 	return NETDEV_TX_OK;
1634 }
1635 
1636 /**
1637  * liquidio_tx_timeout - Network device Tx timeout
1638  * @netdev: pointer to network device
1639  * @txqueue: index of the hung transmit queue
1640  */
1641 static void liquidio_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1642 {
1643 	struct lio *lio;
1644 
1645 	lio = GET_LIO(netdev);
1646 
1647 	netif_info(lio, tx_err, lio->netdev,
1648 		   "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
1649 		   netdev->stats.tx_dropped);
1650 	netif_trans_update(netdev);
1651 	wake_txqs(netdev);
1652 }
1653 
1654 static int
1655 liquidio_vlan_rx_add_vid(struct net_device *netdev,
1656 			 __be16 proto __attribute__((unused)), u16 vid)
1657 {
1658 	struct lio *lio = GET_LIO(netdev);
1659 	struct octeon_device *oct = lio->oct_dev;
1660 	struct octnic_ctrl_pkt nctrl;
1661 	int ret = 0;
1662 
1663 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1664 
1665 	nctrl.ncmd.u64 = 0;
1666 	nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
1667 	nctrl.ncmd.s.param1 = vid;
1668 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1669 	nctrl.netpndev = (u64)netdev;
1670 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1671 
1672 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1673 	if (ret) {
1674 		dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
1675 			ret);
1676 		return -EPERM;
1677 	}
1678 
1679 	return 0;
1680 }
1681 
1682 static int
1683 liquidio_vlan_rx_kill_vid(struct net_device *netdev,
1684 			  __be16 proto __attribute__((unused)), u16 vid)
1685 {
1686 	struct lio *lio = GET_LIO(netdev);
1687 	struct octeon_device *oct = lio->oct_dev;
1688 	struct octnic_ctrl_pkt nctrl;
1689 	int ret = 0;
1690 
1691 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1692 
1693 	nctrl.ncmd.u64 = 0;
1694 	nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
1695 	nctrl.ncmd.s.param1 = vid;
1696 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1697 	nctrl.netpndev = (u64)netdev;
1698 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1699 
1700 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1701 	if (ret) {
1702 		dev_err(&oct->pci_dev->dev, "Del VLAN filter failed in core (ret: 0x%x)\n",
1703 			ret);
1704 		if (ret > 0)
1705 			ret = -EIO;
1706 	}
1707 	return ret;
1708 }
1709 
1710 /** Sending command to enable/disable RX checksum offload
1711  * @param netdev                pointer to network device
1712  * @param command               OCTNET_CMD_TNL_RX_CSUM_CTL
1713  * @param rx_cmd_bit            OCTNET_CMD_RXCSUM_ENABLE/
1714  *                              OCTNET_CMD_RXCSUM_DISABLE
1715  * @returns                     SUCCESS or FAILURE
1716  */
1717 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
1718 				       u8 rx_cmd)
1719 {
1720 	struct lio *lio = GET_LIO(netdev);
1721 	struct octeon_device *oct = lio->oct_dev;
1722 	struct octnic_ctrl_pkt nctrl;
1723 	int ret = 0;
1724 
1725 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1726 
1727 	nctrl.ncmd.u64 = 0;
1728 	nctrl.ncmd.s.cmd = command;
1729 	nctrl.ncmd.s.param1 = rx_cmd;
1730 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1731 	nctrl.netpndev = (u64)netdev;
1732 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1733 
1734 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1735 	if (ret) {
1736 		dev_err(&oct->pci_dev->dev, "DEVFLAGS RXCSUM change failed in core (ret:0x%x)\n",
1737 			ret);
1738 		if (ret > 0)
1739 			ret = -EIO;
1740 	}
1741 	return ret;
1742 }
1743 
1744 /** Sending command to add/delete VxLAN UDP port to firmware
1745  * @param netdev                pointer to network device
1746  * @param command               OCTNET_CMD_VXLAN_PORT_CONFIG
1747  * @param vxlan_port            VxLAN port to be added or deleted
1748  * @param vxlan_cmd_bit         OCTNET_CMD_VXLAN_PORT_ADD,
1749  *                              OCTNET_CMD_VXLAN_PORT_DEL
1750  * @returns                     SUCCESS or FAILURE
1751  */
1752 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
1753 				       u16 vxlan_port, u8 vxlan_cmd_bit)
1754 {
1755 	struct lio *lio = GET_LIO(netdev);
1756 	struct octeon_device *oct = lio->oct_dev;
1757 	struct octnic_ctrl_pkt nctrl;
1758 	int ret = 0;
1759 
1760 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1761 
1762 	nctrl.ncmd.u64 = 0;
1763 	nctrl.ncmd.s.cmd = command;
1764 	nctrl.ncmd.s.more = vxlan_cmd_bit;
1765 	nctrl.ncmd.s.param1 = vxlan_port;
1766 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1767 	nctrl.netpndev = (u64)netdev;
1768 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1769 
1770 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1771 	if (ret) {
1772 		dev_err(&oct->pci_dev->dev,
1773 			"DEVFLAGS VxLAN port add/delete failed in core (ret : 0x%x)\n",
1774 			ret);
1775 		if (ret > 0)
1776 			ret = -EIO;
1777 	}
1778 	return ret;
1779 }
1780 
1781 static int liquidio_udp_tunnel_set_port(struct net_device *netdev,
1782 					unsigned int table, unsigned int entry,
1783 					struct udp_tunnel_info *ti)
1784 {
1785 	return liquidio_vxlan_port_command(netdev,
1786 					   OCTNET_CMD_VXLAN_PORT_CONFIG,
1787 					   htons(ti->port),
1788 					   OCTNET_CMD_VXLAN_PORT_ADD);
1789 }
1790 
1791 static int liquidio_udp_tunnel_unset_port(struct net_device *netdev,
1792 					  unsigned int table,
1793 					  unsigned int entry,
1794 					  struct udp_tunnel_info *ti)
1795 {
1796 	return liquidio_vxlan_port_command(netdev,
1797 					   OCTNET_CMD_VXLAN_PORT_CONFIG,
1798 					   htons(ti->port),
1799 					   OCTNET_CMD_VXLAN_PORT_DEL);
1800 }
1801 
1802 static const struct udp_tunnel_nic_info liquidio_udp_tunnels = {
1803 	.set_port	= liquidio_udp_tunnel_set_port,
1804 	.unset_port	= liquidio_udp_tunnel_unset_port,
1805 	.tables		= {
1806 		{ .n_entries = 1024, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
1807 	},
1808 };
1809 
1810 /** \brief Net device fix features
1811  * @param netdev  pointer to network device
1812  * @param request features requested
1813  * @returns updated features list
1814  */
1815 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
1816 					       netdev_features_t request)
1817 {
1818 	struct lio *lio = netdev_priv(netdev);
1819 
1820 	if ((request & NETIF_F_RXCSUM) &&
1821 	    !(lio->dev_capability & NETIF_F_RXCSUM))
1822 		request &= ~NETIF_F_RXCSUM;
1823 
1824 	if ((request & NETIF_F_HW_CSUM) &&
1825 	    !(lio->dev_capability & NETIF_F_HW_CSUM))
1826 		request &= ~NETIF_F_HW_CSUM;
1827 
1828 	if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
1829 		request &= ~NETIF_F_TSO;
1830 
1831 	if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
1832 		request &= ~NETIF_F_TSO6;
1833 
1834 	if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
1835 		request &= ~NETIF_F_LRO;
1836 
1837 	/* Disable LRO if RXCSUM is off */
1838 	if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
1839 	    (lio->dev_capability & NETIF_F_LRO))
1840 		request &= ~NETIF_F_LRO;
1841 
1842 	return request;
1843 }
1844 
1845 /** \brief Net device set features
1846  * @param netdev  pointer to network device
1847  * @param features features to enable/disable
1848  */
1849 static int liquidio_set_features(struct net_device *netdev,
1850 				 netdev_features_t features)
1851 {
1852 	struct lio *lio = netdev_priv(netdev);
1853 
1854 	if (!((netdev->features ^ features) & NETIF_F_LRO))
1855 		return 0;
1856 
1857 	if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO))
1858 		liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
1859 				     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
1860 	else if (!(features & NETIF_F_LRO) &&
1861 		 (lio->dev_capability & NETIF_F_LRO))
1862 		liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
1863 				     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
1864 	if (!(netdev->features & NETIF_F_RXCSUM) &&
1865 	    (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
1866 	    (features & NETIF_F_RXCSUM))
1867 		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
1868 					    OCTNET_CMD_RXCSUM_ENABLE);
1869 	else if ((netdev->features & NETIF_F_RXCSUM) &&
1870 		 (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
1871 		 !(features & NETIF_F_RXCSUM))
1872 		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
1873 					    OCTNET_CMD_RXCSUM_DISABLE);
1874 
1875 	return 0;
1876 }
1877 
1878 static const struct net_device_ops lionetdevops = {
1879 	.ndo_open		= liquidio_open,
1880 	.ndo_stop		= liquidio_stop,
1881 	.ndo_start_xmit		= liquidio_xmit,
1882 	.ndo_get_stats64	= liquidio_get_stats64,
1883 	.ndo_set_mac_address	= liquidio_set_mac,
1884 	.ndo_set_rx_mode	= liquidio_set_mcast_list,
1885 	.ndo_tx_timeout		= liquidio_tx_timeout,
1886 	.ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
1887 	.ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
1888 	.ndo_change_mtu		= liquidio_change_mtu,
1889 	.ndo_eth_ioctl		= liquidio_ioctl,
1890 	.ndo_fix_features	= liquidio_fix_features,
1891 	.ndo_set_features	= liquidio_set_features,
1892 };
1893 
1894 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
1895 {
1896 	struct octeon_device *oct = (struct octeon_device *)buf;
1897 	struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
1898 	union oct_link_status *ls;
1899 	int gmxport = 0;
1900 	int i;
1901 
1902 	if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) {
1903 		dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
1904 			recv_pkt->buffer_size[0],
1905 			recv_pkt->rh.r_nic_info.gmxport);
1906 		goto nic_info_err;
1907 	}
1908 
1909 	gmxport = recv_pkt->rh.r_nic_info.gmxport;
1910 	ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) +
1911 		OCT_DROQ_INFO_SIZE);
1912 
1913 	octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
1914 
1915 	for (i = 0; i < oct->ifcount; i++) {
1916 		if (oct->props[i].gmxport == gmxport) {
1917 			update_link_status(oct->props[i].netdev, ls);
1918 			break;
1919 		}
1920 	}
1921 
1922 nic_info_err:
1923 	for (i = 0; i < recv_pkt->buffer_count; i++)
1924 		recv_buffer_free(recv_pkt->buffer_ptr[i]);
1925 	octeon_free_recv_info(recv_info);
1926 	return 0;
1927 }
1928 
1929 /**
1930  * setup_nic_devices - Setup network interfaces
1931  * @octeon_dev:  octeon device
1932  *
1933  * Called during init time for each device. It assumes the NIC
1934  * is already up and running.  The link information for each
1935  * interface is passed in link_info.
1936  */
1937 static int setup_nic_devices(struct octeon_device *octeon_dev)
1938 {
1939 	int retval, num_iqueues, num_oqueues;
1940 	u32 resp_size, data_size;
1941 	struct liquidio_if_cfg_resp *resp;
1942 	struct octeon_soft_command *sc;
1943 	union oct_nic_if_cfg if_cfg;
1944 	struct octdev_props *props;
1945 	struct net_device *netdev;
1946 	struct lio_version *vdata;
1947 	struct lio *lio = NULL;
1948 	u8 mac[ETH_ALEN], i, j;
1949 	u32 ifidx_or_pfnum;
1950 
1951 	ifidx_or_pfnum = octeon_dev->pf_num;
1952 
1953 	/* This is to handle link status changes */
1954 	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC, OPCODE_NIC_INFO,
1955 				    lio_nic_info, octeon_dev);
1956 
1957 	/* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
1958 	 * They are handled directly.
1959 	 */
1960 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
1961 					free_netbuf);
1962 
1963 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
1964 					free_netsgbuf);
1965 
1966 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
1967 					free_netsgbuf_with_resp);
1968 
1969 	for (i = 0; i < octeon_dev->ifcount; i++) {
1970 		resp_size = sizeof(struct liquidio_if_cfg_resp);
1971 		data_size = sizeof(struct lio_version);
1972 		sc = (struct octeon_soft_command *)
1973 			octeon_alloc_soft_command(octeon_dev, data_size,
1974 						  resp_size, 0);
1975 		resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
1976 		vdata = (struct lio_version *)sc->virtdptr;
1977 
1978 		*((u64 *)vdata) = 0;
1979 		vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
1980 		vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
1981 		vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
1982 
1983 		if_cfg.u64 = 0;
1984 
1985 		if_cfg.s.num_iqueues = octeon_dev->sriov_info.rings_per_vf;
1986 		if_cfg.s.num_oqueues = octeon_dev->sriov_info.rings_per_vf;
1987 		if_cfg.s.base_queue = 0;
1988 
1989 		sc->iq_no = 0;
1990 
1991 		octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
1992 					    OPCODE_NIC_IF_CFG, 0, if_cfg.u64,
1993 					    0);
1994 
1995 		init_completion(&sc->complete);
1996 		sc->sc_status = OCTEON_REQUEST_PENDING;
1997 
1998 		retval = octeon_send_soft_command(octeon_dev, sc);
1999 		if (retval == IQ_SEND_FAILED) {
2000 			dev_err(&octeon_dev->pci_dev->dev,
2001 				"iq/oq config failed status: %x\n", retval);
2002 			/* Soft instr is freed by driver in case of failure. */
2003 			octeon_free_soft_command(octeon_dev, sc);
2004 			return(-EIO);
2005 		}
2006 
2007 		/* Sleep on a wait queue till the cond flag indicates that the
2008 		 * response arrived or timed-out.
2009 		 */
2010 		retval = wait_for_sc_completion_timeout(octeon_dev, sc, 0);
2011 		if (retval)
2012 			return retval;
2013 
2014 		retval = resp->status;
2015 		if (retval) {
2016 			dev_err(&octeon_dev->pci_dev->dev,
2017 				"iq/oq config failed, retval = %d\n", retval);
2018 			WRITE_ONCE(sc->caller_is_done, true);
2019 			return -EIO;
2020 		}
2021 
2022 		snprintf(octeon_dev->fw_info.liquidio_firmware_version,
2023 			 32, "%s",
2024 			 resp->cfg_info.liquidio_firmware_version);
2025 
2026 		octeon_swap_8B_data((u64 *)(&resp->cfg_info),
2027 				    (sizeof(struct liquidio_if_cfg_info)) >> 3);
2028 
2029 		num_iqueues = hweight64(resp->cfg_info.iqmask);
2030 		num_oqueues = hweight64(resp->cfg_info.oqmask);
2031 
2032 		if (!(num_iqueues) || !(num_oqueues)) {
2033 			dev_err(&octeon_dev->pci_dev->dev,
2034 				"Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
2035 				resp->cfg_info.iqmask, resp->cfg_info.oqmask);
2036 			WRITE_ONCE(sc->caller_is_done, true);
2037 			goto setup_nic_dev_done;
2038 		}
2039 		dev_dbg(&octeon_dev->pci_dev->dev,
2040 			"interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
2041 			i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
2042 			num_iqueues, num_oqueues);
2043 
2044 		netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues);
2045 
2046 		if (!netdev) {
2047 			dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
2048 			WRITE_ONCE(sc->caller_is_done, true);
2049 			goto setup_nic_dev_done;
2050 		}
2051 
2052 		SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
2053 
2054 		/* Associate the routines that will handle different
2055 		 * netdev tasks.
2056 		 */
2057 		netdev->netdev_ops = &lionetdevops;
2058 
2059 		lio = GET_LIO(netdev);
2060 
2061 		memset(lio, 0, sizeof(struct lio));
2062 
2063 		lio->ifidx = ifidx_or_pfnum;
2064 
2065 		props = &octeon_dev->props[i];
2066 		props->gmxport = resp->cfg_info.linfo.gmxport;
2067 		props->netdev = netdev;
2068 
2069 		lio->linfo.num_rxpciq = num_oqueues;
2070 		lio->linfo.num_txpciq = num_iqueues;
2071 
2072 		for (j = 0; j < num_oqueues; j++) {
2073 			lio->linfo.rxpciq[j].u64 =
2074 			    resp->cfg_info.linfo.rxpciq[j].u64;
2075 		}
2076 		for (j = 0; j < num_iqueues; j++) {
2077 			lio->linfo.txpciq[j].u64 =
2078 			    resp->cfg_info.linfo.txpciq[j].u64;
2079 		}
2080 
2081 		lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
2082 		lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
2083 		lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
2084 		lio->linfo.macaddr_is_admin_asgnd =
2085 			resp->cfg_info.linfo.macaddr_is_admin_asgnd;
2086 		lio->linfo.macaddr_spoofchk =
2087 			resp->cfg_info.linfo.macaddr_spoofchk;
2088 
2089 		lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2090 
2091 		lio->dev_capability = NETIF_F_HIGHDMA
2092 				      | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM
2093 				      | NETIF_F_SG | NETIF_F_RXCSUM
2094 				      | NETIF_F_TSO | NETIF_F_TSO6
2095 				      | NETIF_F_GRO
2096 				      | NETIF_F_LRO;
2097 		netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
2098 
2099 		/* Copy of transmit encapsulation capabilities:
2100 		 * TSO, TSO6, Checksums for this device
2101 		 */
2102 		lio->enc_dev_capability = NETIF_F_IP_CSUM
2103 					  | NETIF_F_IPV6_CSUM
2104 					  | NETIF_F_GSO_UDP_TUNNEL
2105 					  | NETIF_F_HW_CSUM | NETIF_F_SG
2106 					  | NETIF_F_RXCSUM
2107 					  | NETIF_F_TSO | NETIF_F_TSO6
2108 					  | NETIF_F_LRO;
2109 
2110 		netdev->hw_enc_features =
2111 		    (lio->enc_dev_capability & ~NETIF_F_LRO);
2112 		netdev->udp_tunnel_nic_info = &liquidio_udp_tunnels;
2113 
2114 		netdev->vlan_features = lio->dev_capability;
2115 		/* Add any unchangeable hw features */
2116 		lio->dev_capability |= NETIF_F_HW_VLAN_CTAG_FILTER |
2117 				       NETIF_F_HW_VLAN_CTAG_RX |
2118 				       NETIF_F_HW_VLAN_CTAG_TX;
2119 
2120 		netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
2121 
2122 		netdev->hw_features = lio->dev_capability;
2123 		netdev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
2124 
2125 		/* MTU range: 68 - 16000 */
2126 		netdev->min_mtu = LIO_MIN_MTU_SIZE;
2127 		netdev->max_mtu = LIO_MAX_MTU_SIZE;
2128 
2129 		WRITE_ONCE(sc->caller_is_done, true);
2130 
2131 		/* Point to the  properties for octeon device to which this
2132 		 * interface belongs.
2133 		 */
2134 		lio->oct_dev = octeon_dev;
2135 		lio->octprops = props;
2136 		lio->netdev = netdev;
2137 
2138 		dev_dbg(&octeon_dev->pci_dev->dev,
2139 			"if%d gmx: %d hw_addr: 0x%llx\n", i,
2140 			lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
2141 
2142 		/* 64-bit swap required on LE machines */
2143 		octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
2144 		for (j = 0; j < ETH_ALEN; j++)
2145 			mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
2146 
2147 		/* Copy MAC Address to OS network device structure */
2148 		eth_hw_addr_set(netdev, mac);
2149 
2150 		if (liquidio_setup_io_queues(octeon_dev, i,
2151 					     lio->linfo.num_txpciq,
2152 					     lio->linfo.num_rxpciq)) {
2153 			dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
2154 			goto setup_nic_dev_free;
2155 		}
2156 
2157 		ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
2158 
2159 		/* For VFs, enable Octeon device interrupts here,
2160 		 * as this is contingent upon IO queue setup
2161 		 */
2162 		octeon_dev->fn_list.enable_interrupt(octeon_dev,
2163 						     OCTEON_ALL_INTR);
2164 
2165 		/* By default all interfaces on a single Octeon uses the same
2166 		 * tx and rx queues
2167 		 */
2168 		lio->txq = lio->linfo.txpciq[0].s.q_no;
2169 		lio->rxq = lio->linfo.rxpciq[0].s.q_no;
2170 
2171 		lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
2172 		lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
2173 
2174 		if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {
2175 			dev_err(&octeon_dev->pci_dev->dev,
2176 				"Gather list allocation failed\n");
2177 			goto setup_nic_dev_free;
2178 		}
2179 
2180 		/* Register ethtool support */
2181 		liquidio_set_ethtool_ops(netdev);
2182 		if (lio->oct_dev->chip_id == OCTEON_CN23XX_VF_VID)
2183 			octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;
2184 		else
2185 			octeon_dev->priv_flags = 0x0;
2186 
2187 		if (netdev->features & NETIF_F_LRO)
2188 			liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
2189 					     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2190 
2191 		if (setup_link_status_change_wq(netdev))
2192 			goto setup_nic_dev_free;
2193 
2194 		if (setup_rx_oom_poll_fn(netdev))
2195 			goto setup_nic_dev_free;
2196 
2197 		/* Register the network device with the OS */
2198 		if (register_netdev(netdev)) {
2199 			dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
2200 			goto setup_nic_dev_free;
2201 		}
2202 
2203 		dev_dbg(&octeon_dev->pci_dev->dev,
2204 			"Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
2205 			i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
2206 		netif_carrier_off(netdev);
2207 		lio->link_changes++;
2208 
2209 		ifstate_set(lio, LIO_IFSTATE_REGISTERED);
2210 
2211 		/* Sending command to firmware to enable Rx checksum offload
2212 		 * by default at the time of setup of Liquidio driver for
2213 		 * this device
2214 		 */
2215 		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
2216 					    OCTNET_CMD_RXCSUM_ENABLE);
2217 		liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
2218 				     OCTNET_CMD_TXCSUM_ENABLE);
2219 
2220 		dev_dbg(&octeon_dev->pci_dev->dev,
2221 			"NIC ifidx:%d Setup successful\n", i);
2222 
2223 		octeon_dev->no_speed_setting = 1;
2224 	}
2225 
2226 	return 0;
2227 
2228 setup_nic_dev_free:
2229 
2230 	while (i--) {
2231 		dev_err(&octeon_dev->pci_dev->dev,
2232 			"NIC ifidx:%d Setup failed\n", i);
2233 		liquidio_destroy_nic_device(octeon_dev, i);
2234 	}
2235 
2236 setup_nic_dev_done:
2237 
2238 	return -ENODEV;
2239 }
2240 
2241 /**
2242  * liquidio_init_nic_module - initialize the NIC
2243  * @oct: octeon device
2244  *
2245  * This initialization routine is called once the Octeon device application is
2246  * up and running
2247  */
2248 static int liquidio_init_nic_module(struct octeon_device *oct)
2249 {
2250 	int num_nic_ports = 1;
2251 	int i, retval = 0;
2252 
2253 	dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
2254 
2255 	/* only default iq and oq were initialized
2256 	 * initialize the rest as well run port_config command for each port
2257 	 */
2258 	oct->ifcount = num_nic_ports;
2259 	memset(oct->props, 0,
2260 	       sizeof(struct octdev_props) * num_nic_ports);
2261 
2262 	for (i = 0; i < MAX_OCTEON_LINKS; i++)
2263 		oct->props[i].gmxport = -1;
2264 
2265 	retval = setup_nic_devices(oct);
2266 	if (retval) {
2267 		dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
2268 		goto octnet_init_failure;
2269 	}
2270 
2271 	dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
2272 
2273 	return retval;
2274 
2275 octnet_init_failure:
2276 
2277 	oct->ifcount = 0;
2278 
2279 	return retval;
2280 }
2281 
2282 /**
2283  * octeon_device_init - Device initialization for each Octeon device that is probed
2284  * @oct:  octeon device
2285  */
2286 static int octeon_device_init(struct octeon_device *oct)
2287 {
2288 	u32 rev_id;
2289 	int j;
2290 
2291 	atomic_set(&oct->status, OCT_DEV_BEGIN_STATE);
2292 
2293 	/* Enable access to the octeon device and make its DMA capability
2294 	 * known to the OS.
2295 	 */
2296 	if (octeon_pci_os_setup(oct))
2297 		return 1;
2298 	atomic_set(&oct->status, OCT_DEV_PCI_ENABLE_DONE);
2299 
2300 	oct->chip_id = OCTEON_CN23XX_VF_VID;
2301 	pci_read_config_dword(oct->pci_dev, 8, &rev_id);
2302 	oct->rev_id = rev_id & 0xff;
2303 
2304 	if (cn23xx_setup_octeon_vf_device(oct))
2305 		return 1;
2306 
2307 	atomic_set(&oct->status, OCT_DEV_PCI_MAP_DONE);
2308 
2309 	oct->app_mode = CVM_DRV_NIC_APP;
2310 
2311 	/* Initialize the dispatch mechanism used to push packets arriving on
2312 	 * Octeon Output queues.
2313 	 */
2314 	if (octeon_init_dispatch_list(oct))
2315 		return 1;
2316 
2317 	atomic_set(&oct->status, OCT_DEV_DISPATCH_INIT_DONE);
2318 
2319 	if (octeon_set_io_queues_off(oct)) {
2320 		dev_err(&oct->pci_dev->dev, "setting io queues off failed\n");
2321 		return 1;
2322 	}
2323 
2324 	if (oct->fn_list.setup_device_regs(oct)) {
2325 		dev_err(&oct->pci_dev->dev, "device registers configuration failed\n");
2326 		return 1;
2327 	}
2328 
2329 	/* Initialize soft command buffer pool */
2330 	if (octeon_setup_sc_buffer_pool(oct)) {
2331 		dev_err(&oct->pci_dev->dev, "sc buffer pool allocation failed\n");
2332 		return 1;
2333 	}
2334 	atomic_set(&oct->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
2335 
2336 	/* Setup the data structures that manage this Octeon's Input queues. */
2337 	if (octeon_setup_instr_queues(oct)) {
2338 		dev_err(&oct->pci_dev->dev, "instruction queue initialization failed\n");
2339 		return 1;
2340 	}
2341 	atomic_set(&oct->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
2342 
2343 	/* Initialize lists to manage the requests of different types that
2344 	 * arrive from user & kernel applications for this octeon device.
2345 	 */
2346 	if (octeon_setup_response_list(oct)) {
2347 		dev_err(&oct->pci_dev->dev, "Response list allocation failed\n");
2348 		return 1;
2349 	}
2350 	atomic_set(&oct->status, OCT_DEV_RESP_LIST_INIT_DONE);
2351 
2352 	if (octeon_setup_output_queues(oct)) {
2353 		dev_err(&oct->pci_dev->dev, "Output queue initialization failed\n");
2354 		return 1;
2355 	}
2356 	atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);
2357 
2358 	if (oct->fn_list.setup_mbox(oct)) {
2359 		dev_err(&oct->pci_dev->dev, "Mailbox setup failed\n");
2360 		return 1;
2361 	}
2362 	atomic_set(&oct->status, OCT_DEV_MBOX_SETUP_DONE);
2363 
2364 	if (octeon_allocate_ioq_vector(oct, oct->sriov_info.rings_per_vf)) {
2365 		dev_err(&oct->pci_dev->dev, "ioq vector allocation failed\n");
2366 		return 1;
2367 	}
2368 	atomic_set(&oct->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE);
2369 
2370 	dev_info(&oct->pci_dev->dev, "OCTEON_CN23XX VF: %d ioqs\n",
2371 		 oct->sriov_info.rings_per_vf);
2372 
2373 	/* Setup the interrupt handler and record the INT SUM register address*/
2374 	if (octeon_setup_interrupt(oct, oct->sriov_info.rings_per_vf))
2375 		return 1;
2376 
2377 	atomic_set(&oct->status, OCT_DEV_INTR_SET_DONE);
2378 
2379 	/* ***************************************************************
2380 	 * The interrupts need to be enabled for the PF<-->VF handshake.
2381 	 * They are [re]-enabled after the PF<-->VF handshake so that the
2382 	 * correct OQ tick value is used (i.e. the value retrieved from
2383 	 * the PF as part of the handshake).
2384 	 */
2385 
2386 	/* Enable Octeon device interrupts */
2387 	oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);
2388 
2389 	if (cn23xx_octeon_pfvf_handshake(oct))
2390 		return 1;
2391 
2392 	/* Here we [re]-enable the interrupts so that the correct OQ tick value
2393 	 * is used (i.e. the value that was retrieved during the handshake)
2394 	 */
2395 
2396 	/* Enable Octeon device interrupts */
2397 	oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);
2398 	/* *************************************************************** */
2399 
2400 	/* Enable the input and output queues for this Octeon device */
2401 	if (oct->fn_list.enable_io_queues(oct)) {
2402 		dev_err(&oct->pci_dev->dev, "enabling io queues failed\n");
2403 		return 1;
2404 	}
2405 
2406 	atomic_set(&oct->status, OCT_DEV_IO_QUEUES_DONE);
2407 
2408 	atomic_set(&oct->status, OCT_DEV_HOST_OK);
2409 
2410 	/* Send Credit for Octeon Output queues. Credits are always sent after
2411 	 * the output queue is enabled.
2412 	 */
2413 	for (j = 0; j < oct->num_oqs; j++)
2414 		writel(oct->droq[j]->max_count, oct->droq[j]->pkts_credit_reg);
2415 
2416 	/* Packets can start arriving on the output queues from this point. */
2417 
2418 	atomic_set(&oct->status, OCT_DEV_CORE_OK);
2419 
2420 	atomic_set(&oct->status, OCT_DEV_RUNNING);
2421 
2422 	if (liquidio_init_nic_module(oct))
2423 		return 1;
2424 
2425 	return 0;
2426 }
2427 
2428 static int __init liquidio_vf_init(void)
2429 {
2430 	octeon_init_device_list(0);
2431 	return pci_register_driver(&liquidio_vf_pci_driver);
2432 }
2433 
2434 static void __exit liquidio_vf_exit(void)
2435 {
2436 	pci_unregister_driver(&liquidio_vf_pci_driver);
2437 
2438 	pr_info("LiquidIO_VF network module is now unloaded\n");
2439 }
2440 
2441 module_init(liquidio_vf_init);
2442 module_exit(liquidio_vf_exit);
2443