1 /**********************************************************************
2  * Author: Cavium, Inc.
3  *
4  * Contact: support@cavium.com
5  *          Please include "LiquidIO" in the subject.
6  *
7  * Copyright (c) 2003-2016 Cavium, Inc.
8  *
9  * This file is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License, Version 2, as
11  * published by the Free Software Foundation.
12  *
13  * This file is distributed in the hope that it will be useful, but
14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16  * NONINFRINGEMENT.  See the GNU General Public License for more details.
17  ***********************************************************************/
18 #include <linux/module.h>
19 #include <linux/interrupt.h>
20 #include <linux/pci.h>
21 #include <net/vxlan.h>
22 #include "liquidio_common.h"
23 #include "octeon_droq.h"
24 #include "octeon_iq.h"
25 #include "response_manager.h"
26 #include "octeon_device.h"
27 #include "octeon_nic.h"
28 #include "octeon_main.h"
29 #include "octeon_network.h"
30 #include "cn23xx_vf_device.h"
31 
32 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
33 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Virtual Function Driver");
34 MODULE_LICENSE("GPL");
35 
36 static int debug = -1;
37 module_param(debug, int, 0644);
38 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
39 
40 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
41 
42 struct oct_timestamp_resp {
43 	u64 rh;
44 	u64 timestamp;
45 	u64 status;
46 };
47 
48 union tx_info {
49 	u64 u64;
50 	struct {
51 #ifdef __BIG_ENDIAN_BITFIELD
52 		u16 gso_size;
53 		u16 gso_segs;
54 		u32 reserved;
55 #else
56 		u32 reserved;
57 		u16 gso_segs;
58 		u16 gso_size;
59 #endif
60 	} s;
61 };
62 
63 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
64 #define OCTNIC_GSO_MAX_SIZE \
65 		(CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
66 
67 static int
68 liquidio_vf_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
69 static void liquidio_vf_remove(struct pci_dev *pdev);
70 static int octeon_device_init(struct octeon_device *oct);
71 static int liquidio_stop(struct net_device *netdev);
72 
73 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
74 {
75 	struct octeon_device_priv *oct_priv =
76 	    (struct octeon_device_priv *)oct->priv;
77 	int retry = MAX_IO_PENDING_PKT_COUNT;
78 	int pkt_cnt = 0, pending_pkts;
79 	int i;
80 
81 	do {
82 		pending_pkts = 0;
83 
84 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
85 			if (!(oct->io_qmask.oq & BIT_ULL(i)))
86 				continue;
87 			pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
88 		}
89 		if (pkt_cnt > 0) {
90 			pending_pkts += pkt_cnt;
91 			tasklet_schedule(&oct_priv->droq_tasklet);
92 		}
93 		pkt_cnt = 0;
94 		schedule_timeout_uninterruptible(1);
95 
96 	} while (retry-- && pending_pkts);
97 
98 	return pkt_cnt;
99 }
100 
101 /**
102  * \brief Cause device to go quiet so it can be safely removed/reset/etc
103  * @param oct Pointer to Octeon device
104  */
105 static void pcierror_quiesce_device(struct octeon_device *oct)
106 {
107 	int i;
108 
109 	/* Disable the input and output queues now. No more packets will
110 	 * arrive from Octeon, but we should wait for all packet processing
111 	 * to finish.
112 	 */
113 
114 	/* To allow for in-flight requests */
115 	schedule_timeout_uninterruptible(100);
116 
117 	if (wait_for_pending_requests(oct))
118 		dev_err(&oct->pci_dev->dev, "There were pending requests\n");
119 
120 	/* Force all requests waiting to be fetched by OCTEON to complete. */
121 	for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
122 		struct octeon_instr_queue *iq;
123 
124 		if (!(oct->io_qmask.iq & BIT_ULL(i)))
125 			continue;
126 		iq = oct->instr_queue[i];
127 
128 		if (atomic_read(&iq->instr_pending)) {
129 			spin_lock_bh(&iq->lock);
130 			iq->fill_cnt = 0;
131 			iq->octeon_read_index = iq->host_write_index;
132 			iq->stats.instr_processed +=
133 			    atomic_read(&iq->instr_pending);
134 			lio_process_iq_request_list(oct, iq, 0);
135 			spin_unlock_bh(&iq->lock);
136 		}
137 	}
138 
139 	/* Force all pending ordered list requests to time out. */
140 	lio_process_ordered_list(oct, 1);
141 
142 	/* We do not need to wait for output queue packets to be processed. */
143 }
144 
145 /**
146  * \brief Cleanup PCI AER uncorrectable error status
147  * @param dev Pointer to PCI device
148  */
149 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
150 {
151 	u32 status, mask;
152 	int pos = 0x100;
153 
154 	pr_info("%s :\n", __func__);
155 
156 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
157 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
158 	if (dev->error_state == pci_channel_io_normal)
159 		status &= ~mask; /* Clear corresponding nonfatal bits */
160 	else
161 		status &= mask; /* Clear corresponding fatal bits */
162 	pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
163 }
164 
165 /**
166  * \brief Stop all PCI IO to a given device
167  * @param dev Pointer to Octeon device
168  */
169 static void stop_pci_io(struct octeon_device *oct)
170 {
171 	struct msix_entry *msix_entries;
172 	int i;
173 
174 	/* No more instructions will be forwarded. */
175 	atomic_set(&oct->status, OCT_DEV_IN_RESET);
176 
177 	for (i = 0; i < oct->ifcount; i++)
178 		netif_device_detach(oct->props[i].netdev);
179 
180 	/* Disable interrupts  */
181 	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
182 
183 	pcierror_quiesce_device(oct);
184 	if (oct->msix_on) {
185 		msix_entries = (struct msix_entry *)oct->msix_entries;
186 		for (i = 0; i < oct->num_msix_irqs; i++) {
187 			/* clear the affinity_cpumask */
188 			irq_set_affinity_hint(msix_entries[i].vector,
189 					      NULL);
190 			free_irq(msix_entries[i].vector,
191 				 &oct->ioq_vector[i]);
192 		}
193 		pci_disable_msix(oct->pci_dev);
194 		kfree(oct->msix_entries);
195 		oct->msix_entries = NULL;
196 		octeon_free_ioq_vector(oct);
197 	}
198 	dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
199 		lio_get_state_string(&oct->status));
200 
201 	/* making it a common function for all OCTEON models */
202 	cleanup_aer_uncorrect_error_status(oct->pci_dev);
203 
204 	pci_disable_device(oct->pci_dev);
205 }
206 
207 /**
208  * \brief called when PCI error is detected
209  * @param pdev Pointer to PCI device
210  * @param state The current pci connection state
211  *
212  * This function is called after a PCI bus error affecting
213  * this device has been detected.
214  */
215 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
216 						     pci_channel_state_t state)
217 {
218 	struct octeon_device *oct = pci_get_drvdata(pdev);
219 
220 	/* Non-correctable Non-fatal errors */
221 	if (state == pci_channel_io_normal) {
222 		dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
223 		cleanup_aer_uncorrect_error_status(oct->pci_dev);
224 		return PCI_ERS_RESULT_CAN_RECOVER;
225 	}
226 
227 	/* Non-correctable Fatal errors */
228 	dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
229 	stop_pci_io(oct);
230 
231 	return PCI_ERS_RESULT_DISCONNECT;
232 }
233 
234 /* For PCI-E Advanced Error Recovery (AER) Interface */
235 static const struct pci_error_handlers liquidio_vf_err_handler = {
236 	.error_detected = liquidio_pcie_error_detected,
237 };
238 
239 static const struct pci_device_id liquidio_vf_pci_tbl[] = {
240 	{
241 		PCI_VENDOR_ID_CAVIUM, OCTEON_CN23XX_VF_VID,
242 		PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
243 	},
244 	{
245 		0, 0, 0, 0, 0, 0, 0
246 	}
247 };
248 MODULE_DEVICE_TABLE(pci, liquidio_vf_pci_tbl);
249 
250 static struct pci_driver liquidio_vf_pci_driver = {
251 	.name		= "LiquidIO_VF",
252 	.id_table	= liquidio_vf_pci_tbl,
253 	.probe		= liquidio_vf_probe,
254 	.remove		= liquidio_vf_remove,
255 	.err_handler	= &liquidio_vf_err_handler,    /* For AER */
256 };
257 
258 /**
259  * \brief Print link information
260  * @param netdev network device
261  */
262 static void print_link_info(struct net_device *netdev)
263 {
264 	struct lio *lio = GET_LIO(netdev);
265 
266 	if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
267 	    ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
268 		struct oct_link_info *linfo = &lio->linfo;
269 
270 		if (linfo->link.s.link_up) {
271 			netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
272 				   linfo->link.s.speed,
273 				   (linfo->link.s.duplex) ? "Full" : "Half");
274 		} else {
275 			netif_info(lio, link, lio->netdev, "Link Down\n");
276 		}
277 	}
278 }
279 
280 /**
281  * \brief Routine to notify MTU change
282  * @param work work_struct data structure
283  */
284 static void octnet_link_status_change(struct work_struct *work)
285 {
286 	struct cavium_wk *wk = (struct cavium_wk *)work;
287 	struct lio *lio = (struct lio *)wk->ctxptr;
288 
289 	/* lio->linfo.link.s.mtu always contains max MTU of the lio interface.
290 	 * this API is invoked only when new max-MTU of the interface is
291 	 * less than current MTU.
292 	 */
293 	rtnl_lock();
294 	dev_set_mtu(lio->netdev, lio->linfo.link.s.mtu);
295 	rtnl_unlock();
296 }
297 
298 /**
299  * \brief Sets up the mtu status change work
300  * @param netdev network device
301  */
302 static int setup_link_status_change_wq(struct net_device *netdev)
303 {
304 	struct lio *lio = GET_LIO(netdev);
305 	struct octeon_device *oct = lio->oct_dev;
306 
307 	lio->link_status_wq.wq = alloc_workqueue("link-status",
308 						 WQ_MEM_RECLAIM, 0);
309 	if (!lio->link_status_wq.wq) {
310 		dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");
311 		return -1;
312 	}
313 	INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,
314 			  octnet_link_status_change);
315 	lio->link_status_wq.wk.ctxptr = lio;
316 
317 	return 0;
318 }
319 
320 static void cleanup_link_status_change_wq(struct net_device *netdev)
321 {
322 	struct lio *lio = GET_LIO(netdev);
323 
324 	if (lio->link_status_wq.wq) {
325 		cancel_delayed_work_sync(&lio->link_status_wq.wk.work);
326 		destroy_workqueue(lio->link_status_wq.wq);
327 	}
328 }
329 
330 /**
331  * \brief Update link status
332  * @param netdev network device
333  * @param ls link status structure
334  *
335  * Called on receipt of a link status response from the core application to
336  * update each interface's link status.
337  */
338 static void update_link_status(struct net_device *netdev,
339 			       union oct_link_status *ls)
340 {
341 	struct lio *lio = GET_LIO(netdev);
342 	int current_max_mtu = lio->linfo.link.s.mtu;
343 	struct octeon_device *oct = lio->oct_dev;
344 
345 	if ((lio->intf_open) && (lio->linfo.link.u64 != ls->u64)) {
346 		lio->linfo.link.u64 = ls->u64;
347 
348 		print_link_info(netdev);
349 		lio->link_changes++;
350 
351 		if (lio->linfo.link.s.link_up) {
352 			netif_carrier_on(netdev);
353 			wake_txqs(netdev);
354 		} else {
355 			netif_carrier_off(netdev);
356 			stop_txqs(netdev);
357 		}
358 
359 		if (lio->linfo.link.s.mtu != current_max_mtu) {
360 			dev_info(&oct->pci_dev->dev,
361 				 "Max MTU Changed from %d to %d\n",
362 				 current_max_mtu, lio->linfo.link.s.mtu);
363 			netdev->max_mtu = lio->linfo.link.s.mtu;
364 		}
365 
366 		if (lio->linfo.link.s.mtu < netdev->mtu) {
367 			dev_warn(&oct->pci_dev->dev,
368 				 "Current MTU is higher than new max MTU; Reducing the current mtu from %d to %d\n",
369 				 netdev->mtu, lio->linfo.link.s.mtu);
370 			queue_delayed_work(lio->link_status_wq.wq,
371 					   &lio->link_status_wq.wk.work, 0);
372 		}
373 	}
374 }
375 
376 /**
377  * \brief PCI probe handler
378  * @param pdev PCI device structure
379  * @param ent unused
380  */
381 static int
382 liquidio_vf_probe(struct pci_dev *pdev,
383 		  const struct pci_device_id *ent __attribute__((unused)))
384 {
385 	struct octeon_device *oct_dev = NULL;
386 
387 	oct_dev = octeon_allocate_device(pdev->device,
388 					 sizeof(struct octeon_device_priv));
389 
390 	if (!oct_dev) {
391 		dev_err(&pdev->dev, "Unable to allocate device\n");
392 		return -ENOMEM;
393 	}
394 	oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
395 
396 	dev_info(&pdev->dev, "Initializing device %x:%x.\n",
397 		 (u32)pdev->vendor, (u32)pdev->device);
398 
399 	/* Assign octeon_device for this device to the private data area. */
400 	pci_set_drvdata(pdev, oct_dev);
401 
402 	/* set linux specific device pointer */
403 	oct_dev->pci_dev = pdev;
404 
405 	oct_dev->subsystem_id = pdev->subsystem_vendor |
406 		(pdev->subsystem_device << 16);
407 
408 	if (octeon_device_init(oct_dev)) {
409 		liquidio_vf_remove(pdev);
410 		return -ENOMEM;
411 	}
412 
413 	dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
414 
415 	return 0;
416 }
417 
418 /**
419  * \brief PCI FLR for each Octeon device.
420  * @param oct octeon device
421  */
422 static void octeon_pci_flr(struct octeon_device *oct)
423 {
424 	pci_save_state(oct->pci_dev);
425 
426 	pci_cfg_access_lock(oct->pci_dev);
427 
428 	/* Quiesce the device completely */
429 	pci_write_config_word(oct->pci_dev, PCI_COMMAND,
430 			      PCI_COMMAND_INTX_DISABLE);
431 
432 	pcie_flr(oct->pci_dev);
433 
434 	pci_cfg_access_unlock(oct->pci_dev);
435 
436 	pci_restore_state(oct->pci_dev);
437 }
438 
439 /**
440  *\brief Destroy resources associated with octeon device
441  * @param pdev PCI device structure
442  * @param ent unused
443  */
444 static void octeon_destroy_resources(struct octeon_device *oct)
445 {
446 	struct octeon_device_priv *oct_priv =
447 		(struct octeon_device_priv *)oct->priv;
448 	struct msix_entry *msix_entries;
449 	int i;
450 
451 	switch (atomic_read(&oct->status)) {
452 	case OCT_DEV_RUNNING:
453 	case OCT_DEV_CORE_OK:
454 		/* No more instructions will be forwarded. */
455 		atomic_set(&oct->status, OCT_DEV_IN_RESET);
456 
457 		oct->app_mode = CVM_DRV_INVALID_APP;
458 		dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
459 			lio_get_state_string(&oct->status));
460 
461 		schedule_timeout_uninterruptible(HZ / 10);
462 
463 		/* fallthrough */
464 	case OCT_DEV_HOST_OK:
465 		/* fallthrough */
466 	case OCT_DEV_IO_QUEUES_DONE:
467 		if (lio_wait_for_instr_fetch(oct))
468 			dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
469 
470 		if (wait_for_pending_requests(oct))
471 			dev_err(&oct->pci_dev->dev, "There were pending requests\n");
472 
473 		/* Disable the input and output queues now. No more packets will
474 		 * arrive from Octeon, but we should wait for all packet
475 		 * processing to finish.
476 		 */
477 		oct->fn_list.disable_io_queues(oct);
478 
479 		if (lio_wait_for_oq_pkts(oct))
480 			dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
481 
482 		/* Force all requests waiting to be fetched by OCTEON to
483 		 * complete.
484 		 */
485 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
486 			struct octeon_instr_queue *iq;
487 
488 			if (!(oct->io_qmask.iq & BIT_ULL(i)))
489 				continue;
490 			iq = oct->instr_queue[i];
491 
492 			if (atomic_read(&iq->instr_pending)) {
493 				spin_lock_bh(&iq->lock);
494 				iq->fill_cnt = 0;
495 				iq->octeon_read_index = iq->host_write_index;
496 				iq->stats.instr_processed +=
497 					atomic_read(&iq->instr_pending);
498 				lio_process_iq_request_list(oct, iq, 0);
499 				spin_unlock_bh(&iq->lock);
500 			}
501 		}
502 
503 		lio_process_ordered_list(oct, 1);
504 		octeon_free_sc_done_list(oct);
505 		octeon_free_sc_zombie_list(oct);
506 
507 	/* fall through */
508 	case OCT_DEV_INTR_SET_DONE:
509 		/* Disable interrupts  */
510 		oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
511 
512 		if (oct->msix_on) {
513 			msix_entries = (struct msix_entry *)oct->msix_entries;
514 			for (i = 0; i < oct->num_msix_irqs; i++) {
515 				if (oct->ioq_vector[i].vector) {
516 					irq_set_affinity_hint(
517 							msix_entries[i].vector,
518 							NULL);
519 					free_irq(msix_entries[i].vector,
520 						 &oct->ioq_vector[i]);
521 					oct->ioq_vector[i].vector = 0;
522 				}
523 			}
524 			pci_disable_msix(oct->pci_dev);
525 			kfree(oct->msix_entries);
526 			oct->msix_entries = NULL;
527 			kfree(oct->irq_name_storage);
528 			oct->irq_name_storage = NULL;
529 		}
530 		/* Soft reset the octeon device before exiting */
531 		if (oct->pci_dev->reset_fn)
532 			octeon_pci_flr(oct);
533 		else
534 			cn23xx_vf_ask_pf_to_do_flr(oct);
535 
536 		/* fallthrough */
537 	case OCT_DEV_MSIX_ALLOC_VECTOR_DONE:
538 		octeon_free_ioq_vector(oct);
539 
540 		/* fallthrough */
541 	case OCT_DEV_MBOX_SETUP_DONE:
542 		oct->fn_list.free_mbox(oct);
543 
544 		/* fallthrough */
545 	case OCT_DEV_IN_RESET:
546 	case OCT_DEV_DROQ_INIT_DONE:
547 		mdelay(100);
548 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
549 			if (!(oct->io_qmask.oq & BIT_ULL(i)))
550 				continue;
551 			octeon_delete_droq(oct, i);
552 		}
553 
554 		/* fallthrough */
555 	case OCT_DEV_RESP_LIST_INIT_DONE:
556 		octeon_delete_response_list(oct);
557 
558 		/* fallthrough */
559 	case OCT_DEV_INSTR_QUEUE_INIT_DONE:
560 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
561 			if (!(oct->io_qmask.iq & BIT_ULL(i)))
562 				continue;
563 			octeon_delete_instr_queue(oct, i);
564 		}
565 
566 		/* fallthrough */
567 	case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
568 		octeon_free_sc_buffer_pool(oct);
569 
570 		/* fallthrough */
571 	case OCT_DEV_DISPATCH_INIT_DONE:
572 		octeon_delete_dispatch_list(oct);
573 		cancel_delayed_work_sync(&oct->nic_poll_work.work);
574 
575 		/* fallthrough */
576 	case OCT_DEV_PCI_MAP_DONE:
577 		octeon_unmap_pci_barx(oct, 0);
578 		octeon_unmap_pci_barx(oct, 1);
579 
580 		/* fallthrough */
581 	case OCT_DEV_PCI_ENABLE_DONE:
582 		pci_clear_master(oct->pci_dev);
583 		/* Disable the device, releasing the PCI INT */
584 		pci_disable_device(oct->pci_dev);
585 
586 		/* fallthrough */
587 	case OCT_DEV_BEGIN_STATE:
588 		/* Nothing to be done here either */
589 		break;
590 	}
591 
592 	tasklet_kill(&oct_priv->droq_tasklet);
593 }
594 
595 /**
596  * \brief Send Rx control command
597  * @param lio per-network private data
598  * @param start_stop whether to start or stop
599  */
600 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
601 {
602 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
603 	struct octeon_soft_command *sc;
604 	union octnet_cmd *ncmd;
605 	int retval;
606 
607 	if (oct->props[lio->ifidx].rx_on == start_stop)
608 		return;
609 
610 	sc = (struct octeon_soft_command *)
611 		octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
612 					  16, 0);
613 
614 	ncmd = (union octnet_cmd *)sc->virtdptr;
615 
616 	ncmd->u64 = 0;
617 	ncmd->s.cmd = OCTNET_CMD_RX_CTL;
618 	ncmd->s.param1 = start_stop;
619 
620 	octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
621 
622 	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
623 
624 	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
625 				    OPCODE_NIC_CMD, 0, 0, 0);
626 
627 	init_completion(&sc->complete);
628 	sc->sc_status = OCTEON_REQUEST_PENDING;
629 
630 	retval = octeon_send_soft_command(oct, sc);
631 	if (retval == IQ_SEND_FAILED) {
632 		netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
633 		octeon_free_soft_command(oct, sc);
634 	} else {
635 		/* Sleep on a wait queue till the cond flag indicates that the
636 		 * response arrived or timed-out.
637 		 */
638 		retval = wait_for_sc_completion_timeout(oct, sc, 0);
639 		if (retval)
640 			return;
641 
642 		oct->props[lio->ifidx].rx_on = start_stop;
643 		WRITE_ONCE(sc->caller_is_done, true);
644 	}
645 }
646 
647 /**
648  * \brief Destroy NIC device interface
649  * @param oct octeon device
650  * @param ifidx which interface to destroy
651  *
652  * Cleanup associated with each interface for an Octeon device  when NIC
653  * module is being unloaded or if initialization fails during load.
654  */
655 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
656 {
657 	struct net_device *netdev = oct->props[ifidx].netdev;
658 	struct octeon_device_priv *oct_priv =
659 		(struct octeon_device_priv *)oct->priv;
660 	struct napi_struct *napi, *n;
661 	struct lio *lio;
662 
663 	if (!netdev) {
664 		dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
665 			__func__, ifidx);
666 		return;
667 	}
668 
669 	lio = GET_LIO(netdev);
670 
671 	dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
672 
673 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
674 		liquidio_stop(netdev);
675 
676 	if (oct->props[lio->ifidx].napi_enabled == 1) {
677 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
678 			napi_disable(napi);
679 
680 		oct->props[lio->ifidx].napi_enabled = 0;
681 
682 		oct->droq[0]->ops.poll_mode = 0;
683 	}
684 
685 	/* Delete NAPI */
686 	list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
687 		netif_napi_del(napi);
688 
689 	tasklet_enable(&oct_priv->droq_tasklet);
690 
691 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
692 		unregister_netdev(netdev);
693 
694 	cleanup_rx_oom_poll_fn(netdev);
695 
696 	cleanup_link_status_change_wq(netdev);
697 
698 	lio_delete_glists(lio);
699 
700 	free_netdev(netdev);
701 
702 	oct->props[ifidx].gmxport = -1;
703 
704 	oct->props[ifidx].netdev = NULL;
705 }
706 
707 /**
708  * \brief Stop complete NIC functionality
709  * @param oct octeon device
710  */
711 static int liquidio_stop_nic_module(struct octeon_device *oct)
712 {
713 	struct lio *lio;
714 	int i, j;
715 
716 	dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
717 	if (!oct->ifcount) {
718 		dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
719 		return 1;
720 	}
721 
722 	spin_lock_bh(&oct->cmd_resp_wqlock);
723 	oct->cmd_resp_state = OCT_DRV_OFFLINE;
724 	spin_unlock_bh(&oct->cmd_resp_wqlock);
725 
726 	for (i = 0; i < oct->ifcount; i++) {
727 		lio = GET_LIO(oct->props[i].netdev);
728 		for (j = 0; j < oct->num_oqs; j++)
729 			octeon_unregister_droq_ops(oct,
730 						   lio->linfo.rxpciq[j].s.q_no);
731 	}
732 
733 	for (i = 0; i < oct->ifcount; i++)
734 		liquidio_destroy_nic_device(oct, i);
735 
736 	dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
737 	return 0;
738 }
739 
740 /**
741  * \brief Cleans up resources at unload time
742  * @param pdev PCI device structure
743  */
744 static void liquidio_vf_remove(struct pci_dev *pdev)
745 {
746 	struct octeon_device *oct_dev = pci_get_drvdata(pdev);
747 
748 	dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
749 
750 	if (oct_dev->app_mode == CVM_DRV_NIC_APP)
751 		liquidio_stop_nic_module(oct_dev);
752 
753 	/* Reset the octeon device and cleanup all memory allocated for
754 	 * the octeon device by driver.
755 	 */
756 	octeon_destroy_resources(oct_dev);
757 
758 	dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
759 
760 	/* This octeon device has been removed. Update the global
761 	 * data structure to reflect this. Free the device structure.
762 	 */
763 	octeon_free_device_mem(oct_dev);
764 }
765 
766 /**
767  * \brief PCI initialization for each Octeon device.
768  * @param oct octeon device
769  */
770 static int octeon_pci_os_setup(struct octeon_device *oct)
771 {
772 #ifdef CONFIG_PCI_IOV
773 	/* setup PCI stuff first */
774 	if (!oct->pci_dev->physfn)
775 		octeon_pci_flr(oct);
776 #endif
777 
778 	if (pci_enable_device(oct->pci_dev)) {
779 		dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
780 		return 1;
781 	}
782 
783 	if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
784 		dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
785 		pci_disable_device(oct->pci_dev);
786 		return 1;
787 	}
788 
789 	/* Enable PCI DMA Master. */
790 	pci_set_master(oct->pci_dev);
791 
792 	return 0;
793 }
794 
795 /**
796  * \brief Unmap and free network buffer
797  * @param buf buffer
798  */
799 static void free_netbuf(void *buf)
800 {
801 	struct octnet_buf_free_info *finfo;
802 	struct sk_buff *skb;
803 	struct lio *lio;
804 
805 	finfo = (struct octnet_buf_free_info *)buf;
806 	skb = finfo->skb;
807 	lio = finfo->lio;
808 
809 	dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
810 			 DMA_TO_DEVICE);
811 
812 	tx_buffer_free(skb);
813 }
814 
815 /**
816  * \brief Unmap and free gather buffer
817  * @param buf buffer
818  */
819 static void free_netsgbuf(void *buf)
820 {
821 	struct octnet_buf_free_info *finfo;
822 	struct octnic_gather *g;
823 	struct sk_buff *skb;
824 	int i, frags, iq;
825 	struct lio *lio;
826 
827 	finfo = (struct octnet_buf_free_info *)buf;
828 	skb = finfo->skb;
829 	lio = finfo->lio;
830 	g = finfo->g;
831 	frags = skb_shinfo(skb)->nr_frags;
832 
833 	dma_unmap_single(&lio->oct_dev->pci_dev->dev,
834 			 g->sg[0].ptr[0], (skb->len - skb->data_len),
835 			 DMA_TO_DEVICE);
836 
837 	i = 1;
838 	while (frags--) {
839 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
840 
841 		pci_unmap_page((lio->oct_dev)->pci_dev,
842 			       g->sg[(i >> 2)].ptr[(i & 3)],
843 			       skb_frag_size(frag), DMA_TO_DEVICE);
844 		i++;
845 	}
846 
847 	iq = skb_iq(lio->oct_dev, skb);
848 
849 	spin_lock(&lio->glist_lock[iq]);
850 	list_add_tail(&g->list, &lio->glist[iq]);
851 	spin_unlock(&lio->glist_lock[iq]);
852 
853 	tx_buffer_free(skb);
854 }
855 
856 /**
857  * \brief Unmap and free gather buffer with response
858  * @param buf buffer
859  */
860 static void free_netsgbuf_with_resp(void *buf)
861 {
862 	struct octnet_buf_free_info *finfo;
863 	struct octeon_soft_command *sc;
864 	struct octnic_gather *g;
865 	struct sk_buff *skb;
866 	int i, frags, iq;
867 	struct lio *lio;
868 
869 	sc = (struct octeon_soft_command *)buf;
870 	skb = (struct sk_buff *)sc->callback_arg;
871 	finfo = (struct octnet_buf_free_info *)&skb->cb;
872 
873 	lio = finfo->lio;
874 	g = finfo->g;
875 	frags = skb_shinfo(skb)->nr_frags;
876 
877 	dma_unmap_single(&lio->oct_dev->pci_dev->dev,
878 			 g->sg[0].ptr[0], (skb->len - skb->data_len),
879 			 DMA_TO_DEVICE);
880 
881 	i = 1;
882 	while (frags--) {
883 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
884 
885 		pci_unmap_page((lio->oct_dev)->pci_dev,
886 			       g->sg[(i >> 2)].ptr[(i & 3)],
887 			       skb_frag_size(frag), DMA_TO_DEVICE);
888 		i++;
889 	}
890 
891 	iq = skb_iq(lio->oct_dev, skb);
892 
893 	spin_lock(&lio->glist_lock[iq]);
894 	list_add_tail(&g->list, &lio->glist[iq]);
895 	spin_unlock(&lio->glist_lock[iq]);
896 
897 	/* Don't free the skb yet */
898 }
899 
900 /**
901  * \brief Net device open for LiquidIO
902  * @param netdev network device
903  */
904 static int liquidio_open(struct net_device *netdev)
905 {
906 	struct lio *lio = GET_LIO(netdev);
907 	struct octeon_device *oct = lio->oct_dev;
908 	struct octeon_device_priv *oct_priv =
909 		(struct octeon_device_priv *)oct->priv;
910 	struct napi_struct *napi, *n;
911 
912 	if (!oct->props[lio->ifidx].napi_enabled) {
913 		tasklet_disable(&oct_priv->droq_tasklet);
914 
915 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
916 			napi_enable(napi);
917 
918 		oct->props[lio->ifidx].napi_enabled = 1;
919 
920 		oct->droq[0]->ops.poll_mode = 1;
921 	}
922 
923 	ifstate_set(lio, LIO_IFSTATE_RUNNING);
924 
925 	/* Ready for link status updates */
926 	lio->intf_open = 1;
927 
928 	netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
929 	start_txqs(netdev);
930 
931 	INIT_DELAYED_WORK(&lio->stats_wk.work, lio_fetch_stats);
932 	lio->stats_wk.ctxptr = lio;
933 	schedule_delayed_work(&lio->stats_wk.work, msecs_to_jiffies
934 					(LIQUIDIO_NDEV_STATS_POLL_TIME_MS));
935 
936 	/* tell Octeon to start forwarding packets to host */
937 	send_rx_ctrl_cmd(lio, 1);
938 
939 	dev_info(&oct->pci_dev->dev, "%s interface is opened\n", netdev->name);
940 
941 	return 0;
942 }
943 
944 /**
945  * \brief Net device stop for LiquidIO
946  * @param netdev network device
947  */
948 static int liquidio_stop(struct net_device *netdev)
949 {
950 	struct lio *lio = GET_LIO(netdev);
951 	struct octeon_device *oct = lio->oct_dev;
952 	struct octeon_device_priv *oct_priv =
953 		(struct octeon_device_priv *)oct->priv;
954 	struct napi_struct *napi, *n;
955 
956 	/* tell Octeon to stop forwarding packets to host */
957 	send_rx_ctrl_cmd(lio, 0);
958 
959 	netif_info(lio, ifdown, lio->netdev, "Stopping interface!\n");
960 	/* Inform that netif carrier is down */
961 	lio->intf_open = 0;
962 	lio->linfo.link.s.link_up = 0;
963 
964 	netif_carrier_off(netdev);
965 	lio->link_changes++;
966 
967 	ifstate_reset(lio, LIO_IFSTATE_RUNNING);
968 
969 	stop_txqs(netdev);
970 
971 	/* Wait for any pending Rx descriptors */
972 	if (lio_wait_for_clean_oq(oct))
973 		netif_info(lio, rx_err, lio->netdev,
974 			   "Proceeding with stop interface after partial RX desc processing\n");
975 
976 	if (oct->props[lio->ifidx].napi_enabled == 1) {
977 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
978 			napi_disable(napi);
979 
980 		oct->props[lio->ifidx].napi_enabled = 0;
981 
982 		oct->droq[0]->ops.poll_mode = 0;
983 
984 		tasklet_enable(&oct_priv->droq_tasklet);
985 	}
986 
987 	cancel_delayed_work_sync(&lio->stats_wk.work);
988 
989 	dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
990 
991 	return 0;
992 }
993 
994 /**
995  * \brief Converts a mask based on net device flags
996  * @param netdev network device
997  *
998  * This routine generates a octnet_ifflags mask from the net device flags
999  * received from the OS.
1000  */
1001 static enum octnet_ifflags get_new_flags(struct net_device *netdev)
1002 {
1003 	enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
1004 
1005 	if (netdev->flags & IFF_PROMISC)
1006 		f |= OCTNET_IFFLAG_PROMISC;
1007 
1008 	if (netdev->flags & IFF_ALLMULTI)
1009 		f |= OCTNET_IFFLAG_ALLMULTI;
1010 
1011 	if (netdev->flags & IFF_MULTICAST) {
1012 		f |= OCTNET_IFFLAG_MULTICAST;
1013 
1014 		/* Accept all multicast addresses if there are more than we
1015 		 * can handle
1016 		 */
1017 		if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
1018 			f |= OCTNET_IFFLAG_ALLMULTI;
1019 	}
1020 
1021 	if (netdev->flags & IFF_BROADCAST)
1022 		f |= OCTNET_IFFLAG_BROADCAST;
1023 
1024 	return f;
1025 }
1026 
1027 static void liquidio_set_uc_list(struct net_device *netdev)
1028 {
1029 	struct lio *lio = GET_LIO(netdev);
1030 	struct octeon_device *oct = lio->oct_dev;
1031 	struct octnic_ctrl_pkt nctrl;
1032 	struct netdev_hw_addr *ha;
1033 	u64 *mac;
1034 
1035 	if (lio->netdev_uc_count == netdev_uc_count(netdev))
1036 		return;
1037 
1038 	if (netdev_uc_count(netdev) > MAX_NCTRL_UDD) {
1039 		dev_err(&oct->pci_dev->dev, "too many MAC addresses in netdev uc list\n");
1040 		return;
1041 	}
1042 
1043 	lio->netdev_uc_count = netdev_uc_count(netdev);
1044 
1045 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1046 	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_UC_LIST;
1047 	nctrl.ncmd.s.more = lio->netdev_uc_count;
1048 	nctrl.ncmd.s.param1 = oct->vf_num;
1049 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1050 	nctrl.netpndev = (u64)netdev;
1051 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1052 
1053 	/* copy all the addresses into the udd */
1054 	mac = &nctrl.udd[0];
1055 	netdev_for_each_uc_addr(ha, netdev) {
1056 		ether_addr_copy(((u8 *)mac) + 2, ha->addr);
1057 		mac++;
1058 	}
1059 
1060 	octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1061 }
1062 
1063 /**
1064  * \brief Net device set_multicast_list
1065  * @param netdev network device
1066  */
1067 static void liquidio_set_mcast_list(struct net_device *netdev)
1068 {
1069 	int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
1070 	struct lio *lio = GET_LIO(netdev);
1071 	struct octeon_device *oct = lio->oct_dev;
1072 	struct octnic_ctrl_pkt nctrl;
1073 	struct netdev_hw_addr *ha;
1074 	u64 *mc;
1075 	int ret;
1076 
1077 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1078 
1079 	/* Create a ctrl pkt command to be sent to core app. */
1080 	nctrl.ncmd.u64 = 0;
1081 	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
1082 	nctrl.ncmd.s.param1 = get_new_flags(netdev);
1083 	nctrl.ncmd.s.param2 = mc_count;
1084 	nctrl.ncmd.s.more = mc_count;
1085 	nctrl.netpndev = (u64)netdev;
1086 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1087 
1088 	/* copy all the addresses into the udd */
1089 	mc = &nctrl.udd[0];
1090 	netdev_for_each_mc_addr(ha, netdev) {
1091 		*mc = 0;
1092 		ether_addr_copy(((u8 *)mc) + 2, ha->addr);
1093 		/* no need to swap bytes */
1094 		if (++mc > &nctrl.udd[mc_count])
1095 			break;
1096 	}
1097 
1098 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1099 
1100 	/* Apparently, any activity in this call from the kernel has to
1101 	 * be atomic. So we won't wait for response.
1102 	 */
1103 
1104 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1105 	if (ret) {
1106 		dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
1107 			ret);
1108 	}
1109 
1110 	liquidio_set_uc_list(netdev);
1111 }
1112 
1113 /**
1114  * \brief Net device set_mac_address
1115  * @param netdev network device
1116  */
1117 static int liquidio_set_mac(struct net_device *netdev, void *p)
1118 {
1119 	struct sockaddr *addr = (struct sockaddr *)p;
1120 	struct lio *lio = GET_LIO(netdev);
1121 	struct octeon_device *oct = lio->oct_dev;
1122 	struct octnic_ctrl_pkt nctrl;
1123 	int ret = 0;
1124 
1125 	if (!is_valid_ether_addr(addr->sa_data))
1126 		return -EADDRNOTAVAIL;
1127 
1128 	if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
1129 		return 0;
1130 
1131 	if (lio->linfo.macaddr_is_admin_asgnd)
1132 		return -EPERM;
1133 
1134 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1135 
1136 	nctrl.ncmd.u64 = 0;
1137 	nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
1138 	nctrl.ncmd.s.param1 = 0;
1139 	nctrl.ncmd.s.more = 1;
1140 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1141 	nctrl.netpndev = (u64)netdev;
1142 
1143 	nctrl.udd[0] = 0;
1144 	/* The MAC Address is presented in network byte order. */
1145 	ether_addr_copy((u8 *)&nctrl.udd[0] + 2, addr->sa_data);
1146 
1147 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1148 	if (ret < 0) {
1149 		dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
1150 		return -ENOMEM;
1151 	}
1152 
1153 	if (nctrl.sc_status ==
1154 	    FIRMWARE_STATUS_CODE(OCTEON_REQUEST_NO_PERMISSION)) {
1155 		dev_err(&oct->pci_dev->dev, "MAC Address change failed: no permission\n");
1156 		return -EPERM;
1157 	}
1158 
1159 	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
1160 	ether_addr_copy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data);
1161 
1162 	return 0;
1163 }
1164 
1165 static void
1166 liquidio_get_stats64(struct net_device *netdev,
1167 		     struct rtnl_link_stats64 *lstats)
1168 {
1169 	struct lio *lio = GET_LIO(netdev);
1170 	struct octeon_device *oct;
1171 	u64 pkts = 0, drop = 0, bytes = 0;
1172 	struct oct_droq_stats *oq_stats;
1173 	struct oct_iq_stats *iq_stats;
1174 	int i, iq_no, oq_no;
1175 
1176 	oct = lio->oct_dev;
1177 
1178 	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
1179 		return;
1180 
1181 	for (i = 0; i < oct->num_iqs; i++) {
1182 		iq_no = lio->linfo.txpciq[i].s.q_no;
1183 		iq_stats = &oct->instr_queue[iq_no]->stats;
1184 		pkts += iq_stats->tx_done;
1185 		drop += iq_stats->tx_dropped;
1186 		bytes += iq_stats->tx_tot_bytes;
1187 	}
1188 
1189 	lstats->tx_packets = pkts;
1190 	lstats->tx_bytes = bytes;
1191 	lstats->tx_dropped = drop;
1192 
1193 	pkts = 0;
1194 	drop = 0;
1195 	bytes = 0;
1196 
1197 	for (i = 0; i < oct->num_oqs; i++) {
1198 		oq_no = lio->linfo.rxpciq[i].s.q_no;
1199 		oq_stats = &oct->droq[oq_no]->stats;
1200 		pkts += oq_stats->rx_pkts_received;
1201 		drop += (oq_stats->rx_dropped +
1202 			 oq_stats->dropped_nodispatch +
1203 			 oq_stats->dropped_toomany +
1204 			 oq_stats->dropped_nomem);
1205 		bytes += oq_stats->rx_bytes_received;
1206 	}
1207 
1208 	lstats->rx_bytes = bytes;
1209 	lstats->rx_packets = pkts;
1210 	lstats->rx_dropped = drop;
1211 
1212 	lstats->multicast = oct->link_stats.fromwire.fw_total_mcast;
1213 
1214 	/* detailed rx_errors: */
1215 	lstats->rx_length_errors = oct->link_stats.fromwire.l2_err;
1216 	/* recved pkt with crc error */
1217 	lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err;
1218 	/* recv'd frame alignment error */
1219 	lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err;
1220 
1221 	lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors +
1222 			    lstats->rx_frame_errors;
1223 
1224 	/* detailed tx_errors */
1225 	lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko;
1226 	lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link;
1227 
1228 	lstats->tx_errors = lstats->tx_aborted_errors +
1229 		lstats->tx_carrier_errors;
1230 }
1231 
1232 /**
1233  * \brief Handler for SIOCSHWTSTAMP ioctl
1234  * @param netdev network device
1235  * @param ifr interface request
1236  * @param cmd command
1237  */
1238 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
1239 {
1240 	struct lio *lio = GET_LIO(netdev);
1241 	struct hwtstamp_config conf;
1242 
1243 	if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
1244 		return -EFAULT;
1245 
1246 	if (conf.flags)
1247 		return -EINVAL;
1248 
1249 	switch (conf.tx_type) {
1250 	case HWTSTAMP_TX_ON:
1251 	case HWTSTAMP_TX_OFF:
1252 		break;
1253 	default:
1254 		return -ERANGE;
1255 	}
1256 
1257 	switch (conf.rx_filter) {
1258 	case HWTSTAMP_FILTER_NONE:
1259 		break;
1260 	case HWTSTAMP_FILTER_ALL:
1261 	case HWTSTAMP_FILTER_SOME:
1262 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1263 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1264 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1265 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1266 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1267 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1268 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1269 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1270 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1271 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
1272 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
1273 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1274 	case HWTSTAMP_FILTER_NTP_ALL:
1275 		conf.rx_filter = HWTSTAMP_FILTER_ALL;
1276 		break;
1277 	default:
1278 		return -ERANGE;
1279 	}
1280 
1281 	if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
1282 		ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
1283 
1284 	else
1285 		ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
1286 
1287 	return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
1288 }
1289 
1290 /**
1291  * \brief ioctl handler
1292  * @param netdev network device
1293  * @param ifr interface request
1294  * @param cmd command
1295  */
1296 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
1297 {
1298 	switch (cmd) {
1299 	case SIOCSHWTSTAMP:
1300 		return hwtstamp_ioctl(netdev, ifr);
1301 	default:
1302 		return -EOPNOTSUPP;
1303 	}
1304 }
1305 
1306 static void handle_timestamp(struct octeon_device *oct, u32 status, void *buf)
1307 {
1308 	struct sk_buff *skb = (struct sk_buff *)buf;
1309 	struct octnet_buf_free_info *finfo;
1310 	struct oct_timestamp_resp *resp;
1311 	struct octeon_soft_command *sc;
1312 	struct lio *lio;
1313 
1314 	finfo = (struct octnet_buf_free_info *)skb->cb;
1315 	lio = finfo->lio;
1316 	sc = finfo->sc;
1317 	oct = lio->oct_dev;
1318 	resp = (struct oct_timestamp_resp *)sc->virtrptr;
1319 
1320 	if (status != OCTEON_REQUEST_DONE) {
1321 		dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
1322 			CVM_CAST64(status));
1323 		resp->timestamp = 0;
1324 	}
1325 
1326 	octeon_swap_8B_data(&resp->timestamp, 1);
1327 
1328 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) {
1329 		struct skb_shared_hwtstamps ts;
1330 		u64 ns = resp->timestamp;
1331 
1332 		netif_info(lio, tx_done, lio->netdev,
1333 			   "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
1334 			   skb, (unsigned long long)ns);
1335 		ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
1336 		skb_tstamp_tx(skb, &ts);
1337 	}
1338 
1339 	octeon_free_soft_command(oct, sc);
1340 	tx_buffer_free(skb);
1341 }
1342 
1343 /* \brief Send a data packet that will be timestamped
1344  * @param oct octeon device
1345  * @param ndata pointer to network data
1346  * @param finfo pointer to private network data
1347  */
1348 static int send_nic_timestamp_pkt(struct octeon_device *oct,
1349 				  struct octnic_data_pkt *ndata,
1350 				  struct octnet_buf_free_info *finfo,
1351 				  int xmit_more)
1352 {
1353 	struct octeon_soft_command *sc;
1354 	int ring_doorbell;
1355 	struct lio *lio;
1356 	int retval;
1357 	u32 len;
1358 
1359 	lio = finfo->lio;
1360 
1361 	sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
1362 					    sizeof(struct oct_timestamp_resp));
1363 	finfo->sc = sc;
1364 
1365 	if (!sc) {
1366 		dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
1367 		return IQ_SEND_FAILED;
1368 	}
1369 
1370 	if (ndata->reqtype == REQTYPE_NORESP_NET)
1371 		ndata->reqtype = REQTYPE_RESP_NET;
1372 	else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
1373 		ndata->reqtype = REQTYPE_RESP_NET_SG;
1374 
1375 	sc->callback = handle_timestamp;
1376 	sc->callback_arg = finfo->skb;
1377 	sc->iq_no = ndata->q_no;
1378 
1379 	len = (u32)((struct octeon_instr_ih3 *)(&sc->cmd.cmd3.ih3))->dlengsz;
1380 
1381 	ring_doorbell = !xmit_more;
1382 
1383 	retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
1384 				     sc, len, ndata->reqtype);
1385 
1386 	if (retval == IQ_SEND_FAILED) {
1387 		dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
1388 			retval);
1389 		octeon_free_soft_command(oct, sc);
1390 	} else {
1391 		netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
1392 	}
1393 
1394 	return retval;
1395 }
1396 
1397 /** \brief Transmit networks packets to the Octeon interface
1398  * @param skbuff   skbuff struct to be passed to network layer.
1399  * @param netdev   pointer to network device
1400  * @returns whether the packet was transmitted to the device okay or not
1401  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
1402  */
1403 static netdev_tx_t liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
1404 {
1405 	struct octnet_buf_free_info *finfo;
1406 	union octnic_cmd_setup cmdsetup;
1407 	struct octnic_data_pkt ndata;
1408 	struct octeon_instr_irh *irh;
1409 	struct oct_iq_stats *stats;
1410 	struct octeon_device *oct;
1411 	int q_idx = 0, iq_no = 0;
1412 	union tx_info *tx_info;
1413 	int xmit_more = 0;
1414 	struct lio *lio;
1415 	int status = 0;
1416 	u64 dptr = 0;
1417 	u32 tag = 0;
1418 	int j;
1419 
1420 	lio = GET_LIO(netdev);
1421 	oct = lio->oct_dev;
1422 
1423 	q_idx = skb_iq(lio->oct_dev, skb);
1424 	tag = q_idx;
1425 	iq_no = lio->linfo.txpciq[q_idx].s.q_no;
1426 
1427 	stats = &oct->instr_queue[iq_no]->stats;
1428 
1429 	/* Check for all conditions in which the current packet cannot be
1430 	 * transmitted.
1431 	 */
1432 	if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
1433 	    (!lio->linfo.link.s.link_up) || (skb->len <= 0)) {
1434 		netif_info(lio, tx_err, lio->netdev, "Transmit failed link_status : %d\n",
1435 			   lio->linfo.link.s.link_up);
1436 		goto lio_xmit_failed;
1437 	}
1438 
1439 	/* Use space in skb->cb to store info used to unmap and
1440 	 * free the buffers.
1441 	 */
1442 	finfo = (struct octnet_buf_free_info *)skb->cb;
1443 	finfo->lio = lio;
1444 	finfo->skb = skb;
1445 	finfo->sc = NULL;
1446 
1447 	/* Prepare the attributes for the data to be passed to OSI. */
1448 	memset(&ndata, 0, sizeof(struct octnic_data_pkt));
1449 
1450 	ndata.buf = finfo;
1451 
1452 	ndata.q_no = iq_no;
1453 
1454 	if (octnet_iq_is_full(oct, ndata.q_no)) {
1455 		/* defer sending if queue is full */
1456 		netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
1457 			   ndata.q_no);
1458 		stats->tx_iq_busy++;
1459 		return NETDEV_TX_BUSY;
1460 	}
1461 
1462 	ndata.datasize = skb->len;
1463 
1464 	cmdsetup.u64 = 0;
1465 	cmdsetup.s.iq_no = iq_no;
1466 
1467 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
1468 		if (skb->encapsulation) {
1469 			cmdsetup.s.tnl_csum = 1;
1470 			stats->tx_vxlan++;
1471 		} else {
1472 			cmdsetup.s.transport_csum = 1;
1473 		}
1474 	}
1475 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
1476 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1477 		cmdsetup.s.timestamp = 1;
1478 	}
1479 
1480 	if (!skb_shinfo(skb)->nr_frags) {
1481 		cmdsetup.s.u.datasize = skb->len;
1482 		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
1483 		/* Offload checksum calculation for TCP/UDP packets */
1484 		dptr = dma_map_single(&oct->pci_dev->dev,
1485 				      skb->data,
1486 				      skb->len,
1487 				      DMA_TO_DEVICE);
1488 		if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
1489 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
1490 				__func__);
1491 			return NETDEV_TX_BUSY;
1492 		}
1493 
1494 		ndata.cmd.cmd3.dptr = dptr;
1495 		finfo->dptr = dptr;
1496 		ndata.reqtype = REQTYPE_NORESP_NET;
1497 
1498 	} else {
1499 		skb_frag_t *frag;
1500 		struct octnic_gather *g;
1501 		int i, frags;
1502 
1503 		spin_lock(&lio->glist_lock[q_idx]);
1504 		g = (struct octnic_gather *)
1505 			lio_list_delete_head(&lio->glist[q_idx]);
1506 		spin_unlock(&lio->glist_lock[q_idx]);
1507 
1508 		if (!g) {
1509 			netif_info(lio, tx_err, lio->netdev,
1510 				   "Transmit scatter gather: glist null!\n");
1511 			goto lio_xmit_failed;
1512 		}
1513 
1514 		cmdsetup.s.gather = 1;
1515 		cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
1516 		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
1517 
1518 		memset(g->sg, 0, g->sg_size);
1519 
1520 		g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
1521 						 skb->data,
1522 						 (skb->len - skb->data_len),
1523 						 DMA_TO_DEVICE);
1524 		if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
1525 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
1526 				__func__);
1527 			return NETDEV_TX_BUSY;
1528 		}
1529 		add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
1530 
1531 		frags = skb_shinfo(skb)->nr_frags;
1532 		i = 1;
1533 		while (frags--) {
1534 			frag = &skb_shinfo(skb)->frags[i - 1];
1535 
1536 			g->sg[(i >> 2)].ptr[(i & 3)] =
1537 				skb_frag_dma_map(&oct->pci_dev->dev,
1538 						 frag, 0, skb_frag_size(frag),
1539 						 DMA_TO_DEVICE);
1540 			if (dma_mapping_error(&oct->pci_dev->dev,
1541 					      g->sg[i >> 2].ptr[i & 3])) {
1542 				dma_unmap_single(&oct->pci_dev->dev,
1543 						 g->sg[0].ptr[0],
1544 						 skb->len - skb->data_len,
1545 						 DMA_TO_DEVICE);
1546 				for (j = 1; j < i; j++) {
1547 					frag = &skb_shinfo(skb)->frags[j - 1];
1548 					dma_unmap_page(&oct->pci_dev->dev,
1549 						       g->sg[j >> 2].ptr[j & 3],
1550 						       skb_frag_size(frag),
1551 						       DMA_TO_DEVICE);
1552 				}
1553 				dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
1554 					__func__);
1555 				return NETDEV_TX_BUSY;
1556 			}
1557 
1558 			add_sg_size(&g->sg[(i >> 2)], skb_frag_size(frag),
1559 				    (i & 3));
1560 			i++;
1561 		}
1562 
1563 		dptr = g->sg_dma_ptr;
1564 
1565 		ndata.cmd.cmd3.dptr = dptr;
1566 		finfo->dptr = dptr;
1567 		finfo->g = g;
1568 
1569 		ndata.reqtype = REQTYPE_NORESP_NET_SG;
1570 	}
1571 
1572 	irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
1573 	tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];
1574 
1575 	if (skb_shinfo(skb)->gso_size) {
1576 		tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
1577 		tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
1578 	}
1579 
1580 	/* HW insert VLAN tag */
1581 	if (skb_vlan_tag_present(skb)) {
1582 		irh->priority = skb_vlan_tag_get(skb) >> VLAN_PRIO_SHIFT;
1583 		irh->vlan = skb_vlan_tag_get(skb) & VLAN_VID_MASK;
1584 	}
1585 
1586 	xmit_more = netdev_xmit_more();
1587 
1588 	if (unlikely(cmdsetup.s.timestamp))
1589 		status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
1590 	else
1591 		status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
1592 	if (status == IQ_SEND_FAILED)
1593 		goto lio_xmit_failed;
1594 
1595 	netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
1596 
1597 	if (status == IQ_SEND_STOP) {
1598 		dev_err(&oct->pci_dev->dev, "Rcvd IQ_SEND_STOP signal; stopping IQ-%d\n",
1599 			iq_no);
1600 		netif_stop_subqueue(netdev, q_idx);
1601 	}
1602 
1603 	netif_trans_update(netdev);
1604 
1605 	if (tx_info->s.gso_segs)
1606 		stats->tx_done += tx_info->s.gso_segs;
1607 	else
1608 		stats->tx_done++;
1609 	stats->tx_tot_bytes += ndata.datasize;
1610 
1611 	return NETDEV_TX_OK;
1612 
1613 lio_xmit_failed:
1614 	stats->tx_dropped++;
1615 	netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
1616 		   iq_no, stats->tx_dropped);
1617 	if (dptr)
1618 		dma_unmap_single(&oct->pci_dev->dev, dptr,
1619 				 ndata.datasize, DMA_TO_DEVICE);
1620 
1621 	octeon_ring_doorbell_locked(oct, iq_no);
1622 
1623 	tx_buffer_free(skb);
1624 	return NETDEV_TX_OK;
1625 }
1626 
1627 /** \brief Network device Tx timeout
1628  * @param netdev    pointer to network device
1629  */
1630 static void liquidio_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1631 {
1632 	struct lio *lio;
1633 
1634 	lio = GET_LIO(netdev);
1635 
1636 	netif_info(lio, tx_err, lio->netdev,
1637 		   "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
1638 		   netdev->stats.tx_dropped);
1639 	netif_trans_update(netdev);
1640 	wake_txqs(netdev);
1641 }
1642 
1643 static int
1644 liquidio_vlan_rx_add_vid(struct net_device *netdev,
1645 			 __be16 proto __attribute__((unused)), u16 vid)
1646 {
1647 	struct lio *lio = GET_LIO(netdev);
1648 	struct octeon_device *oct = lio->oct_dev;
1649 	struct octnic_ctrl_pkt nctrl;
1650 	int ret = 0;
1651 
1652 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1653 
1654 	nctrl.ncmd.u64 = 0;
1655 	nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
1656 	nctrl.ncmd.s.param1 = vid;
1657 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1658 	nctrl.netpndev = (u64)netdev;
1659 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1660 
1661 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1662 	if (ret) {
1663 		dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
1664 			ret);
1665 		return -EPERM;
1666 	}
1667 
1668 	return 0;
1669 }
1670 
1671 static int
1672 liquidio_vlan_rx_kill_vid(struct net_device *netdev,
1673 			  __be16 proto __attribute__((unused)), u16 vid)
1674 {
1675 	struct lio *lio = GET_LIO(netdev);
1676 	struct octeon_device *oct = lio->oct_dev;
1677 	struct octnic_ctrl_pkt nctrl;
1678 	int ret = 0;
1679 
1680 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1681 
1682 	nctrl.ncmd.u64 = 0;
1683 	nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
1684 	nctrl.ncmd.s.param1 = vid;
1685 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1686 	nctrl.netpndev = (u64)netdev;
1687 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1688 
1689 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1690 	if (ret) {
1691 		dev_err(&oct->pci_dev->dev, "Del VLAN filter failed in core (ret: 0x%x)\n",
1692 			ret);
1693 		if (ret > 0)
1694 			ret = -EIO;
1695 	}
1696 	return ret;
1697 }
1698 
1699 /** Sending command to enable/disable RX checksum offload
1700  * @param netdev                pointer to network device
1701  * @param command               OCTNET_CMD_TNL_RX_CSUM_CTL
1702  * @param rx_cmd_bit            OCTNET_CMD_RXCSUM_ENABLE/
1703  *                              OCTNET_CMD_RXCSUM_DISABLE
1704  * @returns                     SUCCESS or FAILURE
1705  */
1706 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
1707 				       u8 rx_cmd)
1708 {
1709 	struct lio *lio = GET_LIO(netdev);
1710 	struct octeon_device *oct = lio->oct_dev;
1711 	struct octnic_ctrl_pkt nctrl;
1712 	int ret = 0;
1713 
1714 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1715 
1716 	nctrl.ncmd.u64 = 0;
1717 	nctrl.ncmd.s.cmd = command;
1718 	nctrl.ncmd.s.param1 = rx_cmd;
1719 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1720 	nctrl.netpndev = (u64)netdev;
1721 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1722 
1723 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1724 	if (ret) {
1725 		dev_err(&oct->pci_dev->dev, "DEVFLAGS RXCSUM change failed in core (ret:0x%x)\n",
1726 			ret);
1727 		if (ret > 0)
1728 			ret = -EIO;
1729 	}
1730 	return ret;
1731 }
1732 
1733 /** Sending command to add/delete VxLAN UDP port to firmware
1734  * @param netdev                pointer to network device
1735  * @param command               OCTNET_CMD_VXLAN_PORT_CONFIG
1736  * @param vxlan_port            VxLAN port to be added or deleted
1737  * @param vxlan_cmd_bit         OCTNET_CMD_VXLAN_PORT_ADD,
1738  *                              OCTNET_CMD_VXLAN_PORT_DEL
1739  * @returns                     SUCCESS or FAILURE
1740  */
1741 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
1742 				       u16 vxlan_port, u8 vxlan_cmd_bit)
1743 {
1744 	struct lio *lio = GET_LIO(netdev);
1745 	struct octeon_device *oct = lio->oct_dev;
1746 	struct octnic_ctrl_pkt nctrl;
1747 	int ret = 0;
1748 
1749 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1750 
1751 	nctrl.ncmd.u64 = 0;
1752 	nctrl.ncmd.s.cmd = command;
1753 	nctrl.ncmd.s.more = vxlan_cmd_bit;
1754 	nctrl.ncmd.s.param1 = vxlan_port;
1755 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1756 	nctrl.netpndev = (u64)netdev;
1757 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1758 
1759 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1760 	if (ret) {
1761 		dev_err(&oct->pci_dev->dev,
1762 			"DEVFLAGS VxLAN port add/delete failed in core (ret : 0x%x)\n",
1763 			ret);
1764 		if (ret > 0)
1765 			ret = -EIO;
1766 	}
1767 	return ret;
1768 }
1769 
1770 /** \brief Net device fix features
1771  * @param netdev  pointer to network device
1772  * @param request features requested
1773  * @returns updated features list
1774  */
1775 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
1776 					       netdev_features_t request)
1777 {
1778 	struct lio *lio = netdev_priv(netdev);
1779 
1780 	if ((request & NETIF_F_RXCSUM) &&
1781 	    !(lio->dev_capability & NETIF_F_RXCSUM))
1782 		request &= ~NETIF_F_RXCSUM;
1783 
1784 	if ((request & NETIF_F_HW_CSUM) &&
1785 	    !(lio->dev_capability & NETIF_F_HW_CSUM))
1786 		request &= ~NETIF_F_HW_CSUM;
1787 
1788 	if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
1789 		request &= ~NETIF_F_TSO;
1790 
1791 	if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
1792 		request &= ~NETIF_F_TSO6;
1793 
1794 	if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
1795 		request &= ~NETIF_F_LRO;
1796 
1797 	/* Disable LRO if RXCSUM is off */
1798 	if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
1799 	    (lio->dev_capability & NETIF_F_LRO))
1800 		request &= ~NETIF_F_LRO;
1801 
1802 	return request;
1803 }
1804 
1805 /** \brief Net device set features
1806  * @param netdev  pointer to network device
1807  * @param features features to enable/disable
1808  */
1809 static int liquidio_set_features(struct net_device *netdev,
1810 				 netdev_features_t features)
1811 {
1812 	struct lio *lio = netdev_priv(netdev);
1813 
1814 	if (!((netdev->features ^ features) & NETIF_F_LRO))
1815 		return 0;
1816 
1817 	if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO))
1818 		liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
1819 				     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
1820 	else if (!(features & NETIF_F_LRO) &&
1821 		 (lio->dev_capability & NETIF_F_LRO))
1822 		liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
1823 				     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
1824 	if (!(netdev->features & NETIF_F_RXCSUM) &&
1825 	    (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
1826 	    (features & NETIF_F_RXCSUM))
1827 		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
1828 					    OCTNET_CMD_RXCSUM_ENABLE);
1829 	else if ((netdev->features & NETIF_F_RXCSUM) &&
1830 		 (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
1831 		 !(features & NETIF_F_RXCSUM))
1832 		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
1833 					    OCTNET_CMD_RXCSUM_DISABLE);
1834 
1835 	return 0;
1836 }
1837 
1838 static void liquidio_add_vxlan_port(struct net_device *netdev,
1839 				    struct udp_tunnel_info *ti)
1840 {
1841 	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
1842 		return;
1843 
1844 	liquidio_vxlan_port_command(netdev,
1845 				    OCTNET_CMD_VXLAN_PORT_CONFIG,
1846 				    htons(ti->port),
1847 				    OCTNET_CMD_VXLAN_PORT_ADD);
1848 }
1849 
1850 static void liquidio_del_vxlan_port(struct net_device *netdev,
1851 				    struct udp_tunnel_info *ti)
1852 {
1853 	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
1854 		return;
1855 
1856 	liquidio_vxlan_port_command(netdev,
1857 				    OCTNET_CMD_VXLAN_PORT_CONFIG,
1858 				    htons(ti->port),
1859 				    OCTNET_CMD_VXLAN_PORT_DEL);
1860 }
1861 
1862 static const struct net_device_ops lionetdevops = {
1863 	.ndo_open		= liquidio_open,
1864 	.ndo_stop		= liquidio_stop,
1865 	.ndo_start_xmit		= liquidio_xmit,
1866 	.ndo_get_stats64	= liquidio_get_stats64,
1867 	.ndo_set_mac_address	= liquidio_set_mac,
1868 	.ndo_set_rx_mode	= liquidio_set_mcast_list,
1869 	.ndo_tx_timeout		= liquidio_tx_timeout,
1870 	.ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
1871 	.ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
1872 	.ndo_change_mtu		= liquidio_change_mtu,
1873 	.ndo_do_ioctl		= liquidio_ioctl,
1874 	.ndo_fix_features	= liquidio_fix_features,
1875 	.ndo_set_features	= liquidio_set_features,
1876 	.ndo_udp_tunnel_add     = liquidio_add_vxlan_port,
1877 	.ndo_udp_tunnel_del     = liquidio_del_vxlan_port,
1878 };
1879 
1880 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
1881 {
1882 	struct octeon_device *oct = (struct octeon_device *)buf;
1883 	struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
1884 	union oct_link_status *ls;
1885 	int gmxport = 0;
1886 	int i;
1887 
1888 	if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) {
1889 		dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
1890 			recv_pkt->buffer_size[0],
1891 			recv_pkt->rh.r_nic_info.gmxport);
1892 		goto nic_info_err;
1893 	}
1894 
1895 	gmxport = recv_pkt->rh.r_nic_info.gmxport;
1896 	ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) +
1897 		OCT_DROQ_INFO_SIZE);
1898 
1899 	octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
1900 
1901 	for (i = 0; i < oct->ifcount; i++) {
1902 		if (oct->props[i].gmxport == gmxport) {
1903 			update_link_status(oct->props[i].netdev, ls);
1904 			break;
1905 		}
1906 	}
1907 
1908 nic_info_err:
1909 	for (i = 0; i < recv_pkt->buffer_count; i++)
1910 		recv_buffer_free(recv_pkt->buffer_ptr[i]);
1911 	octeon_free_recv_info(recv_info);
1912 	return 0;
1913 }
1914 
1915 /**
1916  * \brief Setup network interfaces
1917  * @param octeon_dev  octeon device
1918  *
1919  * Called during init time for each device. It assumes the NIC
1920  * is already up and running.  The link information for each
1921  * interface is passed in link_info.
1922  */
1923 static int setup_nic_devices(struct octeon_device *octeon_dev)
1924 {
1925 	int retval, num_iqueues, num_oqueues;
1926 	u32 resp_size, data_size;
1927 	struct liquidio_if_cfg_resp *resp;
1928 	struct octeon_soft_command *sc;
1929 	union oct_nic_if_cfg if_cfg;
1930 	struct octdev_props *props;
1931 	struct net_device *netdev;
1932 	struct lio_version *vdata;
1933 	struct lio *lio = NULL;
1934 	u8 mac[ETH_ALEN], i, j;
1935 	u32 ifidx_or_pfnum;
1936 
1937 	ifidx_or_pfnum = octeon_dev->pf_num;
1938 
1939 	/* This is to handle link status changes */
1940 	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC, OPCODE_NIC_INFO,
1941 				    lio_nic_info, octeon_dev);
1942 
1943 	/* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
1944 	 * They are handled directly.
1945 	 */
1946 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
1947 					free_netbuf);
1948 
1949 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
1950 					free_netsgbuf);
1951 
1952 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
1953 					free_netsgbuf_with_resp);
1954 
1955 	for (i = 0; i < octeon_dev->ifcount; i++) {
1956 		resp_size = sizeof(struct liquidio_if_cfg_resp);
1957 		data_size = sizeof(struct lio_version);
1958 		sc = (struct octeon_soft_command *)
1959 			octeon_alloc_soft_command(octeon_dev, data_size,
1960 						  resp_size, 0);
1961 		resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
1962 		vdata = (struct lio_version *)sc->virtdptr;
1963 
1964 		*((u64 *)vdata) = 0;
1965 		vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
1966 		vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
1967 		vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
1968 
1969 		if_cfg.u64 = 0;
1970 
1971 		if_cfg.s.num_iqueues = octeon_dev->sriov_info.rings_per_vf;
1972 		if_cfg.s.num_oqueues = octeon_dev->sriov_info.rings_per_vf;
1973 		if_cfg.s.base_queue = 0;
1974 
1975 		sc->iq_no = 0;
1976 
1977 		octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
1978 					    OPCODE_NIC_IF_CFG, 0, if_cfg.u64,
1979 					    0);
1980 
1981 		init_completion(&sc->complete);
1982 		sc->sc_status = OCTEON_REQUEST_PENDING;
1983 
1984 		retval = octeon_send_soft_command(octeon_dev, sc);
1985 		if (retval == IQ_SEND_FAILED) {
1986 			dev_err(&octeon_dev->pci_dev->dev,
1987 				"iq/oq config failed status: %x\n", retval);
1988 			/* Soft instr is freed by driver in case of failure. */
1989 			octeon_free_soft_command(octeon_dev, sc);
1990 			return(-EIO);
1991 		}
1992 
1993 		/* Sleep on a wait queue till the cond flag indicates that the
1994 		 * response arrived or timed-out.
1995 		 */
1996 		retval = wait_for_sc_completion_timeout(octeon_dev, sc, 0);
1997 		if (retval)
1998 			return retval;
1999 
2000 		retval = resp->status;
2001 		if (retval) {
2002 			dev_err(&octeon_dev->pci_dev->dev,
2003 				"iq/oq config failed, retval = %d\n", retval);
2004 			WRITE_ONCE(sc->caller_is_done, true);
2005 			return -EIO;
2006 		}
2007 
2008 		snprintf(octeon_dev->fw_info.liquidio_firmware_version,
2009 			 32, "%s",
2010 			 resp->cfg_info.liquidio_firmware_version);
2011 
2012 		octeon_swap_8B_data((u64 *)(&resp->cfg_info),
2013 				    (sizeof(struct liquidio_if_cfg_info)) >> 3);
2014 
2015 		num_iqueues = hweight64(resp->cfg_info.iqmask);
2016 		num_oqueues = hweight64(resp->cfg_info.oqmask);
2017 
2018 		if (!(num_iqueues) || !(num_oqueues)) {
2019 			dev_err(&octeon_dev->pci_dev->dev,
2020 				"Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
2021 				resp->cfg_info.iqmask, resp->cfg_info.oqmask);
2022 			WRITE_ONCE(sc->caller_is_done, true);
2023 			goto setup_nic_dev_done;
2024 		}
2025 		dev_dbg(&octeon_dev->pci_dev->dev,
2026 			"interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
2027 			i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
2028 			num_iqueues, num_oqueues);
2029 
2030 		netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues);
2031 
2032 		if (!netdev) {
2033 			dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
2034 			WRITE_ONCE(sc->caller_is_done, true);
2035 			goto setup_nic_dev_done;
2036 		}
2037 
2038 		SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
2039 
2040 		/* Associate the routines that will handle different
2041 		 * netdev tasks.
2042 		 */
2043 		netdev->netdev_ops = &lionetdevops;
2044 
2045 		lio = GET_LIO(netdev);
2046 
2047 		memset(lio, 0, sizeof(struct lio));
2048 
2049 		lio->ifidx = ifidx_or_pfnum;
2050 
2051 		props = &octeon_dev->props[i];
2052 		props->gmxport = resp->cfg_info.linfo.gmxport;
2053 		props->netdev = netdev;
2054 
2055 		lio->linfo.num_rxpciq = num_oqueues;
2056 		lio->linfo.num_txpciq = num_iqueues;
2057 
2058 		for (j = 0; j < num_oqueues; j++) {
2059 			lio->linfo.rxpciq[j].u64 =
2060 			    resp->cfg_info.linfo.rxpciq[j].u64;
2061 		}
2062 		for (j = 0; j < num_iqueues; j++) {
2063 			lio->linfo.txpciq[j].u64 =
2064 			    resp->cfg_info.linfo.txpciq[j].u64;
2065 		}
2066 
2067 		lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
2068 		lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
2069 		lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
2070 		lio->linfo.macaddr_is_admin_asgnd =
2071 			resp->cfg_info.linfo.macaddr_is_admin_asgnd;
2072 		lio->linfo.macaddr_spoofchk =
2073 			resp->cfg_info.linfo.macaddr_spoofchk;
2074 
2075 		lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2076 
2077 		lio->dev_capability = NETIF_F_HIGHDMA
2078 				      | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM
2079 				      | NETIF_F_SG | NETIF_F_RXCSUM
2080 				      | NETIF_F_TSO | NETIF_F_TSO6
2081 				      | NETIF_F_GRO
2082 				      | NETIF_F_LRO;
2083 		netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
2084 
2085 		/* Copy of transmit encapsulation capabilities:
2086 		 * TSO, TSO6, Checksums for this device
2087 		 */
2088 		lio->enc_dev_capability = NETIF_F_IP_CSUM
2089 					  | NETIF_F_IPV6_CSUM
2090 					  | NETIF_F_GSO_UDP_TUNNEL
2091 					  | NETIF_F_HW_CSUM | NETIF_F_SG
2092 					  | NETIF_F_RXCSUM
2093 					  | NETIF_F_TSO | NETIF_F_TSO6
2094 					  | NETIF_F_LRO;
2095 
2096 		netdev->hw_enc_features =
2097 		    (lio->enc_dev_capability & ~NETIF_F_LRO);
2098 		netdev->vlan_features = lio->dev_capability;
2099 		/* Add any unchangeable hw features */
2100 		lio->dev_capability |= NETIF_F_HW_VLAN_CTAG_FILTER |
2101 				       NETIF_F_HW_VLAN_CTAG_RX |
2102 				       NETIF_F_HW_VLAN_CTAG_TX;
2103 
2104 		netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
2105 
2106 		netdev->hw_features = lio->dev_capability;
2107 		netdev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
2108 
2109 		/* MTU range: 68 - 16000 */
2110 		netdev->min_mtu = LIO_MIN_MTU_SIZE;
2111 		netdev->max_mtu = LIO_MAX_MTU_SIZE;
2112 
2113 		WRITE_ONCE(sc->caller_is_done, true);
2114 
2115 		/* Point to the  properties for octeon device to which this
2116 		 * interface belongs.
2117 		 */
2118 		lio->oct_dev = octeon_dev;
2119 		lio->octprops = props;
2120 		lio->netdev = netdev;
2121 
2122 		dev_dbg(&octeon_dev->pci_dev->dev,
2123 			"if%d gmx: %d hw_addr: 0x%llx\n", i,
2124 			lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
2125 
2126 		/* 64-bit swap required on LE machines */
2127 		octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
2128 		for (j = 0; j < ETH_ALEN; j++)
2129 			mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
2130 
2131 		/* Copy MAC Address to OS network device structure */
2132 		ether_addr_copy(netdev->dev_addr, mac);
2133 
2134 		if (liquidio_setup_io_queues(octeon_dev, i,
2135 					     lio->linfo.num_txpciq,
2136 					     lio->linfo.num_rxpciq)) {
2137 			dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
2138 			goto setup_nic_dev_free;
2139 		}
2140 
2141 		ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
2142 
2143 		/* For VFs, enable Octeon device interrupts here,
2144 		 * as this is contingent upon IO queue setup
2145 		 */
2146 		octeon_dev->fn_list.enable_interrupt(octeon_dev,
2147 						     OCTEON_ALL_INTR);
2148 
2149 		/* By default all interfaces on a single Octeon uses the same
2150 		 * tx and rx queues
2151 		 */
2152 		lio->txq = lio->linfo.txpciq[0].s.q_no;
2153 		lio->rxq = lio->linfo.rxpciq[0].s.q_no;
2154 
2155 		lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
2156 		lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
2157 
2158 		if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {
2159 			dev_err(&octeon_dev->pci_dev->dev,
2160 				"Gather list allocation failed\n");
2161 			goto setup_nic_dev_free;
2162 		}
2163 
2164 		/* Register ethtool support */
2165 		liquidio_set_ethtool_ops(netdev);
2166 		if (lio->oct_dev->chip_id == OCTEON_CN23XX_VF_VID)
2167 			octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;
2168 		else
2169 			octeon_dev->priv_flags = 0x0;
2170 
2171 		if (netdev->features & NETIF_F_LRO)
2172 			liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
2173 					     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2174 
2175 		if (setup_link_status_change_wq(netdev))
2176 			goto setup_nic_dev_free;
2177 
2178 		if (setup_rx_oom_poll_fn(netdev))
2179 			goto setup_nic_dev_free;
2180 
2181 		/* Register the network device with the OS */
2182 		if (register_netdev(netdev)) {
2183 			dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
2184 			goto setup_nic_dev_free;
2185 		}
2186 
2187 		dev_dbg(&octeon_dev->pci_dev->dev,
2188 			"Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
2189 			i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
2190 		netif_carrier_off(netdev);
2191 		lio->link_changes++;
2192 
2193 		ifstate_set(lio, LIO_IFSTATE_REGISTERED);
2194 
2195 		/* Sending command to firmware to enable Rx checksum offload
2196 		 * by default at the time of setup of Liquidio driver for
2197 		 * this device
2198 		 */
2199 		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
2200 					    OCTNET_CMD_RXCSUM_ENABLE);
2201 		liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
2202 				     OCTNET_CMD_TXCSUM_ENABLE);
2203 
2204 		dev_dbg(&octeon_dev->pci_dev->dev,
2205 			"NIC ifidx:%d Setup successful\n", i);
2206 
2207 		octeon_dev->no_speed_setting = 1;
2208 	}
2209 
2210 	return 0;
2211 
2212 setup_nic_dev_free:
2213 
2214 	while (i--) {
2215 		dev_err(&octeon_dev->pci_dev->dev,
2216 			"NIC ifidx:%d Setup failed\n", i);
2217 		liquidio_destroy_nic_device(octeon_dev, i);
2218 	}
2219 
2220 setup_nic_dev_done:
2221 
2222 	return -ENODEV;
2223 }
2224 
2225 /**
2226  * \brief initialize the NIC
2227  * @param oct octeon device
2228  *
2229  * This initialization routine is called once the Octeon device application is
2230  * up and running
2231  */
2232 static int liquidio_init_nic_module(struct octeon_device *oct)
2233 {
2234 	int num_nic_ports = 1;
2235 	int i, retval = 0;
2236 
2237 	dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
2238 
2239 	/* only default iq and oq were initialized
2240 	 * initialize the rest as well run port_config command for each port
2241 	 */
2242 	oct->ifcount = num_nic_ports;
2243 	memset(oct->props, 0,
2244 	       sizeof(struct octdev_props) * num_nic_ports);
2245 
2246 	for (i = 0; i < MAX_OCTEON_LINKS; i++)
2247 		oct->props[i].gmxport = -1;
2248 
2249 	retval = setup_nic_devices(oct);
2250 	if (retval) {
2251 		dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
2252 		goto octnet_init_failure;
2253 	}
2254 
2255 	dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
2256 
2257 	return retval;
2258 
2259 octnet_init_failure:
2260 
2261 	oct->ifcount = 0;
2262 
2263 	return retval;
2264 }
2265 
2266 /**
2267  * \brief Device initialization for each Octeon device that is probed
2268  * @param octeon_dev  octeon device
2269  */
2270 static int octeon_device_init(struct octeon_device *oct)
2271 {
2272 	u32 rev_id;
2273 	int j;
2274 
2275 	atomic_set(&oct->status, OCT_DEV_BEGIN_STATE);
2276 
2277 	/* Enable access to the octeon device and make its DMA capability
2278 	 * known to the OS.
2279 	 */
2280 	if (octeon_pci_os_setup(oct))
2281 		return 1;
2282 	atomic_set(&oct->status, OCT_DEV_PCI_ENABLE_DONE);
2283 
2284 	oct->chip_id = OCTEON_CN23XX_VF_VID;
2285 	pci_read_config_dword(oct->pci_dev, 8, &rev_id);
2286 	oct->rev_id = rev_id & 0xff;
2287 
2288 	if (cn23xx_setup_octeon_vf_device(oct))
2289 		return 1;
2290 
2291 	atomic_set(&oct->status, OCT_DEV_PCI_MAP_DONE);
2292 
2293 	oct->app_mode = CVM_DRV_NIC_APP;
2294 
2295 	/* Initialize the dispatch mechanism used to push packets arriving on
2296 	 * Octeon Output queues.
2297 	 */
2298 	if (octeon_init_dispatch_list(oct))
2299 		return 1;
2300 
2301 	atomic_set(&oct->status, OCT_DEV_DISPATCH_INIT_DONE);
2302 
2303 	if (octeon_set_io_queues_off(oct)) {
2304 		dev_err(&oct->pci_dev->dev, "setting io queues off failed\n");
2305 		return 1;
2306 	}
2307 
2308 	if (oct->fn_list.setup_device_regs(oct)) {
2309 		dev_err(&oct->pci_dev->dev, "device registers configuration failed\n");
2310 		return 1;
2311 	}
2312 
2313 	/* Initialize soft command buffer pool */
2314 	if (octeon_setup_sc_buffer_pool(oct)) {
2315 		dev_err(&oct->pci_dev->dev, "sc buffer pool allocation failed\n");
2316 		return 1;
2317 	}
2318 	atomic_set(&oct->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
2319 
2320 	/* Setup the data structures that manage this Octeon's Input queues. */
2321 	if (octeon_setup_instr_queues(oct)) {
2322 		dev_err(&oct->pci_dev->dev, "instruction queue initialization failed\n");
2323 		return 1;
2324 	}
2325 	atomic_set(&oct->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
2326 
2327 	/* Initialize lists to manage the requests of different types that
2328 	 * arrive from user & kernel applications for this octeon device.
2329 	 */
2330 	if (octeon_setup_response_list(oct)) {
2331 		dev_err(&oct->pci_dev->dev, "Response list allocation failed\n");
2332 		return 1;
2333 	}
2334 	atomic_set(&oct->status, OCT_DEV_RESP_LIST_INIT_DONE);
2335 
2336 	if (octeon_setup_output_queues(oct)) {
2337 		dev_err(&oct->pci_dev->dev, "Output queue initialization failed\n");
2338 		return 1;
2339 	}
2340 	atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);
2341 
2342 	if (oct->fn_list.setup_mbox(oct)) {
2343 		dev_err(&oct->pci_dev->dev, "Mailbox setup failed\n");
2344 		return 1;
2345 	}
2346 	atomic_set(&oct->status, OCT_DEV_MBOX_SETUP_DONE);
2347 
2348 	if (octeon_allocate_ioq_vector(oct, oct->sriov_info.rings_per_vf)) {
2349 		dev_err(&oct->pci_dev->dev, "ioq vector allocation failed\n");
2350 		return 1;
2351 	}
2352 	atomic_set(&oct->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE);
2353 
2354 	dev_info(&oct->pci_dev->dev, "OCTEON_CN23XX VF: %d ioqs\n",
2355 		 oct->sriov_info.rings_per_vf);
2356 
2357 	/* Setup the interrupt handler and record the INT SUM register address*/
2358 	if (octeon_setup_interrupt(oct, oct->sriov_info.rings_per_vf))
2359 		return 1;
2360 
2361 	atomic_set(&oct->status, OCT_DEV_INTR_SET_DONE);
2362 
2363 	/* ***************************************************************
2364 	 * The interrupts need to be enabled for the PF<-->VF handshake.
2365 	 * They are [re]-enabled after the PF<-->VF handshake so that the
2366 	 * correct OQ tick value is used (i.e. the value retrieved from
2367 	 * the PF as part of the handshake).
2368 	 */
2369 
2370 	/* Enable Octeon device interrupts */
2371 	oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);
2372 
2373 	if (cn23xx_octeon_pfvf_handshake(oct))
2374 		return 1;
2375 
2376 	/* Here we [re]-enable the interrupts so that the correct OQ tick value
2377 	 * is used (i.e. the value that was retrieved during the handshake)
2378 	 */
2379 
2380 	/* Enable Octeon device interrupts */
2381 	oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);
2382 	/* *************************************************************** */
2383 
2384 	/* Enable the input and output queues for this Octeon device */
2385 	if (oct->fn_list.enable_io_queues(oct)) {
2386 		dev_err(&oct->pci_dev->dev, "enabling io queues failed\n");
2387 		return 1;
2388 	}
2389 
2390 	atomic_set(&oct->status, OCT_DEV_IO_QUEUES_DONE);
2391 
2392 	atomic_set(&oct->status, OCT_DEV_HOST_OK);
2393 
2394 	/* Send Credit for Octeon Output queues. Credits are always sent after
2395 	 * the output queue is enabled.
2396 	 */
2397 	for (j = 0; j < oct->num_oqs; j++)
2398 		writel(oct->droq[j]->max_count, oct->droq[j]->pkts_credit_reg);
2399 
2400 	/* Packets can start arriving on the output queues from this point. */
2401 
2402 	atomic_set(&oct->status, OCT_DEV_CORE_OK);
2403 
2404 	atomic_set(&oct->status, OCT_DEV_RUNNING);
2405 
2406 	if (liquidio_init_nic_module(oct))
2407 		return 1;
2408 
2409 	return 0;
2410 }
2411 
2412 static int __init liquidio_vf_init(void)
2413 {
2414 	octeon_init_device_list(0);
2415 	return pci_register_driver(&liquidio_vf_pci_driver);
2416 }
2417 
2418 static void __exit liquidio_vf_exit(void)
2419 {
2420 	pci_unregister_driver(&liquidio_vf_pci_driver);
2421 
2422 	pr_info("LiquidIO_VF network module is now unloaded\n");
2423 }
2424 
2425 module_init(liquidio_vf_init);
2426 module_exit(liquidio_vf_exit);
2427