70 static int octeon_device_init(struct octeon_device *oct);
73 static int lio_wait_for_oq_pkts(struct octeon_device *oct)  in lio_wait_for_oq_pkts()  argument
75 	struct octeon_device_priv *oct_priv = oct->priv;  in lio_wait_for_oq_pkts()
83 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {  in lio_wait_for_oq_pkts()
84 			if (!(oct->io_qmask.oq & BIT_ULL(i)))  in lio_wait_for_oq_pkts()
86 			pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);  in lio_wait_for_oq_pkts()
104 static void pcierror_quiesce_device(struct octeon_device *oct)  in pcierror_quiesce_device()  argument
116 	if (wait_for_pending_requests(oct))  in pcierror_quiesce_device()
117 		dev_err(&oct->pci_dev->dev, "There were pending requests\n");  in pcierror_quiesce_device()
120 	for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {  in pcierror_quiesce_device()
123 		if (!(oct->io_qmask.iq & BIT_ULL(i)))  in pcierror_quiesce_device()
125 		iq = oct->instr_queue[i];  in pcierror_quiesce_device()
133 			lio_process_iq_request_list(oct, iq, 0);  in pcierror_quiesce_device()
139 	lio_process_ordered_list(oct, 1);  in pcierror_quiesce_device()
168 static void stop_pci_io(struct octeon_device *oct)  in stop_pci_io()  argument
174 	atomic_set(&oct->status, OCT_DEV_IN_RESET);  in stop_pci_io()
176 	for (i = 0; i < oct->ifcount; i++)  in stop_pci_io()
177 		netif_device_detach(oct->props[i].netdev);  in stop_pci_io()
180 	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);  in stop_pci_io()
182 	pcierror_quiesce_device(oct);  in stop_pci_io()
183 	if (oct->msix_on) {  in stop_pci_io()
184 		msix_entries = (struct msix_entry *)oct->msix_entries;  in stop_pci_io()
185 		for (i = 0; i < oct->num_msix_irqs; i++) {  in stop_pci_io()
190 				 &oct->ioq_vector[i]);  in stop_pci_io()
192 		pci_disable_msix(oct->pci_dev);  in stop_pci_io()
193 		kfree(oct->msix_entries);  in stop_pci_io()
194 		oct->msix_entries = NULL;  in stop_pci_io()
195 		octeon_free_ioq_vector(oct);  in stop_pci_io()
197 	dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",  in stop_pci_io()
198 		lio_get_state_string(&oct->status));  in stop_pci_io()
201 	cleanup_aer_uncorrect_error_status(oct->pci_dev);  in stop_pci_io()
203 	pci_disable_device(oct->pci_dev);  in stop_pci_io()
217 	struct octeon_device *oct = pci_get_drvdata(pdev);  in liquidio_pcie_error_detected()  local
221 		dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");  in liquidio_pcie_error_detected()
222 		cleanup_aer_uncorrect_error_status(oct->pci_dev);  in liquidio_pcie_error_detected()
227 	dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");  in liquidio_pcie_error_detected()
228 	stop_pci_io(oct);  in liquidio_pcie_error_detected()
304 	struct octeon_device *oct = lio->oct_dev;  in setup_link_status_change_wq()  local
309 		dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");  in setup_link_status_change_wq()
342 	struct octeon_device *oct = lio->oct_dev;  in update_link_status()  local
359 			dev_info(&oct->pci_dev->dev,  in update_link_status()
366 			dev_warn(&oct->pci_dev->dev,  in update_link_status()
421 static void octeon_pci_flr(struct octeon_device *oct)  in octeon_pci_flr()  argument
423 	pci_save_state(oct->pci_dev);  in octeon_pci_flr()
425 	pci_cfg_access_lock(oct->pci_dev);  in octeon_pci_flr()
428 	pci_write_config_word(oct->pci_dev, PCI_COMMAND,  in octeon_pci_flr()
431 	pcie_flr(oct->pci_dev);  in octeon_pci_flr()
433 	pci_cfg_access_unlock(oct->pci_dev);  in octeon_pci_flr()
435 	pci_restore_state(oct->pci_dev);  in octeon_pci_flr()
442 static void octeon_destroy_resources(struct octeon_device *oct)  in octeon_destroy_resources()  argument
444 	struct octeon_device_priv *oct_priv = oct->priv;  in octeon_destroy_resources()
448 	switch (atomic_read(&oct->status)) {  in octeon_destroy_resources()
452 		atomic_set(&oct->status, OCT_DEV_IN_RESET);  in octeon_destroy_resources()
454 		oct->app_mode = CVM_DRV_INVALID_APP;  in octeon_destroy_resources()
455 		dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",  in octeon_destroy_resources()
456 			lio_get_state_string(&oct->status));  in octeon_destroy_resources()
463 		if (lio_wait_for_instr_fetch(oct))  in octeon_destroy_resources()
464 			dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");  in octeon_destroy_resources()
466 		if (wait_for_pending_requests(oct))  in octeon_destroy_resources()
467 			dev_err(&oct->pci_dev->dev, "There were pending requests\n");  in octeon_destroy_resources()
473 		oct->fn_list.disable_io_queues(oct);  in octeon_destroy_resources()
475 		if (lio_wait_for_oq_pkts(oct))  in octeon_destroy_resources()
476 			dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");  in octeon_destroy_resources()
481 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {  in octeon_destroy_resources()
484 			if (!(oct->io_qmask.iq & BIT_ULL(i)))  in octeon_destroy_resources()
486 			iq = oct->instr_queue[i];  in octeon_destroy_resources()
494 				lio_process_iq_request_list(oct, iq, 0);  in octeon_destroy_resources()
499 		lio_process_ordered_list(oct, 1);  in octeon_destroy_resources()
500 		octeon_free_sc_done_list(oct);  in octeon_destroy_resources()
501 		octeon_free_sc_zombie_list(oct);  in octeon_destroy_resources()
506 		oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);  in octeon_destroy_resources()
508 		if (oct->msix_on) {  in octeon_destroy_resources()
509 			msix_entries = (struct msix_entry *)oct->msix_entries;  in octeon_destroy_resources()
510 			for (i = 0; i < oct->num_msix_irqs; i++) {  in octeon_destroy_resources()
511 				if (oct->ioq_vector[i].vector) {  in octeon_destroy_resources()
516 						 &oct->ioq_vector[i]);  in octeon_destroy_resources()
517 					oct->ioq_vector[i].vector = 0;  in octeon_destroy_resources()
520 			pci_disable_msix(oct->pci_dev);  in octeon_destroy_resources()
521 			kfree(oct->msix_entries);  in octeon_destroy_resources()
522 			oct->msix_entries = NULL;  in octeon_destroy_resources()
523 			kfree(oct->irq_name_storage);  in octeon_destroy_resources()
524 			oct->irq_name_storage = NULL;  in octeon_destroy_resources()
527 		if (!pcie_reset_flr(oct->pci_dev, PCI_RESET_PROBE))  in octeon_destroy_resources()
528 			octeon_pci_flr(oct);  in octeon_destroy_resources()
530 			cn23xx_vf_ask_pf_to_do_flr(oct);  in octeon_destroy_resources()
534 		octeon_free_ioq_vector(oct);  in octeon_destroy_resources()
538 		oct->fn_list.free_mbox(oct);  in octeon_destroy_resources()
544 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {  in octeon_destroy_resources()
545 			if (!(oct->io_qmask.oq & BIT_ULL(i)))  in octeon_destroy_resources()
547 			octeon_delete_droq(oct, i);  in octeon_destroy_resources()
552 		octeon_delete_response_list(oct);  in octeon_destroy_resources()
556 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {  in octeon_destroy_resources()
557 			if (!(oct->io_qmask.iq & BIT_ULL(i)))  in octeon_destroy_resources()
559 			octeon_delete_instr_queue(oct, i);  in octeon_destroy_resources()
564 		octeon_free_sc_buffer_pool(oct);  in octeon_destroy_resources()
568 		octeon_delete_dispatch_list(oct);  in octeon_destroy_resources()
569 		cancel_delayed_work_sync(&oct->nic_poll_work.work);  in octeon_destroy_resources()
573 		octeon_unmap_pci_barx(oct, 0);  in octeon_destroy_resources()
574 		octeon_unmap_pci_barx(oct, 1);  in octeon_destroy_resources()
579 		pci_disable_device(oct->pci_dev);  in octeon_destroy_resources()
597 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;  in send_rx_ctrl_cmd()  local
602 	if (oct->props[lio->ifidx].rx_on == start_stop)  in send_rx_ctrl_cmd()
606 		octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,  in send_rx_ctrl_cmd()
624 	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,  in send_rx_ctrl_cmd()
630 	retval = octeon_send_soft_command(oct, sc);  in send_rx_ctrl_cmd()
633 		octeon_free_soft_command(oct, sc);  in send_rx_ctrl_cmd()
638 		retval = wait_for_sc_completion_timeout(oct, sc, 0);  in send_rx_ctrl_cmd()
642 		oct->props[lio->ifidx].rx_on = start_stop;  in send_rx_ctrl_cmd()
657 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)  in liquidio_destroy_nic_device()  argument
659 	struct net_device *netdev = oct->props[ifidx].netdev;  in liquidio_destroy_nic_device()
660 	struct octeon_device_priv *oct_priv = oct->priv;  in liquidio_destroy_nic_device()
665 		dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",  in liquidio_destroy_nic_device()
672 	dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");  in liquidio_destroy_nic_device()
677 	if (oct->props[lio->ifidx].napi_enabled == 1) {  in liquidio_destroy_nic_device()
681 		oct->props[lio->ifidx].napi_enabled = 0;  in liquidio_destroy_nic_device()
683 		oct->droq[0]->ops.poll_mode = 0;  in liquidio_destroy_nic_device()
703 	oct->props[ifidx].gmxport = -1;  in liquidio_destroy_nic_device()
705 	oct->props[ifidx].netdev = NULL;  in liquidio_destroy_nic_device()
712 static int liquidio_stop_nic_module(struct octeon_device *oct)  in liquidio_stop_nic_module()  argument
717 	dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");  in liquidio_stop_nic_module()
718 	if (!oct->ifcount) {  in liquidio_stop_nic_module()
719 		dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");  in liquidio_stop_nic_module()
723 	spin_lock_bh(&oct->cmd_resp_wqlock);  in liquidio_stop_nic_module()
724 	oct->cmd_resp_state = OCT_DRV_OFFLINE;  in liquidio_stop_nic_module()
725 	spin_unlock_bh(&oct->cmd_resp_wqlock);  in liquidio_stop_nic_module()
727 	for (i = 0; i < oct->ifcount; i++) {  in liquidio_stop_nic_module()
728 		lio = GET_LIO(oct->props[i].netdev);  in liquidio_stop_nic_module()
729 		for (j = 0; j < oct->num_oqs; j++)  in liquidio_stop_nic_module()
730 			octeon_unregister_droq_ops(oct,  in liquidio_stop_nic_module()
734 	for (i = 0; i < oct->ifcount; i++)  in liquidio_stop_nic_module()
735 		liquidio_destroy_nic_device(oct, i);  in liquidio_stop_nic_module()
737 	dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");  in liquidio_stop_nic_module()
771 static int octeon_pci_os_setup(struct octeon_device *oct)  in octeon_pci_os_setup()  argument
775 	if (!oct->pci_dev->physfn)  in octeon_pci_os_setup()
776 		octeon_pci_flr(oct);  in octeon_pci_os_setup()
779 	if (pci_enable_device(oct->pci_dev)) {  in octeon_pci_os_setup()
780 		dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");  in octeon_pci_os_setup()
784 	if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {  in octeon_pci_os_setup()
785 		dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");  in octeon_pci_os_setup()
786 		pci_disable_device(oct->pci_dev);  in octeon_pci_os_setup()
791 	pci_set_master(oct->pci_dev);  in octeon_pci_os_setup()
908 	struct octeon_device *oct = lio->oct_dev;  in liquidio_open()  local
909 	struct octeon_device_priv *oct_priv = oct->priv;  in liquidio_open()
913 	if (!oct->props[lio->ifidx].napi_enabled) {  in liquidio_open()
919 		oct->props[lio->ifidx].napi_enabled = 1;  in liquidio_open()
921 		oct->droq[0]->ops.poll_mode = 1;  in liquidio_open()
942 	dev_info(&oct->pci_dev->dev, "%s interface is opened\n", netdev->name);  in liquidio_open()
954 	struct octeon_device *oct = lio->oct_dev;  in liquidio_stop()  local
955 	struct octeon_device_priv *oct_priv = oct->priv;  in liquidio_stop()
977 	if (lio_wait_for_clean_oq(oct))  in liquidio_stop()
981 	if (oct->props[lio->ifidx].napi_enabled == 1) {  in liquidio_stop()
985 		oct->props[lio->ifidx].napi_enabled = 0;  in liquidio_stop()
987 		oct->droq[0]->ops.poll_mode = 0;  in liquidio_stop()
994 	dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);  in liquidio_stop()
1035 	struct octeon_device *oct = lio->oct_dev;  in liquidio_set_uc_list()  local
1044 		dev_err(&oct->pci_dev->dev, "too many MAC addresses in netdev uc list\n");  in liquidio_set_uc_list()
1053 	nctrl.ncmd.s.param1 = oct->vf_num;  in liquidio_set_uc_list()
1076 	struct octeon_device *oct = lio->oct_dev;  in liquidio_set_mcast_list()  local
1111 		dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",  in liquidio_set_mcast_list()
1127 	struct octeon_device *oct = lio->oct_dev;  in liquidio_set_mac()  local
1155 		dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");  in liquidio_set_mac()
1161 		dev_err(&oct->pci_dev->dev, "MAC Address change failed: no permission\n");  in liquidio_set_mac()
1176 	struct octeon_device *oct;  in liquidio_get_stats64()  local
1182 	oct = lio->oct_dev;  in liquidio_get_stats64()
1187 	for (i = 0; i < oct->num_iqs; i++) {  in liquidio_get_stats64()
1189 		iq_stats = &oct->instr_queue[iq_no]->stats;  in liquidio_get_stats64()
1203 	for (i = 0; i < oct->num_oqs; i++) {  in liquidio_get_stats64()
1205 		oq_stats = &oct->droq[oq_no]->stats;  in liquidio_get_stats64()
1218 	lstats->multicast = oct->link_stats.fromwire.fw_total_mcast;  in liquidio_get_stats64()
1221 	lstats->rx_length_errors = oct->link_stats.fromwire.l2_err;  in liquidio_get_stats64()
1223 	lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err;  in liquidio_get_stats64()
1225 	lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err;  in liquidio_get_stats64()
1231 	lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko;  in liquidio_get_stats64()
1232 	lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link;  in liquidio_get_stats64()
1308 static void handle_timestamp(struct octeon_device *oct, u32 status, void *buf)  in handle_timestamp()  argument
1319 	oct = lio->oct_dev;  in handle_timestamp()
1323 		dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",  in handle_timestamp()
1341 	octeon_free_soft_command(oct, sc);  in handle_timestamp()
1350 static int send_nic_timestamp_pkt(struct octeon_device *oct,  in send_nic_timestamp_pkt()  argument
1363 	sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,  in send_nic_timestamp_pkt()
1368 		dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");  in send_nic_timestamp_pkt()
1385 	retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,  in send_nic_timestamp_pkt()
1389 		dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",  in send_nic_timestamp_pkt()
1391 		octeon_free_soft_command(oct, sc);  in send_nic_timestamp_pkt()
1413 	struct octeon_device *oct;  in liquidio_xmit()  local
1424 	oct = lio->oct_dev;  in liquidio_xmit()
1430 	stats = &oct->instr_queue[iq_no]->stats;  in liquidio_xmit()
1457 	if (octnet_iq_is_full(oct, ndata.q_no)) {  in liquidio_xmit()
1485 		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);  in liquidio_xmit()
1487 		dptr = dma_map_single(&oct->pci_dev->dev,  in liquidio_xmit()
1491 		if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {  in liquidio_xmit()
1492 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",  in liquidio_xmit()
1519 		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);  in liquidio_xmit()
1523 		g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,  in liquidio_xmit()
1527 		if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {  in liquidio_xmit()
1528 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",  in liquidio_xmit()
1540 				skb_frag_dma_map(&oct->pci_dev->dev,  in liquidio_xmit()
1543 			if (dma_mapping_error(&oct->pci_dev->dev,  in liquidio_xmit()
1545 				dma_unmap_single(&oct->pci_dev->dev,  in liquidio_xmit()
1551 					dma_unmap_page(&oct->pci_dev->dev,  in liquidio_xmit()
1556 				dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",  in liquidio_xmit()
1592 		status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);  in liquidio_xmit()
1594 		status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);  in liquidio_xmit()
1601 		dev_err(&oct->pci_dev->dev, "Rcvd IQ_SEND_STOP signal; stopping IQ-%d\n",  in liquidio_xmit()
1621 		dma_unmap_single(&oct->pci_dev->dev, dptr,  in liquidio_xmit()
1624 	octeon_ring_doorbell_locked(oct, iq_no);  in liquidio_xmit()
1653 	struct octeon_device *oct = lio->oct_dev;  in liquidio_vlan_rx_add_vid()  local
1668 		dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",  in liquidio_vlan_rx_add_vid()
1681 	struct octeon_device *oct = lio->oct_dev;  in liquidio_vlan_rx_kill_vid()  local
1696 		dev_err(&oct->pci_dev->dev, "Del VLAN filter failed in core (ret: 0x%x)\n",  in liquidio_vlan_rx_kill_vid()
1715 	struct octeon_device *oct = lio->oct_dev;  in liquidio_set_rxcsum_command()  local
1730 		dev_err(&oct->pci_dev->dev, "DEVFLAGS RXCSUM change failed in core (ret:0x%x)\n",  in liquidio_set_rxcsum_command()
1750 	struct octeon_device *oct = lio->oct_dev;  in liquidio_vxlan_port_command()  local
1766 		dev_err(&oct->pci_dev->dev,  in liquidio_vxlan_port_command()
1890 	struct octeon_device *oct = (struct octeon_device *)buf;  in lio_nic_info()  local
1897 		dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",  in lio_nic_info()
1909 	for (i = 0; i < oct->ifcount; i++) {  in lio_nic_info()
1910 		if (oct->props[i].gmxport == gmxport) {  in lio_nic_info()
1911 			update_link_status(oct->props[i].netdev, ls);  in lio_nic_info()
2242 static int liquidio_init_nic_module(struct octeon_device *oct)  in liquidio_init_nic_module()  argument
2247 	dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");  in liquidio_init_nic_module()
2252 	oct->ifcount = num_nic_ports;  in liquidio_init_nic_module()
2253 	memset(oct->props, 0,  in liquidio_init_nic_module()
2257 		oct->props[i].gmxport = -1;  in liquidio_init_nic_module()
2259 	retval = setup_nic_devices(oct);  in liquidio_init_nic_module()
2261 		dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");  in liquidio_init_nic_module()
2265 	dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");  in liquidio_init_nic_module()
2271 	oct->ifcount = 0;  in liquidio_init_nic_module()
2280 static int octeon_device_init(struct octeon_device *oct)  in octeon_device_init()  argument
2285 	atomic_set(&oct->status, OCT_DEV_BEGIN_STATE);  in octeon_device_init()
2290 	if (octeon_pci_os_setup(oct))  in octeon_device_init()
2292 	atomic_set(&oct->status, OCT_DEV_PCI_ENABLE_DONE);  in octeon_device_init()
2294 	oct->chip_id = OCTEON_CN23XX_VF_VID;  in octeon_device_init()
2295 	pci_read_config_dword(oct->pci_dev, 8, &rev_id);  in octeon_device_init()
2296 	oct->rev_id = rev_id & 0xff;  in octeon_device_init()
2298 	if (cn23xx_setup_octeon_vf_device(oct))  in octeon_device_init()
2301 	atomic_set(&oct->status, OCT_DEV_PCI_MAP_DONE);  in octeon_device_init()
2303 	oct->app_mode = CVM_DRV_NIC_APP;  in octeon_device_init()
2308 	if (octeon_init_dispatch_list(oct))  in octeon_device_init()
2311 	atomic_set(&oct->status, OCT_DEV_DISPATCH_INIT_DONE);  in octeon_device_init()
2313 	if (octeon_set_io_queues_off(oct)) {  in octeon_device_init()
2314 		dev_err(&oct->pci_dev->dev, "setting io queues off failed\n");  in octeon_device_init()
2318 	if (oct->fn_list.setup_device_regs(oct)) {  in octeon_device_init()
2319 		dev_err(&oct->pci_dev->dev, "device registers configuration failed\n");  in octeon_device_init()
2324 	if (octeon_setup_sc_buffer_pool(oct)) {  in octeon_device_init()
2325 		dev_err(&oct->pci_dev->dev, "sc buffer pool allocation failed\n");  in octeon_device_init()
2328 	atomic_set(&oct->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);  in octeon_device_init()
2331 	if (octeon_setup_instr_queues(oct)) {  in octeon_device_init()
2332 		dev_err(&oct->pci_dev->dev, "instruction queue initialization failed\n");  in octeon_device_init()
2335 	atomic_set(&oct->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);  in octeon_device_init()
2340 	if (octeon_setup_response_list(oct)) {  in octeon_device_init()
2341 		dev_err(&oct->pci_dev->dev, "Response list allocation failed\n");  in octeon_device_init()
2344 	atomic_set(&oct->status, OCT_DEV_RESP_LIST_INIT_DONE);  in octeon_device_init()
2346 	if (octeon_setup_output_queues(oct)) {  in octeon_device_init()
2347 		dev_err(&oct->pci_dev->dev, "Output queue initialization failed\n");  in octeon_device_init()
2350 	atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);  in octeon_device_init()
2352 	if (oct->fn_list.setup_mbox(oct)) {  in octeon_device_init()
2353 		dev_err(&oct->pci_dev->dev, "Mailbox setup failed\n");  in octeon_device_init()
2356 	atomic_set(&oct->status, OCT_DEV_MBOX_SETUP_DONE);  in octeon_device_init()
2358 	if (octeon_allocate_ioq_vector(oct, oct->sriov_info.rings_per_vf)) {  in octeon_device_init()
2359 		dev_err(&oct->pci_dev->dev, "ioq vector allocation failed\n");  in octeon_device_init()
2362 	atomic_set(&oct->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE);  in octeon_device_init()
2364 	dev_info(&oct->pci_dev->dev, "OCTEON_CN23XX VF: %d ioqs\n",  in octeon_device_init()
2365 		 oct->sriov_info.rings_per_vf);  in octeon_device_init()
2368 	if (octeon_setup_interrupt(oct, oct->sriov_info.rings_per_vf))  in octeon_device_init()
2371 	atomic_set(&oct->status, OCT_DEV_INTR_SET_DONE);  in octeon_device_init()
2381 	oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);  in octeon_device_init()
2383 	if (cn23xx_octeon_pfvf_handshake(oct))  in octeon_device_init()
2391 	oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);  in octeon_device_init()
2395 	if (oct->fn_list.enable_io_queues(oct)) {  in octeon_device_init()
2396 		dev_err(&oct->pci_dev->dev, "enabling io queues failed\n");  in octeon_device_init()
2400 	atomic_set(&oct->status, OCT_DEV_IO_QUEUES_DONE);  in octeon_device_init()
2402 	atomic_set(&oct->status, OCT_DEV_HOST_OK);  in octeon_device_init()
2407 	for (j = 0; j < oct->num_oqs; j++)  in octeon_device_init()
2408 		writel(oct->droq[j]->max_count, oct->droq[j]->pkts_credit_reg);  in octeon_device_init()
2412 	atomic_set(&oct->status, OCT_DEV_CORE_OK);  in octeon_device_init()
2414 	atomic_set(&oct->status, OCT_DEV_RUNNING);  in octeon_device_init()
2416 	if (liquidio_init_nic_module(oct))  in octeon_device_init()