103 static void qtnf_deassert_intx(struct qtnf_pcie_topaz_state *ts)  in qtnf_deassert_intx()  argument
105 	void __iomem *reg = ts->base.sysctl_bar + TOPAZ_PCIE_CFG0_OFFSET;  in qtnf_deassert_intx()
113 static inline int qtnf_topaz_intx_asserted(struct qtnf_pcie_topaz_state *ts)  in qtnf_topaz_intx_asserted()  argument
115 	void __iomem *reg = ts->base.sysctl_bar + TOPAZ_PCIE_CFG0_OFFSET;  in qtnf_topaz_intx_asserted()
121 static void qtnf_topaz_reset_ep(struct qtnf_pcie_topaz_state *ts)  in qtnf_topaz_reset_ep()  argument
124 	       TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));  in qtnf_topaz_reset_ep()
126 	pci_restore_state(ts->base.pdev);  in qtnf_topaz_reset_ep()
129 static void setup_rx_irqs(struct qtnf_pcie_topaz_state *ts)  in setup_rx_irqs()  argument
131 	void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar);  in setup_rx_irqs()
133 	ts->dma_msi_imwr = readl(reg);  in setup_rx_irqs()
136 static void enable_rx_irqs(struct qtnf_pcie_topaz_state *ts)  in enable_rx_irqs()  argument
138 	void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar);  in enable_rx_irqs()
140 	qtnf_non_posted_write(ts->dma_msi_imwr, reg);  in enable_rx_irqs()
143 static void disable_rx_irqs(struct qtnf_pcie_topaz_state *ts)  in disable_rx_irqs()  argument
145 	void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar);  in disable_rx_irqs()
147 	qtnf_non_posted_write(QTN_HOST_LO32(ts->dma_msi_dummy), reg);  in disable_rx_irqs()
152 	struct qtnf_pcie_topaz_state *ts = arg;  in qtnf_topaz_ipc_gen_ep_int()  local
155 	       TOPAZ_CTL_M2L_INT(ts->base.sysctl_bar));  in qtnf_topaz_ipc_gen_ep_int()
183 static int topaz_alloc_bd_table(struct qtnf_pcie_topaz_state *ts,  in topaz_alloc_bd_table()  argument
187 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in topaz_alloc_bd_table()
205 	ts->tx_bd_vbase = vaddr;  in topaz_alloc_bd_table()
209 		ts->tx_bd_vbase[i].info |= cpu_to_le32(QTN_BD_EMPTY);  in topaz_alloc_bd_table()
221 	ts->rx_bd_vbase = vaddr;  in topaz_alloc_bd_table()
233 	ts->ep_next_rx_pkt = &extra_params->param1;  in topaz_alloc_bd_table()
236 	ts->txqueue_wake = &extra_params->param2;  in topaz_alloc_bd_table()
237 	ts->ep_pmstate = &extra_params->param3;  in topaz_alloc_bd_table()
238 	ts->dma_msi_dummy = paddr + QTNF_BD_PARAM_OFFSET(4);  in topaz_alloc_bd_table()
244 topaz_skb2rbd_attach(struct qtnf_pcie_topaz_state *ts, u16 index, u32 wrap)  in topaz_skb2rbd_attach()  argument
246 	struct qtnf_topaz_rx_bd *rxbd = &ts->rx_bd_vbase[index];  in topaz_skb2rbd_attach()
252 		ts->base.rx_skb[index] = NULL;  in topaz_skb2rbd_attach()
256 	ts->base.rx_skb[index] = skb;  in topaz_skb2rbd_attach()
258 	paddr = dma_map_single(&ts->base.pdev->dev, skb->data, SKB_BUF_SIZE,  in topaz_skb2rbd_attach()
260 	if (dma_mapping_error(&ts->base.pdev->dev, paddr)) {  in topaz_skb2rbd_attach()
268 	ts->base.rx_bd_w_index = index;  in topaz_skb2rbd_attach()
273 static int topaz_alloc_rx_buffers(struct qtnf_pcie_topaz_state *ts)  in topaz_alloc_rx_buffers()  argument
278 	memset(ts->rx_bd_vbase, 0x0,  in topaz_alloc_rx_buffers()
279 	       ts->base.rx_bd_num * sizeof(struct qtnf_topaz_rx_bd));  in topaz_alloc_rx_buffers()
281 	for (i = 0; i < ts->base.rx_bd_num; i++) {  in topaz_alloc_rx_buffers()
282 		ret = topaz_skb2rbd_attach(ts, i, 0);  in topaz_alloc_rx_buffers()
287 	ts->rx_bd_vbase[ts->base.rx_bd_num - 1].info |=  in topaz_alloc_rx_buffers()
294 static void qtnf_topaz_free_xfer_buffers(struct qtnf_pcie_topaz_state *ts)  in qtnf_topaz_free_xfer_buffers()  argument
296 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_topaz_free_xfer_buffers()
306 			rxbd = &ts->rx_bd_vbase[i];  in qtnf_topaz_free_xfer_buffers()
321 			txbd = &ts->tx_bd_vbase[i];  in qtnf_topaz_free_xfer_buffers()
334 static int qtnf_pcie_topaz_init_xfer(struct qtnf_pcie_topaz_state *ts,  in qtnf_pcie_topaz_init_xfer()  argument
338 	struct qtnf_topaz_bda __iomem *bda = ts->bda;  in qtnf_pcie_topaz_init_xfer()
339 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_pcie_topaz_init_xfer()
376 	ret = topaz_alloc_bd_table(ts, bda);  in qtnf_pcie_topaz_init_xfer()
382 	ret = topaz_alloc_rx_buffers(ts);  in qtnf_pcie_topaz_init_xfer()
391 static void qtnf_topaz_data_tx_reclaim(struct qtnf_pcie_topaz_state *ts)  in qtnf_topaz_data_tx_reclaim()  argument
393 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_topaz_data_tx_reclaim()
404 	tx_done_index = readl(ts->ep_next_rx_pkt);  in qtnf_topaz_data_tx_reclaim()
415 			txbd = &ts->tx_bd_vbase[i];  in qtnf_topaz_data_tx_reclaim()
447 	struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);  in qtnf_try_stop_xmit()  local
451 		ts->base.tx_stopped = 1;  in qtnf_try_stop_xmit()
454 	writel(0x0, ts->txqueue_wake);  in qtnf_try_stop_xmit()
461 	       TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));  in qtnf_try_stop_xmit()
464 	tasklet_hi_schedule(&ts->base.reclaim_tq);  in qtnf_try_stop_xmit()
469 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_try_wake_xmit()  local
472 	ready = readl(ts->txqueue_wake);  in qtnf_try_wake_xmit()
478 		       TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));  in qtnf_try_wake_xmit()
482 static int qtnf_tx_queue_ready(struct qtnf_pcie_topaz_state *ts)  in qtnf_tx_queue_ready()  argument
484 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_tx_queue_ready()
488 		qtnf_topaz_data_tx_reclaim(ts);  in qtnf_tx_queue_ready()
503 	struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);  in qtnf_pcie_data_tx()  local
504 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_pcie_data_tx()
505 	struct qtnf_topaz_bda __iomem *bda = ts->bda;  in qtnf_pcie_data_tx()
515 	if (!qtnf_tx_queue_ready(ts)) {  in qtnf_pcie_data_tx()
532 	txbd = &ts->tx_bd_vbase[i];  in qtnf_pcie_data_tx()
560 	qtnf_topaz_data_tx_reclaim(ts);  in qtnf_pcie_data_tx()
568 	struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);  in qtnf_pcie_topaz_interrupt()  local
569 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_pcie_topaz_interrupt()
571 	if (!priv->msi_enabled && !qtnf_topaz_intx_asserted(ts))  in qtnf_pcie_topaz_interrupt()
575 		qtnf_deassert_intx(ts);  in qtnf_pcie_topaz_interrupt()
583 		disable_rx_irqs(ts);  in qtnf_pcie_topaz_interrupt()
592 static int qtnf_rx_data_ready(struct qtnf_pcie_topaz_state *ts)  in qtnf_rx_data_ready()  argument
594 	u16 index = ts->base.rx_bd_r_index;  in qtnf_rx_data_ready()
598 	rxbd = &ts->rx_bd_vbase[index];  in qtnf_rx_data_ready()
610 	struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);  in qtnf_topaz_rx_poll()  local
611 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_topaz_rx_poll()
626 		if (!qtnf_rx_data_ready(ts))  in qtnf_topaz_rx_poll()
630 		rxbd = &ts->rx_bd_vbase[r_idx];  in qtnf_topaz_rx_poll()
681 		if (((++ts->rx_pkt_count) & RX_DONE_INTR_MSK) == 0)  in qtnf_topaz_rx_poll()
698 			ret = topaz_skb2rbd_attach(ts, w_idx,  in qtnf_topaz_rx_poll()
713 		enable_rx_irqs(ts);  in qtnf_topaz_rx_poll()
722 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_data_tx_timeout()  local
725 	tasklet_hi_schedule(&ts->base.reclaim_tq);  in qtnf_pcie_data_tx_timeout()
730 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_data_rx_start()  local
733 	enable_rx_irqs(ts);  in qtnf_pcie_data_rx_start()
738 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_data_rx_stop()  local
740 	disable_rx_irqs(ts);  in qtnf_pcie_data_rx_stop()
758 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_dbg_irq_stats()  local
760 	seq_printf(s, "pcie_irq_count(%u)\n", ts->base.pcie_irq_count);  in qtnf_dbg_irq_stats()
768 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_dbg_pkt_stats()  local
769 	struct qtnf_pcie_bus_priv *priv = &ts->base;  in qtnf_dbg_pkt_stats()
770 	u32 tx_done_index = readl(ts->ep_next_rx_pkt);  in qtnf_dbg_pkt_stats()
800 static void qtnf_reset_dma_offset(struct qtnf_pcie_topaz_state *ts)  in qtnf_reset_dma_offset()  argument
802 	struct qtnf_topaz_bda __iomem *bda = ts->bda;  in qtnf_reset_dma_offset()
811 static int qtnf_pcie_endian_detect(struct qtnf_pcie_topaz_state *ts)  in qtnf_pcie_endian_detect()  argument
813 	struct qtnf_topaz_bda __iomem *bda = ts->bda;  in qtnf_pcie_endian_detect()
852 	struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);  in qtnf_pre_init_ep()  local
853 	struct qtnf_topaz_bda __iomem *bda = ts->bda;  in qtnf_pre_init_ep()
857 	ret = qtnf_pcie_endian_detect(ts);  in qtnf_pre_init_ep()
863 	writeb(ts->base.msi_enabled, &ts->bda->bda_rc_msi_enabled);  in qtnf_pre_init_ep()
864 	qtnf_reset_dma_offset(ts);  in qtnf_pre_init_ep()
869 	if (ts->base.flashboot)  in qtnf_pre_init_ep()
876 	qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_HOST_RDY);  in qtnf_pre_init_ep()
877 	if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_TARGET_RDY,  in qtnf_pre_init_ep()
886 static int qtnf_post_init_ep(struct qtnf_pcie_topaz_state *ts)  in qtnf_post_init_ep()  argument
888 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_post_init_ep()
890 	setup_rx_irqs(ts);  in qtnf_post_init_ep()
891 	disable_rx_irqs(ts);  in qtnf_post_init_ep()
893 	if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_QLINK_DONE,  in qtnf_post_init_ep()
902 qtnf_ep_fw_load(struct qtnf_pcie_topaz_state *ts, const u8 *fw, u32 fw_size)  in qtnf_ep_fw_load()  argument
904 	struct qtnf_topaz_bda __iomem *bda = ts->bda;  in qtnf_ep_fw_load()
905 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_ep_fw_load()
919 	blksize = ts->base.fw_blksize;  in qtnf_ep_fw_load()
942 	qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_HOST_LOAD);  in qtnf_ep_fw_load()
943 	if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_EP_RDY,  in qtnf_ep_fw_load()
960 		qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_RDY);  in qtnf_ep_fw_load()
961 		if (qtnf_poll_state(&ts->bda->bda_bootstate,  in qtnf_ep_fw_load()
977 	qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_RDY);  in qtnf_ep_fw_load()
978 	if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_DONE,  in qtnf_ep_fw_load()
986 	qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_END);  in qtnf_ep_fw_load()
987 	if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_LOAD_DONE,  in qtnf_ep_fw_load()
1003 static int qtnf_topaz_fw_upload(struct qtnf_pcie_topaz_state *ts,  in qtnf_topaz_fw_upload()  argument
1007 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_topaz_fw_upload()
1010 	if (qtnf_poll_state(&ts->bda->bda_bootstate,  in qtnf_topaz_fw_upload()
1025 	ret = qtnf_ep_fw_load(ts, fw->data, fw->size);  in qtnf_topaz_fw_upload()
1037 	struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);  in qtnf_topaz_fw_work_handler()  local
1038 	int bootloader_needed = readl(&ts->bda->bda_flags) & QTN_BDA_XMIT_UBOOT;  in qtnf_topaz_fw_work_handler()
1039 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_topaz_fw_work_handler()
1042 	qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_TARGET_BOOT);  in qtnf_topaz_fw_work_handler()
1045 		ret = qtnf_topaz_fw_upload(ts, QTN_PCI_TOPAZ_BOOTLD_NAME);  in qtnf_topaz_fw_work_handler()
1053 		qtnf_set_state(&ts->bda->bda_bootstate,  in qtnf_topaz_fw_work_handler()
1057 	if (ts->base.flashboot) {  in qtnf_topaz_fw_work_handler()
1060 		ret = qtnf_poll_state(&ts->bda->bda_bootstate,  in qtnf_topaz_fw_work_handler()
1066 		ret = qtnf_topaz_fw_upload(ts, QTN_PCI_TOPAZ_FW_NAME);  in qtnf_topaz_fw_work_handler()
1070 		qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_START);  in qtnf_topaz_fw_work_handler()
1071 		ret = qtnf_poll_state(&ts->bda->bda_bootstate,  in qtnf_topaz_fw_work_handler()
1079 		qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_RUN);  in qtnf_topaz_fw_work_handler()
1080 		ret = qtnf_poll_state(&ts->bda->bda_bootstate,  in qtnf_topaz_fw_work_handler()
1089 	ret = qtnf_post_init_ep(ts);  in qtnf_topaz_fw_work_handler()
1110 	struct qtnf_pcie_topaz_state *ts = from_tasklet(ts, t, base.reclaim_tq);  in qtnf_reclaim_tasklet_fn()  local
1112 	qtnf_topaz_data_tx_reclaim(ts);  in qtnf_reclaim_tasklet_fn()
1123 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_topaz_probe()  local
1124 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_pcie_topaz_probe()
1131 	ts->bda = ts->base.epmem_bar;  in qtnf_pcie_topaz_probe()
1134 	if (ts->base.msi_enabled)  in qtnf_pcie_topaz_probe()
1155 	ret = qtnf_pcie_topaz_init_xfer(ts, tx_bd_num, rx_bd_num);  in qtnf_pcie_topaz_probe()
1161 	tasklet_setup(&ts->base.reclaim_tq, qtnf_reclaim_tasklet_fn);  in qtnf_pcie_topaz_probe()
1166 	ipc_int.arg = ts;  in qtnf_pcie_topaz_probe()
1167 	qtnf_pcie_init_shm_ipc(&ts->base, &ts->bda->bda_shm_reg1,  in qtnf_pcie_topaz_probe()
1168 			       &ts->bda->bda_shm_reg2, &ipc_int);  in qtnf_pcie_topaz_probe()
1175 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_topaz_remove()  local
1177 	qtnf_topaz_reset_ep(ts);  in qtnf_pcie_topaz_remove()
1178 	qtnf_topaz_free_xfer_buffers(ts);  in qtnf_pcie_topaz_remove()
1184 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_topaz_suspend()  local
1185 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_pcie_topaz_suspend()
1187 	writel((u32 __force)PCI_D3hot, ts->ep_pmstate);  in qtnf_pcie_topaz_suspend()
1190 	       TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));  in qtnf_pcie_topaz_suspend()
1201 	struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);  in qtnf_pcie_topaz_resume()  local
1202 	struct pci_dev *pdev = ts->base.pdev;  in qtnf_pcie_topaz_resume()
1208 	writel((u32 __force)PCI_D0, ts->ep_pmstate);  in qtnf_pcie_topaz_resume()
1211 	       TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));  in qtnf_pcie_topaz_resume()
1220 	struct qtnf_pcie_topaz_state *ts;  in qtnf_pcie_topaz_alloc()  local
1222 	bus = devm_kzalloc(&pdev->dev, sizeof(*bus) + sizeof(*ts), GFP_KERNEL);  in qtnf_pcie_topaz_alloc()
1226 	ts = get_bus_priv(bus);  in qtnf_pcie_topaz_alloc()
1227 	ts->base.probe_cb = qtnf_pcie_topaz_probe;  in qtnf_pcie_topaz_alloc()
1228 	ts->base.remove_cb = qtnf_pcie_topaz_remove;  in qtnf_pcie_topaz_alloc()
1229 	ts->base.dma_mask_get_cb = qtnf_topaz_dma_mask_get;  in qtnf_pcie_topaz_alloc()
1231 	ts->base.resume_cb = qtnf_pcie_topaz_resume;  in qtnf_pcie_topaz_alloc()
1232 	ts->base.suspend_cb = qtnf_pcie_topaz_suspend;  in qtnf_pcie_topaz_alloc()