1e401fa25SSergey Matyukevich // SPDX-License-Identifier: GPL-2.0+
2e401fa25SSergey Matyukevich /* Copyright (c) 2018 Quantenna Communications */
3e401fa25SSergey Matyukevich
4e401fa25SSergey Matyukevich #include <linux/kernel.h>
5e401fa25SSergey Matyukevich #include <linux/firmware.h>
6e401fa25SSergey Matyukevich #include <linux/pci.h>
7e401fa25SSergey Matyukevich #include <linux/vmalloc.h>
8e401fa25SSergey Matyukevich #include <linux/delay.h>
9e401fa25SSergey Matyukevich #include <linux/interrupt.h>
10e401fa25SSergey Matyukevich #include <linux/sched.h>
11e401fa25SSergey Matyukevich #include <linux/crc32.h>
12e401fa25SSergey Matyukevich #include <linux/completion.h>
13e401fa25SSergey Matyukevich #include <linux/spinlock.h>
14e401fa25SSergey Matyukevich #include <linux/circ_buf.h>
15e401fa25SSergey Matyukevich
16e401fa25SSergey Matyukevich #include "pcie_priv.h"
17e401fa25SSergey Matyukevich #include "topaz_pcie_regs.h"
18e401fa25SSergey Matyukevich #include "topaz_pcie_ipc.h"
19e401fa25SSergey Matyukevich #include "qtn_hw_ids.h"
20e401fa25SSergey Matyukevich #include "core.h"
21e401fa25SSergey Matyukevich #include "bus.h"
22e401fa25SSergey Matyukevich #include "shm_ipc.h"
23e401fa25SSergey Matyukevich #include "debug.h"
24e401fa25SSergey Matyukevich
25e401fa25SSergey Matyukevich #define TOPAZ_TX_BD_SIZE_DEFAULT 128
2697aef03cSSergey Matyukevich #define TOPAZ_RX_BD_SIZE_DEFAULT 256
27e401fa25SSergey Matyukevich
28e401fa25SSergey Matyukevich struct qtnf_topaz_tx_bd {
29e401fa25SSergey Matyukevich __le32 addr;
30e401fa25SSergey Matyukevich __le32 info;
31e401fa25SSergey Matyukevich } __packed;
32e401fa25SSergey Matyukevich
33e401fa25SSergey Matyukevich struct qtnf_topaz_rx_bd {
34e401fa25SSergey Matyukevich __le32 addr;
35e401fa25SSergey Matyukevich __le32 info;
36e401fa25SSergey Matyukevich } __packed;
37e401fa25SSergey Matyukevich
38e401fa25SSergey Matyukevich struct qtnf_extra_bd_params {
39e401fa25SSergey Matyukevich __le32 param1;
40e401fa25SSergey Matyukevich __le32 param2;
41e401fa25SSergey Matyukevich __le32 param3;
42e401fa25SSergey Matyukevich __le32 param4;
43e401fa25SSergey Matyukevich } __packed;
44e401fa25SSergey Matyukevich
45e401fa25SSergey Matyukevich #define QTNF_BD_PARAM_OFFSET(n) offsetof(struct qtnf_extra_bd_params, param##n)
46e401fa25SSergey Matyukevich
47e401fa25SSergey Matyukevich struct vmac_pkt_info {
48e401fa25SSergey Matyukevich __le32 addr;
49e401fa25SSergey Matyukevich __le32 info;
50e401fa25SSergey Matyukevich };
51e401fa25SSergey Matyukevich
52e401fa25SSergey Matyukevich struct qtnf_topaz_bda {
53e401fa25SSergey Matyukevich __le16 bda_len;
54e401fa25SSergey Matyukevich __le16 bda_version;
55e401fa25SSergey Matyukevich __le32 bda_bootstate;
56e401fa25SSergey Matyukevich __le32 bda_dma_mask;
57e401fa25SSergey Matyukevich __le32 bda_dma_offset;
58e401fa25SSergey Matyukevich __le32 bda_flags;
59e401fa25SSergey Matyukevich __le32 bda_img;
60e401fa25SSergey Matyukevich __le32 bda_img_size;
61e401fa25SSergey Matyukevich __le32 bda_ep2h_irqstatus;
62e401fa25SSergey Matyukevich __le32 bda_h2ep_irqstatus;
63e401fa25SSergey Matyukevich __le32 bda_msi_addr;
64e401fa25SSergey Matyukevich u8 reserved1[56];
65e401fa25SSergey Matyukevich __le32 bda_flashsz;
66e401fa25SSergey Matyukevich u8 bda_boardname[PCIE_BDA_NAMELEN];
67e401fa25SSergey Matyukevich __le32 bda_pci_pre_status;
68e401fa25SSergey Matyukevich __le32 bda_pci_endian;
69e401fa25SSergey Matyukevich __le32 bda_pci_post_status;
70e401fa25SSergey Matyukevich __le32 bda_h2ep_txd_budget;
71e401fa25SSergey Matyukevich __le32 bda_ep2h_txd_budget;
72e401fa25SSergey Matyukevich __le32 bda_rc_rx_bd_base;
73e401fa25SSergey Matyukevich __le32 bda_rc_rx_bd_num;
74e401fa25SSergey Matyukevich __le32 bda_rc_tx_bd_base;
75e401fa25SSergey Matyukevich __le32 bda_rc_tx_bd_num;
76e401fa25SSergey Matyukevich u8 bda_ep_link_state;
77e401fa25SSergey Matyukevich u8 bda_rc_link_state;
78e401fa25SSergey Matyukevich u8 bda_rc_msi_enabled;
79e401fa25SSergey Matyukevich u8 reserved2;
80e401fa25SSergey Matyukevich __le32 bda_ep_next_pkt;
81e401fa25SSergey Matyukevich struct vmac_pkt_info request[QTN_PCIE_RC_TX_QUEUE_LEN];
82e401fa25SSergey Matyukevich struct qtnf_shm_ipc_region bda_shm_reg1 __aligned(4096);
83e401fa25SSergey Matyukevich struct qtnf_shm_ipc_region bda_shm_reg2 __aligned(4096);
84e401fa25SSergey Matyukevich } __packed;
85e401fa25SSergey Matyukevich
86e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state {
87e401fa25SSergey Matyukevich struct qtnf_pcie_bus_priv base;
88e401fa25SSergey Matyukevich struct qtnf_topaz_bda __iomem *bda;
89e401fa25SSergey Matyukevich
90e401fa25SSergey Matyukevich dma_addr_t dma_msi_dummy;
91e401fa25SSergey Matyukevich u32 dma_msi_imwr;
92e401fa25SSergey Matyukevich
93e401fa25SSergey Matyukevich struct qtnf_topaz_tx_bd *tx_bd_vbase;
94e401fa25SSergey Matyukevich struct qtnf_topaz_rx_bd *rx_bd_vbase;
95e401fa25SSergey Matyukevich
96e401fa25SSergey Matyukevich __le32 __iomem *ep_next_rx_pkt;
97e401fa25SSergey Matyukevich __le32 __iomem *txqueue_wake;
98e401fa25SSergey Matyukevich __le32 __iomem *ep_pmstate;
99e401fa25SSergey Matyukevich
100e401fa25SSergey Matyukevich unsigned long rx_pkt_count;
101e401fa25SSergey Matyukevich };
102e401fa25SSergey Matyukevich
qtnf_deassert_intx(struct qtnf_pcie_topaz_state * ts)103e401fa25SSergey Matyukevich static void qtnf_deassert_intx(struct qtnf_pcie_topaz_state *ts)
104e401fa25SSergey Matyukevich {
105e401fa25SSergey Matyukevich void __iomem *reg = ts->base.sysctl_bar + TOPAZ_PCIE_CFG0_OFFSET;
106e401fa25SSergey Matyukevich u32 cfg;
107e401fa25SSergey Matyukevich
108e401fa25SSergey Matyukevich cfg = readl(reg);
109e401fa25SSergey Matyukevich cfg &= ~TOPAZ_ASSERT_INTX;
110e401fa25SSergey Matyukevich qtnf_non_posted_write(cfg, reg);
111e401fa25SSergey Matyukevich }
112e401fa25SSergey Matyukevich
qtnf_topaz_intx_asserted(struct qtnf_pcie_topaz_state * ts)113e401fa25SSergey Matyukevich static inline int qtnf_topaz_intx_asserted(struct qtnf_pcie_topaz_state *ts)
114e401fa25SSergey Matyukevich {
115e401fa25SSergey Matyukevich void __iomem *reg = ts->base.sysctl_bar + TOPAZ_PCIE_CFG0_OFFSET;
116e401fa25SSergey Matyukevich u32 cfg = readl(reg);
117e401fa25SSergey Matyukevich
118e401fa25SSergey Matyukevich return !!(cfg & TOPAZ_ASSERT_INTX);
119e401fa25SSergey Matyukevich }
120e401fa25SSergey Matyukevich
qtnf_topaz_reset_ep(struct qtnf_pcie_topaz_state * ts)121e401fa25SSergey Matyukevich static void qtnf_topaz_reset_ep(struct qtnf_pcie_topaz_state *ts)
122e401fa25SSergey Matyukevich {
123e401fa25SSergey Matyukevich writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_RST_EP_IRQ),
124e401fa25SSergey Matyukevich TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));
125e401fa25SSergey Matyukevich msleep(QTN_EP_RESET_WAIT_MS);
126e401fa25SSergey Matyukevich pci_restore_state(ts->base.pdev);
127e401fa25SSergey Matyukevich }
128e401fa25SSergey Matyukevich
setup_rx_irqs(struct qtnf_pcie_topaz_state * ts)129e401fa25SSergey Matyukevich static void setup_rx_irqs(struct qtnf_pcie_topaz_state *ts)
130e401fa25SSergey Matyukevich {
131e401fa25SSergey Matyukevich void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar);
132e401fa25SSergey Matyukevich
133e401fa25SSergey Matyukevich ts->dma_msi_imwr = readl(reg);
134e401fa25SSergey Matyukevich }
135e401fa25SSergey Matyukevich
enable_rx_irqs(struct qtnf_pcie_topaz_state * ts)136e401fa25SSergey Matyukevich static void enable_rx_irqs(struct qtnf_pcie_topaz_state *ts)
137e401fa25SSergey Matyukevich {
138e401fa25SSergey Matyukevich void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar);
139e401fa25SSergey Matyukevich
140e401fa25SSergey Matyukevich qtnf_non_posted_write(ts->dma_msi_imwr, reg);
141e401fa25SSergey Matyukevich }
142e401fa25SSergey Matyukevich
disable_rx_irqs(struct qtnf_pcie_topaz_state * ts)143e401fa25SSergey Matyukevich static void disable_rx_irqs(struct qtnf_pcie_topaz_state *ts)
144e401fa25SSergey Matyukevich {
145e401fa25SSergey Matyukevich void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar);
146e401fa25SSergey Matyukevich
147e401fa25SSergey Matyukevich qtnf_non_posted_write(QTN_HOST_LO32(ts->dma_msi_dummy), reg);
148e401fa25SSergey Matyukevich }
149e401fa25SSergey Matyukevich
qtnf_topaz_ipc_gen_ep_int(void * arg)150e401fa25SSergey Matyukevich static void qtnf_topaz_ipc_gen_ep_int(void *arg)
151e401fa25SSergey Matyukevich {
152e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = arg;
153e401fa25SSergey Matyukevich
154e401fa25SSergey Matyukevich writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_CTRL_IRQ),
155e401fa25SSergey Matyukevich TOPAZ_CTL_M2L_INT(ts->base.sysctl_bar));
156e401fa25SSergey Matyukevich }
157e401fa25SSergey Matyukevich
qtnf_is_state(__le32 __iomem * reg,u32 state)158e401fa25SSergey Matyukevich static int qtnf_is_state(__le32 __iomem *reg, u32 state)
159e401fa25SSergey Matyukevich {
160e401fa25SSergey Matyukevich u32 s = readl(reg);
161e401fa25SSergey Matyukevich
162e401fa25SSergey Matyukevich return (s == state);
163e401fa25SSergey Matyukevich }
164e401fa25SSergey Matyukevich
qtnf_set_state(__le32 __iomem * reg,u32 state)165e401fa25SSergey Matyukevich static void qtnf_set_state(__le32 __iomem *reg, u32 state)
166e401fa25SSergey Matyukevich {
167e401fa25SSergey Matyukevich qtnf_non_posted_write(state, reg);
168e401fa25SSergey Matyukevich }
169e401fa25SSergey Matyukevich
qtnf_poll_state(__le32 __iomem * reg,u32 state,u32 delay_in_ms)170e401fa25SSergey Matyukevich static int qtnf_poll_state(__le32 __iomem *reg, u32 state, u32 delay_in_ms)
171e401fa25SSergey Matyukevich {
172e401fa25SSergey Matyukevich u32 timeout = 0;
173e401fa25SSergey Matyukevich
174e401fa25SSergey Matyukevich while ((qtnf_is_state(reg, state) == 0)) {
175e401fa25SSergey Matyukevich usleep_range(1000, 1200);
176e401fa25SSergey Matyukevich if (++timeout > delay_in_ms)
177e401fa25SSergey Matyukevich return -1;
178e401fa25SSergey Matyukevich }
179e401fa25SSergey Matyukevich
180e401fa25SSergey Matyukevich return 0;
181e401fa25SSergey Matyukevich }
182e401fa25SSergey Matyukevich
topaz_alloc_bd_table(struct qtnf_pcie_topaz_state * ts,struct qtnf_topaz_bda __iomem * bda)183e401fa25SSergey Matyukevich static int topaz_alloc_bd_table(struct qtnf_pcie_topaz_state *ts,
184e401fa25SSergey Matyukevich struct qtnf_topaz_bda __iomem *bda)
185e401fa25SSergey Matyukevich {
186e401fa25SSergey Matyukevich struct qtnf_extra_bd_params __iomem *extra_params;
187e401fa25SSergey Matyukevich struct qtnf_pcie_bus_priv *priv = &ts->base;
188e401fa25SSergey Matyukevich dma_addr_t paddr;
189e401fa25SSergey Matyukevich void *vaddr;
190e401fa25SSergey Matyukevich int len;
191e401fa25SSergey Matyukevich int i;
192e401fa25SSergey Matyukevich
193e401fa25SSergey Matyukevich /* bd table */
194e401fa25SSergey Matyukevich
195e401fa25SSergey Matyukevich len = priv->tx_bd_num * sizeof(struct qtnf_topaz_tx_bd) +
196e401fa25SSergey Matyukevich priv->rx_bd_num * sizeof(struct qtnf_topaz_rx_bd) +
197e401fa25SSergey Matyukevich sizeof(struct qtnf_extra_bd_params);
198e401fa25SSergey Matyukevich
199e401fa25SSergey Matyukevich vaddr = dmam_alloc_coherent(&priv->pdev->dev, len, &paddr, GFP_KERNEL);
200e401fa25SSergey Matyukevich if (!vaddr)
201e401fa25SSergey Matyukevich return -ENOMEM;
202e401fa25SSergey Matyukevich
203e401fa25SSergey Matyukevich /* tx bd */
204e401fa25SSergey Matyukevich
205e401fa25SSergey Matyukevich ts->tx_bd_vbase = vaddr;
206e401fa25SSergey Matyukevich qtnf_non_posted_write(paddr, &bda->bda_rc_tx_bd_base);
207e401fa25SSergey Matyukevich
208e401fa25SSergey Matyukevich for (i = 0; i < priv->tx_bd_num; i++)
209e401fa25SSergey Matyukevich ts->tx_bd_vbase[i].info |= cpu_to_le32(QTN_BD_EMPTY);
210e401fa25SSergey Matyukevich
211e401fa25SSergey Matyukevich pr_debug("TX descriptor table: vaddr=0x%p paddr=%pad\n", vaddr, &paddr);
212e401fa25SSergey Matyukevich
213e401fa25SSergey Matyukevich priv->tx_bd_r_index = 0;
214e401fa25SSergey Matyukevich priv->tx_bd_w_index = 0;
215e401fa25SSergey Matyukevich
216e401fa25SSergey Matyukevich /* rx bd */
217e401fa25SSergey Matyukevich
218e401fa25SSergey Matyukevich vaddr = ((struct qtnf_topaz_tx_bd *)vaddr) + priv->tx_bd_num;
219e401fa25SSergey Matyukevich paddr += priv->tx_bd_num * sizeof(struct qtnf_topaz_tx_bd);
220e401fa25SSergey Matyukevich
221e401fa25SSergey Matyukevich ts->rx_bd_vbase = vaddr;
222e401fa25SSergey Matyukevich qtnf_non_posted_write(paddr, &bda->bda_rc_rx_bd_base);
223e401fa25SSergey Matyukevich
224e401fa25SSergey Matyukevich pr_debug("RX descriptor table: vaddr=0x%p paddr=%pad\n", vaddr, &paddr);
225e401fa25SSergey Matyukevich
226e401fa25SSergey Matyukevich /* extra shared params */
227e401fa25SSergey Matyukevich
228e401fa25SSergey Matyukevich vaddr = ((struct qtnf_topaz_rx_bd *)vaddr) + priv->rx_bd_num;
229e401fa25SSergey Matyukevich paddr += priv->rx_bd_num * sizeof(struct qtnf_topaz_rx_bd);
230e401fa25SSergey Matyukevich
231e401fa25SSergey Matyukevich extra_params = (struct qtnf_extra_bd_params __iomem *)vaddr;
232e401fa25SSergey Matyukevich
233e401fa25SSergey Matyukevich ts->ep_next_rx_pkt = &extra_params->param1;
234e401fa25SSergey Matyukevich qtnf_non_posted_write(paddr + QTNF_BD_PARAM_OFFSET(1),
235e401fa25SSergey Matyukevich &bda->bda_ep_next_pkt);
236e401fa25SSergey Matyukevich ts->txqueue_wake = &extra_params->param2;
237e401fa25SSergey Matyukevich ts->ep_pmstate = &extra_params->param3;
238e401fa25SSergey Matyukevich ts->dma_msi_dummy = paddr + QTNF_BD_PARAM_OFFSET(4);
239e401fa25SSergey Matyukevich
240e401fa25SSergey Matyukevich return 0;
241e401fa25SSergey Matyukevich }
242e401fa25SSergey Matyukevich
243e401fa25SSergey Matyukevich static int
topaz_skb2rbd_attach(struct qtnf_pcie_topaz_state * ts,u16 index,u32 wrap)244e401fa25SSergey Matyukevich topaz_skb2rbd_attach(struct qtnf_pcie_topaz_state *ts, u16 index, u32 wrap)
245e401fa25SSergey Matyukevich {
246e401fa25SSergey Matyukevich struct qtnf_topaz_rx_bd *rxbd = &ts->rx_bd_vbase[index];
247e401fa25SSergey Matyukevich struct sk_buff *skb;
248e401fa25SSergey Matyukevich dma_addr_t paddr;
249e401fa25SSergey Matyukevich
250c960e2b3SChristophe JAILLET skb = netdev_alloc_skb_ip_align(NULL, SKB_BUF_SIZE);
251e401fa25SSergey Matyukevich if (!skb) {
252e401fa25SSergey Matyukevich ts->base.rx_skb[index] = NULL;
253e401fa25SSergey Matyukevich return -ENOMEM;
254e401fa25SSergey Matyukevich }
255e401fa25SSergey Matyukevich
256e401fa25SSergey Matyukevich ts->base.rx_skb[index] = skb;
257e401fa25SSergey Matyukevich
25806e1359cSChristophe JAILLET paddr = dma_map_single(&ts->base.pdev->dev, skb->data, SKB_BUF_SIZE,
25906e1359cSChristophe JAILLET DMA_FROM_DEVICE);
26006e1359cSChristophe JAILLET if (dma_mapping_error(&ts->base.pdev->dev, paddr)) {
261e401fa25SSergey Matyukevich pr_err("skb mapping error: %pad\n", &paddr);
262e401fa25SSergey Matyukevich return -ENOMEM;
263e401fa25SSergey Matyukevich }
264e401fa25SSergey Matyukevich
265e401fa25SSergey Matyukevich rxbd->addr = cpu_to_le32(QTN_HOST_LO32(paddr));
266e401fa25SSergey Matyukevich rxbd->info = cpu_to_le32(QTN_BD_EMPTY | wrap);
267e401fa25SSergey Matyukevich
268e401fa25SSergey Matyukevich ts->base.rx_bd_w_index = index;
269e401fa25SSergey Matyukevich
270e401fa25SSergey Matyukevich return 0;
271e401fa25SSergey Matyukevich }
272e401fa25SSergey Matyukevich
topaz_alloc_rx_buffers(struct qtnf_pcie_topaz_state * ts)273e401fa25SSergey Matyukevich static int topaz_alloc_rx_buffers(struct qtnf_pcie_topaz_state *ts)
274e401fa25SSergey Matyukevich {
275e401fa25SSergey Matyukevich u16 i;
276e401fa25SSergey Matyukevich int ret = 0;
277e401fa25SSergey Matyukevich
278e401fa25SSergey Matyukevich memset(ts->rx_bd_vbase, 0x0,
279e401fa25SSergey Matyukevich ts->base.rx_bd_num * sizeof(struct qtnf_topaz_rx_bd));
280e401fa25SSergey Matyukevich
281e401fa25SSergey Matyukevich for (i = 0; i < ts->base.rx_bd_num; i++) {
282e401fa25SSergey Matyukevich ret = topaz_skb2rbd_attach(ts, i, 0);
283e401fa25SSergey Matyukevich if (ret)
284e401fa25SSergey Matyukevich break;
285e401fa25SSergey Matyukevich }
286e401fa25SSergey Matyukevich
287e401fa25SSergey Matyukevich ts->rx_bd_vbase[ts->base.rx_bd_num - 1].info |=
288e401fa25SSergey Matyukevich cpu_to_le32(QTN_BD_WRAP);
289e401fa25SSergey Matyukevich
290e401fa25SSergey Matyukevich return ret;
291e401fa25SSergey Matyukevich }
292e401fa25SSergey Matyukevich
293e401fa25SSergey Matyukevich /* all rx/tx activity should have ceased before calling this function */
qtnf_topaz_free_xfer_buffers(struct qtnf_pcie_topaz_state * ts)294e401fa25SSergey Matyukevich static void qtnf_topaz_free_xfer_buffers(struct qtnf_pcie_topaz_state *ts)
295e401fa25SSergey Matyukevich {
296e401fa25SSergey Matyukevich struct qtnf_pcie_bus_priv *priv = &ts->base;
297e401fa25SSergey Matyukevich struct qtnf_topaz_rx_bd *rxbd;
298e401fa25SSergey Matyukevich struct qtnf_topaz_tx_bd *txbd;
299e401fa25SSergey Matyukevich struct sk_buff *skb;
300e401fa25SSergey Matyukevich dma_addr_t paddr;
301e401fa25SSergey Matyukevich int i;
302e401fa25SSergey Matyukevich
303e401fa25SSergey Matyukevich /* free rx buffers */
304e401fa25SSergey Matyukevich for (i = 0; i < priv->rx_bd_num; i++) {
305e401fa25SSergey Matyukevich if (priv->rx_skb && priv->rx_skb[i]) {
306e401fa25SSergey Matyukevich rxbd = &ts->rx_bd_vbase[i];
307e401fa25SSergey Matyukevich skb = priv->rx_skb[i];
308e401fa25SSergey Matyukevich paddr = QTN_HOST_ADDR(0x0, le32_to_cpu(rxbd->addr));
30906e1359cSChristophe JAILLET dma_unmap_single(&priv->pdev->dev, paddr,
31006e1359cSChristophe JAILLET SKB_BUF_SIZE, DMA_FROM_DEVICE);
311e401fa25SSergey Matyukevich dev_kfree_skb_any(skb);
312e401fa25SSergey Matyukevich priv->rx_skb[i] = NULL;
313e401fa25SSergey Matyukevich rxbd->addr = 0;
314e401fa25SSergey Matyukevich rxbd->info = 0;
315e401fa25SSergey Matyukevich }
316e401fa25SSergey Matyukevich }
317e401fa25SSergey Matyukevich
318e401fa25SSergey Matyukevich /* free tx buffers */
319e401fa25SSergey Matyukevich for (i = 0; i < priv->tx_bd_num; i++) {
320e401fa25SSergey Matyukevich if (priv->tx_skb && priv->tx_skb[i]) {
321e401fa25SSergey Matyukevich txbd = &ts->tx_bd_vbase[i];
322e401fa25SSergey Matyukevich skb = priv->tx_skb[i];
323e401fa25SSergey Matyukevich paddr = QTN_HOST_ADDR(0x0, le32_to_cpu(txbd->addr));
32406e1359cSChristophe JAILLET dma_unmap_single(&priv->pdev->dev, paddr,
32506e1359cSChristophe JAILLET SKB_BUF_SIZE, DMA_TO_DEVICE);
326e401fa25SSergey Matyukevich dev_kfree_skb_any(skb);
327e401fa25SSergey Matyukevich priv->tx_skb[i] = NULL;
328e401fa25SSergey Matyukevich txbd->addr = 0;
329e401fa25SSergey Matyukevich txbd->info = 0;
330e401fa25SSergey Matyukevich }
331e401fa25SSergey Matyukevich }
332e401fa25SSergey Matyukevich }
333e401fa25SSergey Matyukevich
qtnf_pcie_topaz_init_xfer(struct qtnf_pcie_topaz_state * ts,unsigned int tx_bd_size,unsigned int rx_bd_size)334e401fa25SSergey Matyukevich static int qtnf_pcie_topaz_init_xfer(struct qtnf_pcie_topaz_state *ts,
33597aef03cSSergey Matyukevich unsigned int tx_bd_size,
33697aef03cSSergey Matyukevich unsigned int rx_bd_size)
337e401fa25SSergey Matyukevich {
338e401fa25SSergey Matyukevich struct qtnf_topaz_bda __iomem *bda = ts->bda;
339e401fa25SSergey Matyukevich struct qtnf_pcie_bus_priv *priv = &ts->base;
340e401fa25SSergey Matyukevich int ret;
341e401fa25SSergey Matyukevich
342e401fa25SSergey Matyukevich if (tx_bd_size == 0)
343e401fa25SSergey Matyukevich tx_bd_size = TOPAZ_TX_BD_SIZE_DEFAULT;
344e401fa25SSergey Matyukevich
345e401fa25SSergey Matyukevich /* check TX BD queue max length according to struct qtnf_topaz_bda */
346e401fa25SSergey Matyukevich if (tx_bd_size > QTN_PCIE_RC_TX_QUEUE_LEN) {
347e401fa25SSergey Matyukevich pr_warn("TX BD queue cannot exceed %d\n",
348e401fa25SSergey Matyukevich QTN_PCIE_RC_TX_QUEUE_LEN);
349e401fa25SSergey Matyukevich tx_bd_size = QTN_PCIE_RC_TX_QUEUE_LEN;
350e401fa25SSergey Matyukevich }
351e401fa25SSergey Matyukevich
352e401fa25SSergey Matyukevich priv->tx_bd_num = tx_bd_size;
353e401fa25SSergey Matyukevich qtnf_non_posted_write(priv->tx_bd_num, &bda->bda_rc_tx_bd_num);
35497aef03cSSergey Matyukevich
35597aef03cSSergey Matyukevich if (rx_bd_size == 0)
35697aef03cSSergey Matyukevich rx_bd_size = TOPAZ_RX_BD_SIZE_DEFAULT;
35797aef03cSSergey Matyukevich
35897aef03cSSergey Matyukevich if (rx_bd_size > TOPAZ_RX_BD_SIZE_DEFAULT) {
35997aef03cSSergey Matyukevich pr_warn("RX BD queue cannot exceed %d\n",
36097aef03cSSergey Matyukevich TOPAZ_RX_BD_SIZE_DEFAULT);
36197aef03cSSergey Matyukevich rx_bd_size = TOPAZ_RX_BD_SIZE_DEFAULT;
36297aef03cSSergey Matyukevich }
36397aef03cSSergey Matyukevich
36497aef03cSSergey Matyukevich priv->rx_bd_num = rx_bd_size;
365e401fa25SSergey Matyukevich qtnf_non_posted_write(priv->rx_bd_num, &bda->bda_rc_rx_bd_num);
366e401fa25SSergey Matyukevich
367e401fa25SSergey Matyukevich priv->rx_bd_w_index = 0;
368e401fa25SSergey Matyukevich priv->rx_bd_r_index = 0;
369e401fa25SSergey Matyukevich
370e401fa25SSergey Matyukevich ret = qtnf_pcie_alloc_skb_array(priv);
371e401fa25SSergey Matyukevich if (ret) {
372e401fa25SSergey Matyukevich pr_err("failed to allocate skb array\n");
373e401fa25SSergey Matyukevich return ret;
374e401fa25SSergey Matyukevich }
375e401fa25SSergey Matyukevich
376e401fa25SSergey Matyukevich ret = topaz_alloc_bd_table(ts, bda);
377e401fa25SSergey Matyukevich if (ret) {
378e401fa25SSergey Matyukevich pr_err("failed to allocate bd table\n");
379e401fa25SSergey Matyukevich return ret;
380e401fa25SSergey Matyukevich }
381e401fa25SSergey Matyukevich
382e401fa25SSergey Matyukevich ret = topaz_alloc_rx_buffers(ts);
383e401fa25SSergey Matyukevich if (ret) {
384e401fa25SSergey Matyukevich pr_err("failed to allocate rx buffers\n");
385e401fa25SSergey Matyukevich return ret;
386e401fa25SSergey Matyukevich }
387e401fa25SSergey Matyukevich
388e401fa25SSergey Matyukevich return ret;
389e401fa25SSergey Matyukevich }
390e401fa25SSergey Matyukevich
qtnf_topaz_data_tx_reclaim(struct qtnf_pcie_topaz_state * ts)391e401fa25SSergey Matyukevich static void qtnf_topaz_data_tx_reclaim(struct qtnf_pcie_topaz_state *ts)
392e401fa25SSergey Matyukevich {
393e401fa25SSergey Matyukevich struct qtnf_pcie_bus_priv *priv = &ts->base;
394e401fa25SSergey Matyukevich struct qtnf_topaz_tx_bd *txbd;
395e401fa25SSergey Matyukevich struct sk_buff *skb;
396e401fa25SSergey Matyukevich unsigned long flags;
397e401fa25SSergey Matyukevich dma_addr_t paddr;
398e401fa25SSergey Matyukevich u32 tx_done_index;
399e401fa25SSergey Matyukevich int count = 0;
400e401fa25SSergey Matyukevich int i;
401e401fa25SSergey Matyukevich
402e401fa25SSergey Matyukevich spin_lock_irqsave(&priv->tx_reclaim_lock, flags);
403e401fa25SSergey Matyukevich
404e401fa25SSergey Matyukevich tx_done_index = readl(ts->ep_next_rx_pkt);
405e401fa25SSergey Matyukevich i = priv->tx_bd_r_index;
406e401fa25SSergey Matyukevich
407e401fa25SSergey Matyukevich if (CIRC_CNT(priv->tx_bd_w_index, tx_done_index, priv->tx_bd_num))
408e401fa25SSergey Matyukevich writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_TX_DONE_IRQ),
409e401fa25SSergey Matyukevich TOPAZ_LH_IPC4_INT(priv->sysctl_bar));
410e401fa25SSergey Matyukevich
411e401fa25SSergey Matyukevich while (CIRC_CNT(tx_done_index, i, priv->tx_bd_num)) {
412e401fa25SSergey Matyukevich skb = priv->tx_skb[i];
413e401fa25SSergey Matyukevich
414e401fa25SSergey Matyukevich if (likely(skb)) {
415e401fa25SSergey Matyukevich txbd = &ts->tx_bd_vbase[i];
416e401fa25SSergey Matyukevich paddr = QTN_HOST_ADDR(0x0, le32_to_cpu(txbd->addr));
41706e1359cSChristophe JAILLET dma_unmap_single(&priv->pdev->dev, paddr, skb->len,
41806e1359cSChristophe JAILLET DMA_TO_DEVICE);
419e401fa25SSergey Matyukevich
420e401fa25SSergey Matyukevich if (skb->dev) {
42145fc3fd4SHeiner Kallweit dev_sw_netstats_tx_add(skb->dev, 1, skb->len);
422e401fa25SSergey Matyukevich if (unlikely(priv->tx_stopped)) {
423e401fa25SSergey Matyukevich qtnf_wake_all_queues(skb->dev);
424e401fa25SSergey Matyukevich priv->tx_stopped = 0;
425e401fa25SSergey Matyukevich }
426e401fa25SSergey Matyukevich }
427e401fa25SSergey Matyukevich
428e401fa25SSergey Matyukevich dev_kfree_skb_any(skb);
429e401fa25SSergey Matyukevich }
430e401fa25SSergey Matyukevich
431e401fa25SSergey Matyukevich priv->tx_skb[i] = NULL;
432e401fa25SSergey Matyukevich count++;
433e401fa25SSergey Matyukevich
434e401fa25SSergey Matyukevich if (++i >= priv->tx_bd_num)
435e401fa25SSergey Matyukevich i = 0;
436e401fa25SSergey Matyukevich }
437e401fa25SSergey Matyukevich
438e401fa25SSergey Matyukevich priv->tx_reclaim_done += count;
439e401fa25SSergey Matyukevich priv->tx_reclaim_req++;
440e401fa25SSergey Matyukevich priv->tx_bd_r_index = i;
441e401fa25SSergey Matyukevich
442e401fa25SSergey Matyukevich spin_unlock_irqrestore(&priv->tx_reclaim_lock, flags);
443e401fa25SSergey Matyukevich }
444e401fa25SSergey Matyukevich
qtnf_try_stop_xmit(struct qtnf_bus * bus,struct net_device * ndev)445e401fa25SSergey Matyukevich static void qtnf_try_stop_xmit(struct qtnf_bus *bus, struct net_device *ndev)
446e401fa25SSergey Matyukevich {
447e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);
448e401fa25SSergey Matyukevich
449e401fa25SSergey Matyukevich if (ndev) {
450e401fa25SSergey Matyukevich netif_tx_stop_all_queues(ndev);
451e401fa25SSergey Matyukevich ts->base.tx_stopped = 1;
452e401fa25SSergey Matyukevich }
453e401fa25SSergey Matyukevich
454e401fa25SSergey Matyukevich writel(0x0, ts->txqueue_wake);
455e401fa25SSergey Matyukevich
456e401fa25SSergey Matyukevich /* sync up tx queue status before generating interrupt */
457e401fa25SSergey Matyukevich dma_wmb();
458e401fa25SSergey Matyukevich
459e401fa25SSergey Matyukevich /* send irq to card: tx stopped */
460e401fa25SSergey Matyukevich writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_TX_STOP_IRQ),
461e401fa25SSergey Matyukevich TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));
462e401fa25SSergey Matyukevich
463e401fa25SSergey Matyukevich /* schedule reclaim attempt */
464e401fa25SSergey Matyukevich tasklet_hi_schedule(&ts->base.reclaim_tq);
465e401fa25SSergey Matyukevich }
466e401fa25SSergey Matyukevich
qtnf_try_wake_xmit(struct qtnf_bus * bus,struct net_device * ndev)467e401fa25SSergey Matyukevich static void qtnf_try_wake_xmit(struct qtnf_bus *bus, struct net_device *ndev)
468e401fa25SSergey Matyukevich {
469e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);
470e401fa25SSergey Matyukevich int ready;
471e401fa25SSergey Matyukevich
472e401fa25SSergey Matyukevich ready = readl(ts->txqueue_wake);
473e401fa25SSergey Matyukevich if (ready) {
474e401fa25SSergey Matyukevich netif_wake_queue(ndev);
475e401fa25SSergey Matyukevich } else {
476e401fa25SSergey Matyukevich /* re-send irq to card: tx stopped */
477e401fa25SSergey Matyukevich writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_TX_STOP_IRQ),
478e401fa25SSergey Matyukevich TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));
479e401fa25SSergey Matyukevich }
480e401fa25SSergey Matyukevich }
481e401fa25SSergey Matyukevich
qtnf_tx_queue_ready(struct qtnf_pcie_topaz_state * ts)482e401fa25SSergey Matyukevich static int qtnf_tx_queue_ready(struct qtnf_pcie_topaz_state *ts)
483e401fa25SSergey Matyukevich {
484e401fa25SSergey Matyukevich struct qtnf_pcie_bus_priv *priv = &ts->base;
485e401fa25SSergey Matyukevich
486e401fa25SSergey Matyukevich if (!CIRC_SPACE(priv->tx_bd_w_index, priv->tx_bd_r_index,
487e401fa25SSergey Matyukevich priv->tx_bd_num)) {
488e401fa25SSergey Matyukevich qtnf_topaz_data_tx_reclaim(ts);
489e401fa25SSergey Matyukevich
490e401fa25SSergey Matyukevich if (!CIRC_SPACE(priv->tx_bd_w_index, priv->tx_bd_r_index,
491e401fa25SSergey Matyukevich priv->tx_bd_num)) {
492e401fa25SSergey Matyukevich priv->tx_full_count++;
493e401fa25SSergey Matyukevich return 0;
494e401fa25SSergey Matyukevich }
495e401fa25SSergey Matyukevich }
496e401fa25SSergey Matyukevich
497e401fa25SSergey Matyukevich return 1;
498e401fa25SSergey Matyukevich }
499e401fa25SSergey Matyukevich
qtnf_pcie_data_tx(struct qtnf_bus * bus,struct sk_buff * skb,unsigned int macid,unsigned int vifid)500904628d3SIgor Mitsyanko static int qtnf_pcie_data_tx(struct qtnf_bus *bus, struct sk_buff *skb,
501904628d3SIgor Mitsyanko unsigned int macid, unsigned int vifid)
502e401fa25SSergey Matyukevich {
503e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);
504e401fa25SSergey Matyukevich struct qtnf_pcie_bus_priv *priv = &ts->base;
505e401fa25SSergey Matyukevich struct qtnf_topaz_bda __iomem *bda = ts->bda;
506e401fa25SSergey Matyukevich struct qtnf_topaz_tx_bd *txbd;
507e401fa25SSergey Matyukevich dma_addr_t skb_paddr;
508e401fa25SSergey Matyukevich unsigned long flags;
509e401fa25SSergey Matyukevich int ret = 0;
510e401fa25SSergey Matyukevich int len;
511e401fa25SSergey Matyukevich int i;
512e401fa25SSergey Matyukevich
513e401fa25SSergey Matyukevich spin_lock_irqsave(&priv->tx_lock, flags);
514e401fa25SSergey Matyukevich
515e401fa25SSergey Matyukevich if (!qtnf_tx_queue_ready(ts)) {
516e401fa25SSergey Matyukevich qtnf_try_stop_xmit(bus, skb->dev);
517e401fa25SSergey Matyukevich spin_unlock_irqrestore(&priv->tx_lock, flags);
518e401fa25SSergey Matyukevich return NETDEV_TX_BUSY;
519e401fa25SSergey Matyukevich }
520e401fa25SSergey Matyukevich
521e401fa25SSergey Matyukevich i = priv->tx_bd_w_index;
522e401fa25SSergey Matyukevich priv->tx_skb[i] = skb;
523e401fa25SSergey Matyukevich len = skb->len;
524e401fa25SSergey Matyukevich
52506e1359cSChristophe JAILLET skb_paddr = dma_map_single(&priv->pdev->dev, skb->data, skb->len,
52606e1359cSChristophe JAILLET DMA_TO_DEVICE);
52706e1359cSChristophe JAILLET if (dma_mapping_error(&priv->pdev->dev, skb_paddr)) {
528e401fa25SSergey Matyukevich ret = -ENOMEM;
529e401fa25SSergey Matyukevich goto tx_done;
530e401fa25SSergey Matyukevich }
531e401fa25SSergey Matyukevich
532e401fa25SSergey Matyukevich txbd = &ts->tx_bd_vbase[i];
533e401fa25SSergey Matyukevich txbd->addr = cpu_to_le32(QTN_HOST_LO32(skb_paddr));
534e401fa25SSergey Matyukevich
535e401fa25SSergey Matyukevich writel(QTN_HOST_LO32(skb_paddr), &bda->request[i].addr);
536e401fa25SSergey Matyukevich writel(len | QTN_PCIE_TX_VALID_PKT, &bda->request[i].info);
537e401fa25SSergey Matyukevich
538e401fa25SSergey Matyukevich /* sync up descriptor updates before generating interrupt */
539e401fa25SSergey Matyukevich dma_wmb();
540e401fa25SSergey Matyukevich
541e401fa25SSergey Matyukevich /* generate irq to card: tx done */
542e401fa25SSergey Matyukevich writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_TX_DONE_IRQ),
543e401fa25SSergey Matyukevich TOPAZ_LH_IPC4_INT(priv->sysctl_bar));
544e401fa25SSergey Matyukevich
545e401fa25SSergey Matyukevich if (++i >= priv->tx_bd_num)
546e401fa25SSergey Matyukevich i = 0;
547e401fa25SSergey Matyukevich
548e401fa25SSergey Matyukevich priv->tx_bd_w_index = i;
549e401fa25SSergey Matyukevich
550e401fa25SSergey Matyukevich tx_done:
551e401fa25SSergey Matyukevich if (ret) {
552e401fa25SSergey Matyukevich if (skb->dev)
553e401fa25SSergey Matyukevich skb->dev->stats.tx_dropped++;
554e401fa25SSergey Matyukevich dev_kfree_skb_any(skb);
555e401fa25SSergey Matyukevich }
556e401fa25SSergey Matyukevich
557e401fa25SSergey Matyukevich priv->tx_done_count++;
558e401fa25SSergey Matyukevich spin_unlock_irqrestore(&priv->tx_lock, flags);
559e401fa25SSergey Matyukevich
560e401fa25SSergey Matyukevich qtnf_topaz_data_tx_reclaim(ts);
561e401fa25SSergey Matyukevich
562e401fa25SSergey Matyukevich return NETDEV_TX_OK;
563e401fa25SSergey Matyukevich }
564e401fa25SSergey Matyukevich
qtnf_pcie_topaz_interrupt(int irq,void * data)565e401fa25SSergey Matyukevich static irqreturn_t qtnf_pcie_topaz_interrupt(int irq, void *data)
566e401fa25SSergey Matyukevich {
567e401fa25SSergey Matyukevich struct qtnf_bus *bus = (struct qtnf_bus *)data;
568e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);
569e401fa25SSergey Matyukevich struct qtnf_pcie_bus_priv *priv = &ts->base;
570e401fa25SSergey Matyukevich
571e401fa25SSergey Matyukevich if (!priv->msi_enabled && !qtnf_topaz_intx_asserted(ts))
572e401fa25SSergey Matyukevich return IRQ_NONE;
573e401fa25SSergey Matyukevich
574d1365e79SSergey Matyukevich if (!priv->msi_enabled)
575d1365e79SSergey Matyukevich qtnf_deassert_intx(ts);
576d1365e79SSergey Matyukevich
577e401fa25SSergey Matyukevich priv->pcie_irq_count++;
578e401fa25SSergey Matyukevich
579e401fa25SSergey Matyukevich qtnf_shm_ipc_irq_handler(&priv->shm_ipc_ep_in);
580e401fa25SSergey Matyukevich qtnf_shm_ipc_irq_handler(&priv->shm_ipc_ep_out);
581e401fa25SSergey Matyukevich
582e401fa25SSergey Matyukevich if (napi_schedule_prep(&bus->mux_napi)) {
583e401fa25SSergey Matyukevich disable_rx_irqs(ts);
584e401fa25SSergey Matyukevich __napi_schedule(&bus->mux_napi);
585e401fa25SSergey Matyukevich }
586e401fa25SSergey Matyukevich
587e401fa25SSergey Matyukevich tasklet_hi_schedule(&priv->reclaim_tq);
588e401fa25SSergey Matyukevich
589e401fa25SSergey Matyukevich return IRQ_HANDLED;
590e401fa25SSergey Matyukevich }
591e401fa25SSergey Matyukevich
qtnf_rx_data_ready(struct qtnf_pcie_topaz_state * ts)592e401fa25SSergey Matyukevich static int qtnf_rx_data_ready(struct qtnf_pcie_topaz_state *ts)
593e401fa25SSergey Matyukevich {
594e401fa25SSergey Matyukevich u16 index = ts->base.rx_bd_r_index;
595e401fa25SSergey Matyukevich struct qtnf_topaz_rx_bd *rxbd;
596e401fa25SSergey Matyukevich u32 descw;
597e401fa25SSergey Matyukevich
598e401fa25SSergey Matyukevich rxbd = &ts->rx_bd_vbase[index];
599e401fa25SSergey Matyukevich descw = le32_to_cpu(rxbd->info);
600e401fa25SSergey Matyukevich
601e401fa25SSergey Matyukevich if (descw & QTN_BD_EMPTY)
602e401fa25SSergey Matyukevich return 0;
603e401fa25SSergey Matyukevich
604e401fa25SSergey Matyukevich return 1;
605e401fa25SSergey Matyukevich }
606e401fa25SSergey Matyukevich
qtnf_topaz_rx_poll(struct napi_struct * napi,int budget)607e401fa25SSergey Matyukevich static int qtnf_topaz_rx_poll(struct napi_struct *napi, int budget)
608e401fa25SSergey Matyukevich {
609e401fa25SSergey Matyukevich struct qtnf_bus *bus = container_of(napi, struct qtnf_bus, mux_napi);
610e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);
611e401fa25SSergey Matyukevich struct qtnf_pcie_bus_priv *priv = &ts->base;
612e401fa25SSergey Matyukevich struct net_device *ndev = NULL;
613e401fa25SSergey Matyukevich struct sk_buff *skb = NULL;
614e401fa25SSergey Matyukevich int processed = 0;
615e401fa25SSergey Matyukevich struct qtnf_topaz_rx_bd *rxbd;
616e401fa25SSergey Matyukevich dma_addr_t skb_paddr;
617e401fa25SSergey Matyukevich int consume;
618e401fa25SSergey Matyukevich u32 descw;
619e401fa25SSergey Matyukevich u32 poffset;
620e401fa25SSergey Matyukevich u32 psize;
621e401fa25SSergey Matyukevich u16 r_idx;
622e401fa25SSergey Matyukevich u16 w_idx;
623e401fa25SSergey Matyukevich int ret;
624e401fa25SSergey Matyukevich
625e401fa25SSergey Matyukevich while (processed < budget) {
626e401fa25SSergey Matyukevich if (!qtnf_rx_data_ready(ts))
627e401fa25SSergey Matyukevich goto rx_out;
628e401fa25SSergey Matyukevich
629e401fa25SSergey Matyukevich r_idx = priv->rx_bd_r_index;
630e401fa25SSergey Matyukevich rxbd = &ts->rx_bd_vbase[r_idx];
631e401fa25SSergey Matyukevich descw = le32_to_cpu(rxbd->info);
632e401fa25SSergey Matyukevich
633e401fa25SSergey Matyukevich skb = priv->rx_skb[r_idx];
634e401fa25SSergey Matyukevich poffset = QTN_GET_OFFSET(descw);
635e401fa25SSergey Matyukevich psize = QTN_GET_LEN(descw);
636e401fa25SSergey Matyukevich consume = 1;
637e401fa25SSergey Matyukevich
638e401fa25SSergey Matyukevich if (descw & QTN_BD_EMPTY) {
639e401fa25SSergey Matyukevich pr_warn("skip invalid rxbd[%d]\n", r_idx);
640e401fa25SSergey Matyukevich consume = 0;
641e401fa25SSergey Matyukevich }
642e401fa25SSergey Matyukevich
643e401fa25SSergey Matyukevich if (!skb) {
644e401fa25SSergey Matyukevich pr_warn("skip missing rx_skb[%d]\n", r_idx);
645e401fa25SSergey Matyukevich consume = 0;
646e401fa25SSergey Matyukevich }
647e401fa25SSergey Matyukevich
648e401fa25SSergey Matyukevich if (skb && (skb_tailroom(skb) < psize)) {
649e401fa25SSergey Matyukevich pr_err("skip packet with invalid length: %u > %u\n",
650e401fa25SSergey Matyukevich psize, skb_tailroom(skb));
651e401fa25SSergey Matyukevich consume = 0;
652e401fa25SSergey Matyukevich }
653e401fa25SSergey Matyukevich
654e401fa25SSergey Matyukevich if (skb) {
655e401fa25SSergey Matyukevich skb_paddr = QTN_HOST_ADDR(0x0, le32_to_cpu(rxbd->addr));
65606e1359cSChristophe JAILLET dma_unmap_single(&priv->pdev->dev, skb_paddr,
65706e1359cSChristophe JAILLET SKB_BUF_SIZE, DMA_FROM_DEVICE);
658e401fa25SSergey Matyukevich }
659e401fa25SSergey Matyukevich
660e401fa25SSergey Matyukevich if (consume) {
661e401fa25SSergey Matyukevich skb_reserve(skb, poffset);
662e401fa25SSergey Matyukevich skb_put(skb, psize);
663e401fa25SSergey Matyukevich ndev = qtnf_classify_skb(bus, skb);
664e401fa25SSergey Matyukevich if (likely(ndev)) {
66545fc3fd4SHeiner Kallweit dev_sw_netstats_rx_add(ndev, skb->len);
666e401fa25SSergey Matyukevich skb->protocol = eth_type_trans(skb, ndev);
667e401fa25SSergey Matyukevich netif_receive_skb(skb);
668e401fa25SSergey Matyukevich } else {
669e401fa25SSergey Matyukevich pr_debug("drop untagged skb\n");
670e401fa25SSergey Matyukevich bus->mux_dev.stats.rx_dropped++;
671e401fa25SSergey Matyukevich dev_kfree_skb_any(skb);
672e401fa25SSergey Matyukevich }
673e401fa25SSergey Matyukevich } else {
674e401fa25SSergey Matyukevich if (skb) {
675e401fa25SSergey Matyukevich bus->mux_dev.stats.rx_dropped++;
676e401fa25SSergey Matyukevich dev_kfree_skb_any(skb);
677e401fa25SSergey Matyukevich }
678e401fa25SSergey Matyukevich }
679e401fa25SSergey Matyukevich
680e401fa25SSergey Matyukevich /* notify card about recv packets once per several packets */
681e401fa25SSergey Matyukevich if (((++ts->rx_pkt_count) & RX_DONE_INTR_MSK) == 0)
682e401fa25SSergey Matyukevich writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_RX_DONE_IRQ),
683e401fa25SSergey Matyukevich TOPAZ_LH_IPC4_INT(priv->sysctl_bar));
684e401fa25SSergey Matyukevich
685e401fa25SSergey Matyukevich priv->rx_skb[r_idx] = NULL;
686e401fa25SSergey Matyukevich if (++r_idx >= priv->rx_bd_num)
687e401fa25SSergey Matyukevich r_idx = 0;
688e401fa25SSergey Matyukevich
689e401fa25SSergey Matyukevich priv->rx_bd_r_index = r_idx;
690e401fa25SSergey Matyukevich
691e401fa25SSergey Matyukevich /* repalce processed buffer by a new one */
692e401fa25SSergey Matyukevich w_idx = priv->rx_bd_w_index;
693e401fa25SSergey Matyukevich while (CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index,
694e401fa25SSergey Matyukevich priv->rx_bd_num) > 0) {
695e401fa25SSergey Matyukevich if (++w_idx >= priv->rx_bd_num)
696e401fa25SSergey Matyukevich w_idx = 0;
697e401fa25SSergey Matyukevich
698e401fa25SSergey Matyukevich ret = topaz_skb2rbd_attach(ts, w_idx,
699e401fa25SSergey Matyukevich descw & QTN_BD_WRAP);
700e401fa25SSergey Matyukevich if (ret) {
701e401fa25SSergey Matyukevich pr_err("failed to allocate new rx_skb[%d]\n",
702e401fa25SSergey Matyukevich w_idx);
703e401fa25SSergey Matyukevich break;
704e401fa25SSergey Matyukevich }
705e401fa25SSergey Matyukevich }
706e401fa25SSergey Matyukevich
707e401fa25SSergey Matyukevich processed++;
708e401fa25SSergey Matyukevich }
709e401fa25SSergey Matyukevich
710e401fa25SSergey Matyukevich rx_out:
711e401fa25SSergey Matyukevich if (processed < budget) {
712e401fa25SSergey Matyukevich napi_complete(napi);
713e401fa25SSergey Matyukevich enable_rx_irqs(ts);
714e401fa25SSergey Matyukevich }
715e401fa25SSergey Matyukevich
716e401fa25SSergey Matyukevich return processed;
717e401fa25SSergey Matyukevich }
718e401fa25SSergey Matyukevich
719e401fa25SSergey Matyukevich static void
qtnf_pcie_data_tx_timeout(struct qtnf_bus * bus,struct net_device * ndev)720e401fa25SSergey Matyukevich qtnf_pcie_data_tx_timeout(struct qtnf_bus *bus, struct net_device *ndev)
721e401fa25SSergey Matyukevich {
722e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);
723e401fa25SSergey Matyukevich
724e401fa25SSergey Matyukevich qtnf_try_wake_xmit(bus, ndev);
725e401fa25SSergey Matyukevich tasklet_hi_schedule(&ts->base.reclaim_tq);
726e401fa25SSergey Matyukevich }
727e401fa25SSergey Matyukevich
qtnf_pcie_data_rx_start(struct qtnf_bus * bus)728e401fa25SSergey Matyukevich static void qtnf_pcie_data_rx_start(struct qtnf_bus *bus)
729e401fa25SSergey Matyukevich {
730e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);
731e401fa25SSergey Matyukevich
732e401fa25SSergey Matyukevich napi_enable(&bus->mux_napi);
733e401fa25SSergey Matyukevich enable_rx_irqs(ts);
734e401fa25SSergey Matyukevich }
735e401fa25SSergey Matyukevich
qtnf_pcie_data_rx_stop(struct qtnf_bus * bus)736e401fa25SSergey Matyukevich static void qtnf_pcie_data_rx_stop(struct qtnf_bus *bus)
737e401fa25SSergey Matyukevich {
738e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);
739e401fa25SSergey Matyukevich
740e401fa25SSergey Matyukevich disable_rx_irqs(ts);
741e401fa25SSergey Matyukevich napi_disable(&bus->mux_napi);
742e401fa25SSergey Matyukevich }
743e401fa25SSergey Matyukevich
744904628d3SIgor Mitsyanko static struct qtnf_bus_ops qtnf_pcie_topaz_bus_ops = {
745e401fa25SSergey Matyukevich /* control path methods */
746e401fa25SSergey Matyukevich .control_tx = qtnf_pcie_control_tx,
747e401fa25SSergey Matyukevich
748e401fa25SSergey Matyukevich /* data path methods */
749e401fa25SSergey Matyukevich .data_tx = qtnf_pcie_data_tx,
750e401fa25SSergey Matyukevich .data_tx_timeout = qtnf_pcie_data_tx_timeout,
751e401fa25SSergey Matyukevich .data_rx_start = qtnf_pcie_data_rx_start,
752e401fa25SSergey Matyukevich .data_rx_stop = qtnf_pcie_data_rx_stop,
753e401fa25SSergey Matyukevich };
754e401fa25SSergey Matyukevich
qtnf_dbg_irq_stats(struct seq_file * s,void * data)755e401fa25SSergey Matyukevich static int qtnf_dbg_irq_stats(struct seq_file *s, void *data)
756e401fa25SSergey Matyukevich {
757e401fa25SSergey Matyukevich struct qtnf_bus *bus = dev_get_drvdata(s->private);
758e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);
759e401fa25SSergey Matyukevich
760e401fa25SSergey Matyukevich seq_printf(s, "pcie_irq_count(%u)\n", ts->base.pcie_irq_count);
761e401fa25SSergey Matyukevich
762e401fa25SSergey Matyukevich return 0;
763e401fa25SSergey Matyukevich }
764e401fa25SSergey Matyukevich
qtnf_dbg_pkt_stats(struct seq_file * s,void * data)765e401fa25SSergey Matyukevich static int qtnf_dbg_pkt_stats(struct seq_file *s, void *data)
766e401fa25SSergey Matyukevich {
767e401fa25SSergey Matyukevich struct qtnf_bus *bus = dev_get_drvdata(s->private);
768e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);
769e401fa25SSergey Matyukevich struct qtnf_pcie_bus_priv *priv = &ts->base;
770e401fa25SSergey Matyukevich u32 tx_done_index = readl(ts->ep_next_rx_pkt);
771e401fa25SSergey Matyukevich
772e401fa25SSergey Matyukevich seq_printf(s, "tx_full_count(%u)\n", priv->tx_full_count);
773e401fa25SSergey Matyukevich seq_printf(s, "tx_done_count(%u)\n", priv->tx_done_count);
774e401fa25SSergey Matyukevich seq_printf(s, "tx_reclaim_done(%u)\n", priv->tx_reclaim_done);
775e401fa25SSergey Matyukevich seq_printf(s, "tx_reclaim_req(%u)\n", priv->tx_reclaim_req);
776e401fa25SSergey Matyukevich
777e401fa25SSergey Matyukevich seq_printf(s, "tx_bd_r_index(%u)\n", priv->tx_bd_r_index);
778e401fa25SSergey Matyukevich seq_printf(s, "tx_done_index(%u)\n", tx_done_index);
779e401fa25SSergey Matyukevich seq_printf(s, "tx_bd_w_index(%u)\n", priv->tx_bd_w_index);
780e401fa25SSergey Matyukevich
781e401fa25SSergey Matyukevich seq_printf(s, "tx host queue len(%u)\n",
782e401fa25SSergey Matyukevich CIRC_CNT(priv->tx_bd_w_index, priv->tx_bd_r_index,
783e401fa25SSergey Matyukevich priv->tx_bd_num));
784e401fa25SSergey Matyukevich seq_printf(s, "tx reclaim queue len(%u)\n",
785e401fa25SSergey Matyukevich CIRC_CNT(tx_done_index, priv->tx_bd_r_index,
786e401fa25SSergey Matyukevich priv->tx_bd_num));
787e401fa25SSergey Matyukevich seq_printf(s, "tx card queue len(%u)\n",
788e401fa25SSergey Matyukevich CIRC_CNT(priv->tx_bd_w_index, tx_done_index,
789e401fa25SSergey Matyukevich priv->tx_bd_num));
790e401fa25SSergey Matyukevich
791e401fa25SSergey Matyukevich seq_printf(s, "rx_bd_r_index(%u)\n", priv->rx_bd_r_index);
792e401fa25SSergey Matyukevich seq_printf(s, "rx_bd_w_index(%u)\n", priv->rx_bd_w_index);
793e401fa25SSergey Matyukevich seq_printf(s, "rx alloc queue len(%u)\n",
794e401fa25SSergey Matyukevich CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index,
795e401fa25SSergey Matyukevich priv->rx_bd_num));
796e401fa25SSergey Matyukevich
797e401fa25SSergey Matyukevich return 0;
798e401fa25SSergey Matyukevich }
799e401fa25SSergey Matyukevich
qtnf_reset_dma_offset(struct qtnf_pcie_topaz_state * ts)800e401fa25SSergey Matyukevich static void qtnf_reset_dma_offset(struct qtnf_pcie_topaz_state *ts)
801e401fa25SSergey Matyukevich {
802e401fa25SSergey Matyukevich struct qtnf_topaz_bda __iomem *bda = ts->bda;
803e401fa25SSergey Matyukevich u32 offset = readl(&bda->bda_dma_offset);
804e401fa25SSergey Matyukevich
805e401fa25SSergey Matyukevich if ((offset & PCIE_DMA_OFFSET_ERROR_MASK) != PCIE_DMA_OFFSET_ERROR)
806e401fa25SSergey Matyukevich return;
807e401fa25SSergey Matyukevich
808e401fa25SSergey Matyukevich writel(0x0, &bda->bda_dma_offset);
809e401fa25SSergey Matyukevich }
810e401fa25SSergey Matyukevich
qtnf_pcie_endian_detect(struct qtnf_pcie_topaz_state * ts)811e401fa25SSergey Matyukevich static int qtnf_pcie_endian_detect(struct qtnf_pcie_topaz_state *ts)
812e401fa25SSergey Matyukevich {
813e401fa25SSergey Matyukevich struct qtnf_topaz_bda __iomem *bda = ts->bda;
814e401fa25SSergey Matyukevich u32 timeout = 0;
815e401fa25SSergey Matyukevich u32 endian;
816e401fa25SSergey Matyukevich int ret = 0;
817e401fa25SSergey Matyukevich
818e401fa25SSergey Matyukevich writel(QTN_PCI_ENDIAN_DETECT_DATA, &bda->bda_pci_endian);
819e401fa25SSergey Matyukevich
820e401fa25SSergey Matyukevich /* flush endian modifications before status update */
821e401fa25SSergey Matyukevich dma_wmb();
822e401fa25SSergey Matyukevich
823e401fa25SSergey Matyukevich writel(QTN_PCI_ENDIAN_VALID_STATUS, &bda->bda_pci_pre_status);
824e401fa25SSergey Matyukevich
825e401fa25SSergey Matyukevich while (readl(&bda->bda_pci_post_status) !=
826e401fa25SSergey Matyukevich QTN_PCI_ENDIAN_VALID_STATUS) {
827e401fa25SSergey Matyukevich usleep_range(1000, 1200);
828e401fa25SSergey Matyukevich if (++timeout > QTN_FW_DL_TIMEOUT_MS) {
829e401fa25SSergey Matyukevich pr_err("card endianness detection timed out\n");
830e401fa25SSergey Matyukevich ret = -ETIMEDOUT;
831e401fa25SSergey Matyukevich goto endian_out;
832e401fa25SSergey Matyukevich }
833e401fa25SSergey Matyukevich }
834e401fa25SSergey Matyukevich
835e401fa25SSergey Matyukevich /* do not read before status is updated */
836e401fa25SSergey Matyukevich dma_rmb();
837e401fa25SSergey Matyukevich
838e401fa25SSergey Matyukevich endian = readl(&bda->bda_pci_endian);
839e401fa25SSergey Matyukevich WARN(endian != QTN_PCI_LITTLE_ENDIAN,
840e401fa25SSergey Matyukevich "%s: unexpected card endianness", __func__);
841e401fa25SSergey Matyukevich
842e401fa25SSergey Matyukevich endian_out:
843e401fa25SSergey Matyukevich writel(0, &bda->bda_pci_pre_status);
844e401fa25SSergey Matyukevich writel(0, &bda->bda_pci_post_status);
845e401fa25SSergey Matyukevich writel(0, &bda->bda_pci_endian);
846e401fa25SSergey Matyukevich
847e401fa25SSergey Matyukevich return ret;
848e401fa25SSergey Matyukevich }
849e401fa25SSergey Matyukevich
qtnf_pre_init_ep(struct qtnf_bus * bus)850e401fa25SSergey Matyukevich static int qtnf_pre_init_ep(struct qtnf_bus *bus)
851e401fa25SSergey Matyukevich {
852e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);
853e401fa25SSergey Matyukevich struct qtnf_topaz_bda __iomem *bda = ts->bda;
854e401fa25SSergey Matyukevich u32 flags;
855e401fa25SSergey Matyukevich int ret;
856e401fa25SSergey Matyukevich
857e401fa25SSergey Matyukevich ret = qtnf_pcie_endian_detect(ts);
858e401fa25SSergey Matyukevich if (ret < 0) {
859e401fa25SSergey Matyukevich pr_err("failed to detect card endianness\n");
860e401fa25SSergey Matyukevich return ret;
861e401fa25SSergey Matyukevich }
862e401fa25SSergey Matyukevich
863e401fa25SSergey Matyukevich writeb(ts->base.msi_enabled, &ts->bda->bda_rc_msi_enabled);
864e401fa25SSergey Matyukevich qtnf_reset_dma_offset(ts);
865e401fa25SSergey Matyukevich
866e401fa25SSergey Matyukevich /* notify card about driver type and boot mode */
867e401fa25SSergey Matyukevich flags = readl(&bda->bda_flags) | QTN_BDA_HOST_QLINK_DRV;
868e401fa25SSergey Matyukevich
869e401fa25SSergey Matyukevich if (ts->base.flashboot)
870e401fa25SSergey Matyukevich flags |= QTN_BDA_FLASH_BOOT;
871e401fa25SSergey Matyukevich else
872e401fa25SSergey Matyukevich flags &= ~QTN_BDA_FLASH_BOOT;
873e401fa25SSergey Matyukevich
874e401fa25SSergey Matyukevich writel(flags, &bda->bda_flags);
875e401fa25SSergey Matyukevich
876e401fa25SSergey Matyukevich qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_HOST_RDY);
877e401fa25SSergey Matyukevich if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_TARGET_RDY,
878e401fa25SSergey Matyukevich QTN_FW_DL_TIMEOUT_MS)) {
879e401fa25SSergey Matyukevich pr_err("card is not ready to boot...\n");
880e401fa25SSergey Matyukevich return -ETIMEDOUT;
881e401fa25SSergey Matyukevich }
882e401fa25SSergey Matyukevich
883e401fa25SSergey Matyukevich return ret;
884e401fa25SSergey Matyukevich }
885e401fa25SSergey Matyukevich
qtnf_post_init_ep(struct qtnf_pcie_topaz_state * ts)886e401fa25SSergey Matyukevich static int qtnf_post_init_ep(struct qtnf_pcie_topaz_state *ts)
887e401fa25SSergey Matyukevich {
888e401fa25SSergey Matyukevich struct pci_dev *pdev = ts->base.pdev;
889e401fa25SSergey Matyukevich
890e401fa25SSergey Matyukevich setup_rx_irqs(ts);
891e401fa25SSergey Matyukevich disable_rx_irqs(ts);
892e401fa25SSergey Matyukevich
893e401fa25SSergey Matyukevich if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_QLINK_DONE,
894e401fa25SSergey Matyukevich QTN_FW_QLINK_TIMEOUT_MS))
895e401fa25SSergey Matyukevich return -ETIMEDOUT;
896e401fa25SSergey Matyukevich
897e401fa25SSergey Matyukevich enable_irq(pdev->irq);
898e401fa25SSergey Matyukevich return 0;
899e401fa25SSergey Matyukevich }
900e401fa25SSergey Matyukevich
901e401fa25SSergey Matyukevich static int
qtnf_ep_fw_load(struct qtnf_pcie_topaz_state * ts,const u8 * fw,u32 fw_size)902e401fa25SSergey Matyukevich qtnf_ep_fw_load(struct qtnf_pcie_topaz_state *ts, const u8 *fw, u32 fw_size)
903e401fa25SSergey Matyukevich {
904e401fa25SSergey Matyukevich struct qtnf_topaz_bda __iomem *bda = ts->bda;
905e401fa25SSergey Matyukevich struct pci_dev *pdev = ts->base.pdev;
906e401fa25SSergey Matyukevich u32 remaining = fw_size;
907e401fa25SSergey Matyukevich u8 *curr = (u8 *)fw;
908e401fa25SSergey Matyukevich u32 blksize;
909e401fa25SSergey Matyukevich u32 nblocks;
910e401fa25SSergey Matyukevich u32 offset;
911e401fa25SSergey Matyukevich u32 count;
912e401fa25SSergey Matyukevich u32 size;
913e401fa25SSergey Matyukevich dma_addr_t paddr;
914e401fa25SSergey Matyukevich void *data;
915e401fa25SSergey Matyukevich int ret = 0;
916e401fa25SSergey Matyukevich
917e401fa25SSergey Matyukevich pr_debug("FW upload started: fw_addr = 0x%p, size=%d\n", fw, fw_size);
918e401fa25SSergey Matyukevich
919e401fa25SSergey Matyukevich blksize = ts->base.fw_blksize;
920e401fa25SSergey Matyukevich
921e401fa25SSergey Matyukevich if (blksize < PAGE_SIZE)
922e401fa25SSergey Matyukevich blksize = PAGE_SIZE;
923e401fa25SSergey Matyukevich
924e401fa25SSergey Matyukevich while (blksize >= PAGE_SIZE) {
925e401fa25SSergey Matyukevich pr_debug("allocating %u bytes to upload FW\n", blksize);
926e401fa25SSergey Matyukevich data = dma_alloc_coherent(&pdev->dev, blksize,
927e401fa25SSergey Matyukevich &paddr, GFP_KERNEL);
928e401fa25SSergey Matyukevich if (data)
929e401fa25SSergey Matyukevich break;
930e401fa25SSergey Matyukevich blksize /= 2;
931e401fa25SSergey Matyukevich }
932e401fa25SSergey Matyukevich
933e401fa25SSergey Matyukevich if (!data) {
934e401fa25SSergey Matyukevich pr_err("failed to allocate DMA buffer for FW upload\n");
935e401fa25SSergey Matyukevich ret = -ENOMEM;
936e401fa25SSergey Matyukevich goto fw_load_out;
937e401fa25SSergey Matyukevich }
938e401fa25SSergey Matyukevich
939e401fa25SSergey Matyukevich nblocks = NBLOCKS(fw_size, blksize);
940e401fa25SSergey Matyukevich offset = readl(&bda->bda_dma_offset);
941e401fa25SSergey Matyukevich
942e401fa25SSergey Matyukevich qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_HOST_LOAD);
943e401fa25SSergey Matyukevich if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_EP_RDY,
944e401fa25SSergey Matyukevich QTN_FW_DL_TIMEOUT_MS)) {
945e401fa25SSergey Matyukevich pr_err("card is not ready to download FW\n");
946e401fa25SSergey Matyukevich ret = -ETIMEDOUT;
947e401fa25SSergey Matyukevich goto fw_load_map;
948e401fa25SSergey Matyukevich }
949e401fa25SSergey Matyukevich
950e401fa25SSergey Matyukevich for (count = 0 ; count < nblocks; count++) {
951e401fa25SSergey Matyukevich size = (remaining > blksize) ? blksize : remaining;
952e401fa25SSergey Matyukevich
953e401fa25SSergey Matyukevich memcpy(data, curr, size);
954e401fa25SSergey Matyukevich qtnf_non_posted_write(paddr + offset, &bda->bda_img);
955e401fa25SSergey Matyukevich qtnf_non_posted_write(size, &bda->bda_img_size);
956e401fa25SSergey Matyukevich
957e401fa25SSergey Matyukevich pr_debug("chunk[%u] VA[0x%p] PA[%pad] sz[%u]\n",
958e401fa25SSergey Matyukevich count, (void *)curr, &paddr, size);
959e401fa25SSergey Matyukevich
960e401fa25SSergey Matyukevich qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_RDY);
961e401fa25SSergey Matyukevich if (qtnf_poll_state(&ts->bda->bda_bootstate,
962e401fa25SSergey Matyukevich QTN_BDA_FW_BLOCK_DONE,
963e401fa25SSergey Matyukevich QTN_FW_DL_TIMEOUT_MS)) {
964e401fa25SSergey Matyukevich pr_err("confirmation for block #%d timed out\n", count);
965e401fa25SSergey Matyukevich ret = -ETIMEDOUT;
966e401fa25SSergey Matyukevich goto fw_load_map;
967e401fa25SSergey Matyukevich }
968e401fa25SSergey Matyukevich
969e401fa25SSergey Matyukevich remaining = (remaining < size) ? remaining : (remaining - size);
970e401fa25SSergey Matyukevich curr += size;
971e401fa25SSergey Matyukevich }
972e401fa25SSergey Matyukevich
973e401fa25SSergey Matyukevich /* upload completion mark: zero-sized block */
974e401fa25SSergey Matyukevich qtnf_non_posted_write(0, &bda->bda_img);
975e401fa25SSergey Matyukevich qtnf_non_posted_write(0, &bda->bda_img_size);
976e401fa25SSergey Matyukevich
977e401fa25SSergey Matyukevich qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_RDY);
978e401fa25SSergey Matyukevich if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_DONE,
979e401fa25SSergey Matyukevich QTN_FW_DL_TIMEOUT_MS)) {
980e401fa25SSergey Matyukevich pr_err("confirmation for the last block timed out\n");
981e401fa25SSergey Matyukevich ret = -ETIMEDOUT;
982e401fa25SSergey Matyukevich goto fw_load_map;
983e401fa25SSergey Matyukevich }
984e401fa25SSergey Matyukevich
985e401fa25SSergey Matyukevich /* RC is done */
986e401fa25SSergey Matyukevich qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_END);
987e401fa25SSergey Matyukevich if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_LOAD_DONE,
988e401fa25SSergey Matyukevich QTN_FW_DL_TIMEOUT_MS)) {
989e401fa25SSergey Matyukevich pr_err("confirmation for FW upload completion timed out\n");
990e401fa25SSergey Matyukevich ret = -ETIMEDOUT;
991e401fa25SSergey Matyukevich goto fw_load_map;
992e401fa25SSergey Matyukevich }
993e401fa25SSergey Matyukevich
994e401fa25SSergey Matyukevich pr_debug("FW upload completed: totally sent %d blocks\n", count);
995e401fa25SSergey Matyukevich
996e401fa25SSergey Matyukevich fw_load_map:
997e401fa25SSergey Matyukevich dma_free_coherent(&pdev->dev, blksize, data, paddr);
998e401fa25SSergey Matyukevich
999e401fa25SSergey Matyukevich fw_load_out:
1000e401fa25SSergey Matyukevich return ret;
1001e401fa25SSergey Matyukevich }
1002e401fa25SSergey Matyukevich
qtnf_topaz_fw_upload(struct qtnf_pcie_topaz_state * ts,const char * fwname)1003e401fa25SSergey Matyukevich static int qtnf_topaz_fw_upload(struct qtnf_pcie_topaz_state *ts,
1004e401fa25SSergey Matyukevich const char *fwname)
1005e401fa25SSergey Matyukevich {
1006e401fa25SSergey Matyukevich const struct firmware *fw;
1007e401fa25SSergey Matyukevich struct pci_dev *pdev = ts->base.pdev;
1008e401fa25SSergey Matyukevich int ret;
1009e401fa25SSergey Matyukevich
1010e401fa25SSergey Matyukevich if (qtnf_poll_state(&ts->bda->bda_bootstate,
1011e401fa25SSergey Matyukevich QTN_BDA_FW_LOAD_RDY,
1012e401fa25SSergey Matyukevich QTN_FW_DL_TIMEOUT_MS)) {
1013e401fa25SSergey Matyukevich pr_err("%s: card is not ready\n", fwname);
1014e401fa25SSergey Matyukevich return -1;
1015e401fa25SSergey Matyukevich }
1016e401fa25SSergey Matyukevich
1017e401fa25SSergey Matyukevich pr_info("starting firmware upload: %s\n", fwname);
1018e401fa25SSergey Matyukevich
1019e401fa25SSergey Matyukevich ret = request_firmware(&fw, fwname, &pdev->dev);
1020e401fa25SSergey Matyukevich if (ret < 0) {
1021e401fa25SSergey Matyukevich pr_err("%s: request_firmware error %d\n", fwname, ret);
1022e401fa25SSergey Matyukevich return -1;
1023e401fa25SSergey Matyukevich }
1024e401fa25SSergey Matyukevich
1025e401fa25SSergey Matyukevich ret = qtnf_ep_fw_load(ts, fw->data, fw->size);
1026e401fa25SSergey Matyukevich release_firmware(fw);
1027e401fa25SSergey Matyukevich
1028e401fa25SSergey Matyukevich if (ret)
1029e401fa25SSergey Matyukevich pr_err("%s: FW upload error\n", fwname);
1030e401fa25SSergey Matyukevich
1031e401fa25SSergey Matyukevich return ret;
1032e401fa25SSergey Matyukevich }
1033e401fa25SSergey Matyukevich
qtnf_topaz_fw_work_handler(struct work_struct * work)1034e401fa25SSergey Matyukevich static void qtnf_topaz_fw_work_handler(struct work_struct *work)
1035e401fa25SSergey Matyukevich {
1036e401fa25SSergey Matyukevich struct qtnf_bus *bus = container_of(work, struct qtnf_bus, fw_work);
1037e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);
1038e401fa25SSergey Matyukevich int bootloader_needed = readl(&ts->bda->bda_flags) & QTN_BDA_XMIT_UBOOT;
1039ae1946beSSergey Matyukevich struct pci_dev *pdev = ts->base.pdev;
1040ae1946beSSergey Matyukevich int ret;
1041e401fa25SSergey Matyukevich
1042e401fa25SSergey Matyukevich qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_TARGET_BOOT);
1043e401fa25SSergey Matyukevich
1044e401fa25SSergey Matyukevich if (bootloader_needed) {
1045e401fa25SSergey Matyukevich ret = qtnf_topaz_fw_upload(ts, QTN_PCI_TOPAZ_BOOTLD_NAME);
1046e401fa25SSergey Matyukevich if (ret)
1047e401fa25SSergey Matyukevich goto fw_load_exit;
1048e401fa25SSergey Matyukevich
1049e401fa25SSergey Matyukevich ret = qtnf_pre_init_ep(bus);
1050e401fa25SSergey Matyukevich if (ret)
1051e401fa25SSergey Matyukevich goto fw_load_exit;
1052e401fa25SSergey Matyukevich
1053e401fa25SSergey Matyukevich qtnf_set_state(&ts->bda->bda_bootstate,
1054e401fa25SSergey Matyukevich QTN_BDA_FW_TARGET_BOOT);
1055e401fa25SSergey Matyukevich }
1056e401fa25SSergey Matyukevich
1057e401fa25SSergey Matyukevich if (ts->base.flashboot) {
1058e401fa25SSergey Matyukevich pr_info("booting firmware from flash\n");
1059e401fa25SSergey Matyukevich
1060e401fa25SSergey Matyukevich ret = qtnf_poll_state(&ts->bda->bda_bootstate,
1061e401fa25SSergey Matyukevich QTN_BDA_FW_FLASH_BOOT,
1062e401fa25SSergey Matyukevich QTN_FW_DL_TIMEOUT_MS);
1063e401fa25SSergey Matyukevich if (ret)
1064e401fa25SSergey Matyukevich goto fw_load_exit;
1065e401fa25SSergey Matyukevich } else {
1066e401fa25SSergey Matyukevich ret = qtnf_topaz_fw_upload(ts, QTN_PCI_TOPAZ_FW_NAME);
1067e401fa25SSergey Matyukevich if (ret)
1068e401fa25SSergey Matyukevich goto fw_load_exit;
1069e401fa25SSergey Matyukevich
1070e401fa25SSergey Matyukevich qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_START);
1071e401fa25SSergey Matyukevich ret = qtnf_poll_state(&ts->bda->bda_bootstate,
1072e401fa25SSergey Matyukevich QTN_BDA_FW_CONFIG,
1073e401fa25SSergey Matyukevich QTN_FW_QLINK_TIMEOUT_MS);
1074e401fa25SSergey Matyukevich if (ret) {
1075e401fa25SSergey Matyukevich pr_err("FW bringup timed out\n");
1076e401fa25SSergey Matyukevich goto fw_load_exit;
1077e401fa25SSergey Matyukevich }
1078e401fa25SSergey Matyukevich
1079e401fa25SSergey Matyukevich qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_RUN);
1080e401fa25SSergey Matyukevich ret = qtnf_poll_state(&ts->bda->bda_bootstate,
1081e401fa25SSergey Matyukevich QTN_BDA_FW_RUNNING,
1082e401fa25SSergey Matyukevich QTN_FW_QLINK_TIMEOUT_MS);
1083e401fa25SSergey Matyukevich if (ret) {
1084e401fa25SSergey Matyukevich pr_err("card bringup timed out\n");
1085e401fa25SSergey Matyukevich goto fw_load_exit;
1086e401fa25SSergey Matyukevich }
1087e401fa25SSergey Matyukevich }
1088e401fa25SSergey Matyukevich
1089ae1946beSSergey Matyukevich ret = qtnf_post_init_ep(ts);
1090ae1946beSSergey Matyukevich if (ret) {
1091ae1946beSSergey Matyukevich pr_err("FW runtime failure\n");
1092ae1946beSSergey Matyukevich goto fw_load_exit;
1093ae1946beSSergey Matyukevich }
1094ae1946beSSergey Matyukevich
1095e401fa25SSergey Matyukevich pr_info("firmware is up and running\n");
1096e401fa25SSergey Matyukevich
1097ae1946beSSergey Matyukevich ret = qtnf_pcie_fw_boot_done(bus);
1098e401fa25SSergey Matyukevich if (ret)
1099ae1946beSSergey Matyukevich goto fw_load_exit;
1100e401fa25SSergey Matyukevich
1101e401fa25SSergey Matyukevich qtnf_debugfs_add_entry(bus, "pkt_stats", qtnf_dbg_pkt_stats);
1102e401fa25SSergey Matyukevich qtnf_debugfs_add_entry(bus, "irq_stats", qtnf_dbg_irq_stats);
1103ae1946beSSergey Matyukevich
1104ae1946beSSergey Matyukevich fw_load_exit:
1105ae1946beSSergey Matyukevich put_device(&pdev->dev);
1106e401fa25SSergey Matyukevich }
1107e401fa25SSergey Matyukevich
qtnf_reclaim_tasklet_fn(struct tasklet_struct * t)1108aff8e8d0SAllen Pais static void qtnf_reclaim_tasklet_fn(struct tasklet_struct *t)
1109e401fa25SSergey Matyukevich {
1110aff8e8d0SAllen Pais struct qtnf_pcie_topaz_state *ts = from_tasklet(ts, t, base.reclaim_tq);
1111e401fa25SSergey Matyukevich
1112e401fa25SSergey Matyukevich qtnf_topaz_data_tx_reclaim(ts);
1113e401fa25SSergey Matyukevich }
1114e401fa25SSergey Matyukevich
qtnf_topaz_dma_mask_get(void)1115e401fa25SSergey Matyukevich static u64 qtnf_topaz_dma_mask_get(void)
1116e401fa25SSergey Matyukevich {
1117e401fa25SSergey Matyukevich return DMA_BIT_MASK(32);
1118e401fa25SSergey Matyukevich }
1119e401fa25SSergey Matyukevich
qtnf_pcie_topaz_probe(struct qtnf_bus * bus,unsigned int tx_bd_num,unsigned int rx_bd_num)112097aef03cSSergey Matyukevich static int qtnf_pcie_topaz_probe(struct qtnf_bus *bus,
112197aef03cSSergey Matyukevich unsigned int tx_bd_num, unsigned int rx_bd_num)
1122e401fa25SSergey Matyukevich {
1123e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);
1124e401fa25SSergey Matyukevich struct pci_dev *pdev = ts->base.pdev;
1125e401fa25SSergey Matyukevich struct qtnf_shm_ipc_int ipc_int;
1126e401fa25SSergey Matyukevich unsigned long irqflags;
1127e401fa25SSergey Matyukevich int ret;
1128e401fa25SSergey Matyukevich
1129e401fa25SSergey Matyukevich bus->bus_ops = &qtnf_pcie_topaz_bus_ops;
1130e401fa25SSergey Matyukevich INIT_WORK(&bus->fw_work, qtnf_topaz_fw_work_handler);
1131e401fa25SSergey Matyukevich ts->bda = ts->base.epmem_bar;
1132e401fa25SSergey Matyukevich
1133e401fa25SSergey Matyukevich /* assign host msi irq before card init */
1134e401fa25SSergey Matyukevich if (ts->base.msi_enabled)
1135e401fa25SSergey Matyukevich irqflags = IRQF_NOBALANCING;
1136e401fa25SSergey Matyukevich else
1137e401fa25SSergey Matyukevich irqflags = IRQF_NOBALANCING | IRQF_SHARED;
1138e401fa25SSergey Matyukevich
1139e401fa25SSergey Matyukevich ret = devm_request_irq(&pdev->dev, pdev->irq,
1140e401fa25SSergey Matyukevich &qtnf_pcie_topaz_interrupt,
1141e401fa25SSergey Matyukevich irqflags, "qtnf_topaz_irq", (void *)bus);
1142e401fa25SSergey Matyukevich if (ret) {
1143e401fa25SSergey Matyukevich pr_err("failed to request pcie irq %d\n", pdev->irq);
1144e401fa25SSergey Matyukevich return ret;
1145e401fa25SSergey Matyukevich }
1146e401fa25SSergey Matyukevich
1147e401fa25SSergey Matyukevich disable_irq(pdev->irq);
1148e401fa25SSergey Matyukevich
1149e401fa25SSergey Matyukevich ret = qtnf_pre_init_ep(bus);
1150e401fa25SSergey Matyukevich if (ret) {
1151e401fa25SSergey Matyukevich pr_err("failed to init card\n");
1152e401fa25SSergey Matyukevich return ret;
1153e401fa25SSergey Matyukevich }
1154e401fa25SSergey Matyukevich
115597aef03cSSergey Matyukevich ret = qtnf_pcie_topaz_init_xfer(ts, tx_bd_num, rx_bd_num);
1156e401fa25SSergey Matyukevich if (ret) {
1157e401fa25SSergey Matyukevich pr_err("PCIE xfer init failed\n");
1158e401fa25SSergey Matyukevich return ret;
1159e401fa25SSergey Matyukevich }
1160e401fa25SSergey Matyukevich
1161aff8e8d0SAllen Pais tasklet_setup(&ts->base.reclaim_tq, qtnf_reclaim_tasklet_fn);
1162*193eb523SJakub Kicinski netif_napi_add_weight(&bus->mux_dev, &bus->mux_napi,
1163e401fa25SSergey Matyukevich qtnf_topaz_rx_poll, 10);
1164e401fa25SSergey Matyukevich
1165e401fa25SSergey Matyukevich ipc_int.fn = qtnf_topaz_ipc_gen_ep_int;
1166e401fa25SSergey Matyukevich ipc_int.arg = ts;
1167e401fa25SSergey Matyukevich qtnf_pcie_init_shm_ipc(&ts->base, &ts->bda->bda_shm_reg1,
1168e401fa25SSergey Matyukevich &ts->bda->bda_shm_reg2, &ipc_int);
1169e401fa25SSergey Matyukevich
1170e401fa25SSergey Matyukevich return 0;
1171e401fa25SSergey Matyukevich }
1172e401fa25SSergey Matyukevich
qtnf_pcie_topaz_remove(struct qtnf_bus * bus)1173e401fa25SSergey Matyukevich static void qtnf_pcie_topaz_remove(struct qtnf_bus *bus)
1174e401fa25SSergey Matyukevich {
1175e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);
1176e401fa25SSergey Matyukevich
1177e401fa25SSergey Matyukevich qtnf_topaz_reset_ep(ts);
1178e401fa25SSergey Matyukevich qtnf_topaz_free_xfer_buffers(ts);
1179e401fa25SSergey Matyukevich }
1180e401fa25SSergey Matyukevich
1181e401fa25SSergey Matyukevich #ifdef CONFIG_PM_SLEEP
qtnf_pcie_topaz_suspend(struct qtnf_bus * bus)1182e401fa25SSergey Matyukevich static int qtnf_pcie_topaz_suspend(struct qtnf_bus *bus)
1183e401fa25SSergey Matyukevich {
1184e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);
1185e401fa25SSergey Matyukevich struct pci_dev *pdev = ts->base.pdev;
1186e401fa25SSergey Matyukevich
1187e401fa25SSergey Matyukevich writel((u32 __force)PCI_D3hot, ts->ep_pmstate);
1188e401fa25SSergey Matyukevich dma_wmb();
1189e401fa25SSergey Matyukevich writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_PM_EP_IRQ),
1190e401fa25SSergey Matyukevich TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));
1191e401fa25SSergey Matyukevich
1192e401fa25SSergey Matyukevich pci_save_state(pdev);
1193e401fa25SSergey Matyukevich pci_enable_wake(pdev, PCI_D3hot, 1);
1194e401fa25SSergey Matyukevich pci_set_power_state(pdev, PCI_D3hot);
1195e401fa25SSergey Matyukevich
1196e401fa25SSergey Matyukevich return 0;
1197e401fa25SSergey Matyukevich }
1198e401fa25SSergey Matyukevich
qtnf_pcie_topaz_resume(struct qtnf_bus * bus)1199e401fa25SSergey Matyukevich static int qtnf_pcie_topaz_resume(struct qtnf_bus *bus)
1200e401fa25SSergey Matyukevich {
1201e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);
1202e401fa25SSergey Matyukevich struct pci_dev *pdev = ts->base.pdev;
1203e401fa25SSergey Matyukevich
1204e401fa25SSergey Matyukevich pci_set_power_state(pdev, PCI_D0);
1205e401fa25SSergey Matyukevich pci_restore_state(pdev);
1206e401fa25SSergey Matyukevich pci_enable_wake(pdev, PCI_D0, 0);
1207e401fa25SSergey Matyukevich
1208e401fa25SSergey Matyukevich writel((u32 __force)PCI_D0, ts->ep_pmstate);
1209e401fa25SSergey Matyukevich dma_wmb();
1210e401fa25SSergey Matyukevich writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_PM_EP_IRQ),
1211e401fa25SSergey Matyukevich TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));
1212e401fa25SSergey Matyukevich
1213e401fa25SSergey Matyukevich return 0;
1214e401fa25SSergey Matyukevich }
1215e401fa25SSergey Matyukevich #endif
1216e401fa25SSergey Matyukevich
qtnf_pcie_topaz_alloc(struct pci_dev * pdev)1217e401fa25SSergey Matyukevich struct qtnf_bus *qtnf_pcie_topaz_alloc(struct pci_dev *pdev)
1218e401fa25SSergey Matyukevich {
1219e401fa25SSergey Matyukevich struct qtnf_bus *bus;
1220e401fa25SSergey Matyukevich struct qtnf_pcie_topaz_state *ts;
1221e401fa25SSergey Matyukevich
1222e401fa25SSergey Matyukevich bus = devm_kzalloc(&pdev->dev, sizeof(*bus) + sizeof(*ts), GFP_KERNEL);
1223e401fa25SSergey Matyukevich if (!bus)
1224e401fa25SSergey Matyukevich return NULL;
1225e401fa25SSergey Matyukevich
1226e401fa25SSergey Matyukevich ts = get_bus_priv(bus);
1227e401fa25SSergey Matyukevich ts->base.probe_cb = qtnf_pcie_topaz_probe;
1228e401fa25SSergey Matyukevich ts->base.remove_cb = qtnf_pcie_topaz_remove;
1229e401fa25SSergey Matyukevich ts->base.dma_mask_get_cb = qtnf_topaz_dma_mask_get;
1230e401fa25SSergey Matyukevich #ifdef CONFIG_PM_SLEEP
1231e401fa25SSergey Matyukevich ts->base.resume_cb = qtnf_pcie_topaz_resume;
1232e401fa25SSergey Matyukevich ts->base.suspend_cb = qtnf_pcie_topaz_suspend;
1233e401fa25SSergey Matyukevich #endif
1234e401fa25SSergey Matyukevich
1235e401fa25SSergey Matyukevich return bus;
1236e401fa25SSergey Matyukevich }
1237