1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2015-2019 Netronome Systems, Inc. */
3
4 #include <linux/seq_file.h>
5
6 #include "../nfp_net.h"
7 #include "../nfp_net_dp.h"
8 #include "../nfp_net_xsk.h"
9 #include "nfd3.h"
10
nfp_nfd3_xsk_tx_bufs_free(struct nfp_net_tx_ring * tx_ring)11 static void nfp_nfd3_xsk_tx_bufs_free(struct nfp_net_tx_ring *tx_ring)
12 {
13 struct nfp_nfd3_tx_buf *txbuf;
14 unsigned int idx;
15
16 while (tx_ring->rd_p != tx_ring->wr_p) {
17 idx = D_IDX(tx_ring, tx_ring->rd_p);
18 txbuf = &tx_ring->txbufs[idx];
19
20 txbuf->real_len = 0;
21
22 tx_ring->qcp_rd_p++;
23 tx_ring->rd_p++;
24
25 if (tx_ring->r_vec->xsk_pool) {
26 if (txbuf->is_xsk_tx)
27 nfp_nfd3_xsk_tx_free(txbuf);
28
29 xsk_tx_completed(tx_ring->r_vec->xsk_pool, 1);
30 }
31 }
32 }
33
34 /**
35 * nfp_nfd3_tx_ring_reset() - Free any untransmitted buffers and reset pointers
36 * @dp: NFP Net data path struct
37 * @tx_ring: TX ring structure
38 *
39 * Assumes that the device is stopped, must be idempotent.
40 */
41 static void
nfp_nfd3_tx_ring_reset(struct nfp_net_dp * dp,struct nfp_net_tx_ring * tx_ring)42 nfp_nfd3_tx_ring_reset(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)
43 {
44 struct netdev_queue *nd_q;
45 const skb_frag_t *frag;
46
47 while (!tx_ring->is_xdp && tx_ring->rd_p != tx_ring->wr_p) {
48 struct nfp_nfd3_tx_buf *tx_buf;
49 struct sk_buff *skb;
50 int idx, nr_frags;
51
52 idx = D_IDX(tx_ring, tx_ring->rd_p);
53 tx_buf = &tx_ring->txbufs[idx];
54
55 skb = tx_ring->txbufs[idx].skb;
56 nr_frags = skb_shinfo(skb)->nr_frags;
57
58 if (tx_buf->fidx == -1) {
59 /* unmap head */
60 dma_unmap_single(dp->dev, tx_buf->dma_addr,
61 skb_headlen(skb), DMA_TO_DEVICE);
62 } else {
63 /* unmap fragment */
64 frag = &skb_shinfo(skb)->frags[tx_buf->fidx];
65 dma_unmap_page(dp->dev, tx_buf->dma_addr,
66 skb_frag_size(frag), DMA_TO_DEVICE);
67 }
68
69 /* check for last gather fragment */
70 if (tx_buf->fidx == nr_frags - 1)
71 dev_kfree_skb_any(skb);
72
73 tx_buf->dma_addr = 0;
74 tx_buf->skb = NULL;
75 tx_buf->fidx = -2;
76
77 tx_ring->qcp_rd_p++;
78 tx_ring->rd_p++;
79 }
80
81 if (tx_ring->is_xdp)
82 nfp_nfd3_xsk_tx_bufs_free(tx_ring);
83
84 memset(tx_ring->txds, 0, tx_ring->size);
85 tx_ring->wr_p = 0;
86 tx_ring->rd_p = 0;
87 tx_ring->qcp_rd_p = 0;
88 tx_ring->wr_ptr_add = 0;
89
90 if (tx_ring->is_xdp || !dp->netdev)
91 return;
92
93 nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx);
94 netdev_tx_reset_queue(nd_q);
95 }
96
97 /**
98 * nfp_nfd3_tx_ring_free() - Free resources allocated to a TX ring
99 * @tx_ring: TX ring to free
100 */
nfp_nfd3_tx_ring_free(struct nfp_net_tx_ring * tx_ring)101 static void nfp_nfd3_tx_ring_free(struct nfp_net_tx_ring *tx_ring)
102 {
103 struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
104 struct nfp_net_dp *dp = &r_vec->nfp_net->dp;
105
106 kvfree(tx_ring->txbufs);
107
108 if (tx_ring->txds)
109 dma_free_coherent(dp->dev, tx_ring->size,
110 tx_ring->txds, tx_ring->dma);
111
112 tx_ring->cnt = 0;
113 tx_ring->txbufs = NULL;
114 tx_ring->txds = NULL;
115 tx_ring->dma = 0;
116 tx_ring->size = 0;
117 }
118
119 /**
120 * nfp_nfd3_tx_ring_alloc() - Allocate resource for a TX ring
121 * @dp: NFP Net data path struct
122 * @tx_ring: TX Ring structure to allocate
123 *
124 * Return: 0 on success, negative errno otherwise.
125 */
126 static int
nfp_nfd3_tx_ring_alloc(struct nfp_net_dp * dp,struct nfp_net_tx_ring * tx_ring)127 nfp_nfd3_tx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)
128 {
129 struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
130
131 tx_ring->cnt = dp->txd_cnt;
132
133 tx_ring->size = array_size(tx_ring->cnt, sizeof(*tx_ring->txds));
134 tx_ring->txds = dma_alloc_coherent(dp->dev, tx_ring->size,
135 &tx_ring->dma,
136 GFP_KERNEL | __GFP_NOWARN);
137 if (!tx_ring->txds) {
138 netdev_warn(dp->netdev, "failed to allocate TX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n",
139 tx_ring->cnt);
140 goto err_alloc;
141 }
142
143 tx_ring->txbufs = kvcalloc(tx_ring->cnt, sizeof(*tx_ring->txbufs),
144 GFP_KERNEL);
145 if (!tx_ring->txbufs)
146 goto err_alloc;
147
148 if (!tx_ring->is_xdp && dp->netdev)
149 netif_set_xps_queue(dp->netdev, &r_vec->affinity_mask,
150 tx_ring->idx);
151
152 return 0;
153
154 err_alloc:
155 nfp_nfd3_tx_ring_free(tx_ring);
156 return -ENOMEM;
157 }
158
159 static void
nfp_nfd3_tx_ring_bufs_free(struct nfp_net_dp * dp,struct nfp_net_tx_ring * tx_ring)160 nfp_nfd3_tx_ring_bufs_free(struct nfp_net_dp *dp,
161 struct nfp_net_tx_ring *tx_ring)
162 {
163 unsigned int i;
164
165 if (!tx_ring->is_xdp)
166 return;
167
168 for (i = 0; i < tx_ring->cnt; i++) {
169 if (!tx_ring->txbufs[i].frag)
170 return;
171
172 nfp_net_dma_unmap_rx(dp, tx_ring->txbufs[i].dma_addr);
173 __free_page(virt_to_page(tx_ring->txbufs[i].frag));
174 }
175 }
176
177 static int
nfp_nfd3_tx_ring_bufs_alloc(struct nfp_net_dp * dp,struct nfp_net_tx_ring * tx_ring)178 nfp_nfd3_tx_ring_bufs_alloc(struct nfp_net_dp *dp,
179 struct nfp_net_tx_ring *tx_ring)
180 {
181 struct nfp_nfd3_tx_buf *txbufs = tx_ring->txbufs;
182 unsigned int i;
183
184 if (!tx_ring->is_xdp)
185 return 0;
186
187 for (i = 0; i < tx_ring->cnt; i++) {
188 txbufs[i].frag = nfp_net_rx_alloc_one(dp, &txbufs[i].dma_addr);
189 if (!txbufs[i].frag) {
190 nfp_nfd3_tx_ring_bufs_free(dp, tx_ring);
191 return -ENOMEM;
192 }
193 }
194
195 return 0;
196 }
197
198 static void
nfp_nfd3_print_tx_descs(struct seq_file * file,struct nfp_net_r_vector * r_vec,struct nfp_net_tx_ring * tx_ring,u32 d_rd_p,u32 d_wr_p)199 nfp_nfd3_print_tx_descs(struct seq_file *file,
200 struct nfp_net_r_vector *r_vec,
201 struct nfp_net_tx_ring *tx_ring,
202 u32 d_rd_p, u32 d_wr_p)
203 {
204 struct nfp_nfd3_tx_desc *txd;
205 u32 txd_cnt = tx_ring->cnt;
206 int i;
207
208 for (i = 0; i < txd_cnt; i++) {
209 struct xdp_buff *xdp;
210 struct sk_buff *skb;
211
212 txd = &tx_ring->txds[i];
213 seq_printf(file, "%04d: 0x%08x 0x%08x 0x%08x 0x%08x", i,
214 txd->vals[0], txd->vals[1],
215 txd->vals[2], txd->vals[3]);
216
217 if (!tx_ring->is_xdp) {
218 skb = READ_ONCE(tx_ring->txbufs[i].skb);
219 if (skb)
220 seq_printf(file, " skb->head=%p skb->data=%p",
221 skb->head, skb->data);
222 } else {
223 xdp = READ_ONCE(tx_ring->txbufs[i].xdp);
224 if (xdp)
225 seq_printf(file, " xdp->data=%p", xdp->data);
226 }
227
228 if (tx_ring->txbufs[i].dma_addr)
229 seq_printf(file, " dma_addr=%pad",
230 &tx_ring->txbufs[i].dma_addr);
231
232 if (i == tx_ring->rd_p % txd_cnt)
233 seq_puts(file, " H_RD");
234 if (i == tx_ring->wr_p % txd_cnt)
235 seq_puts(file, " H_WR");
236 if (i == d_rd_p % txd_cnt)
237 seq_puts(file, " D_RD");
238 if (i == d_wr_p % txd_cnt)
239 seq_puts(file, " D_WR");
240
241 seq_putc(file, '\n');
242 }
243 }
244
245 #define NFP_NFD3_CFG_CTRL_SUPPORTED \
246 (NFP_NET_CFG_CTRL_ENABLE | NFP_NET_CFG_CTRL_PROMISC | \
247 NFP_NET_CFG_CTRL_L2BC | NFP_NET_CFG_CTRL_L2MC | \
248 NFP_NET_CFG_CTRL_RXCSUM | NFP_NET_CFG_CTRL_TXCSUM | \
249 NFP_NET_CFG_CTRL_RXVLAN | NFP_NET_CFG_CTRL_TXVLAN | \
250 NFP_NET_CFG_CTRL_RXVLAN_V2 | NFP_NET_CFG_CTRL_RXQINQ | \
251 NFP_NET_CFG_CTRL_TXVLAN_V2 | \
252 NFP_NET_CFG_CTRL_GATHER | NFP_NET_CFG_CTRL_LSO | \
253 NFP_NET_CFG_CTRL_CTAG_FILTER | NFP_NET_CFG_CTRL_CMSG_DATA | \
254 NFP_NET_CFG_CTRL_RINGCFG | NFP_NET_CFG_CTRL_RSS | \
255 NFP_NET_CFG_CTRL_IRQMOD | NFP_NET_CFG_CTRL_TXRWB | \
256 NFP_NET_CFG_CTRL_VEPA | \
257 NFP_NET_CFG_CTRL_VXLAN | NFP_NET_CFG_CTRL_NVGRE | \
258 NFP_NET_CFG_CTRL_BPF | NFP_NET_CFG_CTRL_LSO2 | \
259 NFP_NET_CFG_CTRL_RSS2 | NFP_NET_CFG_CTRL_CSUM_COMPLETE | \
260 NFP_NET_CFG_CTRL_LIVE_ADDR)
261
262 const struct nfp_dp_ops nfp_nfd3_ops = {
263 .version = NFP_NFD_VER_NFD3,
264 .tx_min_desc_per_pkt = 1,
265 .cap_mask = NFP_NFD3_CFG_CTRL_SUPPORTED,
266 .dma_mask = DMA_BIT_MASK(40),
267 .poll = nfp_nfd3_poll,
268 .xsk_poll = nfp_nfd3_xsk_poll,
269 .ctrl_poll = nfp_nfd3_ctrl_poll,
270 .xmit = nfp_nfd3_tx,
271 .ctrl_tx_one = nfp_nfd3_ctrl_tx_one,
272 .rx_ring_fill_freelist = nfp_nfd3_rx_ring_fill_freelist,
273 .tx_ring_alloc = nfp_nfd3_tx_ring_alloc,
274 .tx_ring_reset = nfp_nfd3_tx_ring_reset,
275 .tx_ring_free = nfp_nfd3_tx_ring_free,
276 .tx_ring_bufs_alloc = nfp_nfd3_tx_ring_bufs_alloc,
277 .tx_ring_bufs_free = nfp_nfd3_tx_ring_bufs_free,
278 .print_tx_descs = nfp_nfd3_print_tx_descs
279 };
280