1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2018 Netronome Systems, Inc */
3 /* Copyright (C) 2021 Corigine, Inc */
4
5 #include <linux/bpf_trace.h>
6 #include <linux/netdevice.h>
7
8 #include "../nfp_app.h"
9 #include "../nfp_net.h"
10 #include "../nfp_net_dp.h"
11 #include "../nfp_net_xsk.h"
12 #include "nfd3.h"
13
14 static bool
nfp_nfd3_xsk_tx_xdp(const struct nfp_net_dp * dp,struct nfp_net_r_vector * r_vec,struct nfp_net_rx_ring * rx_ring,struct nfp_net_tx_ring * tx_ring,struct nfp_net_xsk_rx_buf * xrxbuf,unsigned int pkt_len,int pkt_off)15 nfp_nfd3_xsk_tx_xdp(const struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,
16 struct nfp_net_rx_ring *rx_ring,
17 struct nfp_net_tx_ring *tx_ring,
18 struct nfp_net_xsk_rx_buf *xrxbuf, unsigned int pkt_len,
19 int pkt_off)
20 {
21 struct xsk_buff_pool *pool = r_vec->xsk_pool;
22 struct nfp_nfd3_tx_buf *txbuf;
23 struct nfp_nfd3_tx_desc *txd;
24 unsigned int wr_idx;
25
26 if (nfp_net_tx_space(tx_ring) < 1)
27 return false;
28
29 xsk_buff_raw_dma_sync_for_device(pool, xrxbuf->dma_addr + pkt_off,
30 pkt_len);
31
32 wr_idx = D_IDX(tx_ring, tx_ring->wr_p);
33
34 txbuf = &tx_ring->txbufs[wr_idx];
35 txbuf->xdp = xrxbuf->xdp;
36 txbuf->real_len = pkt_len;
37 txbuf->is_xsk_tx = true;
38
39 /* Build TX descriptor */
40 txd = &tx_ring->txds[wr_idx];
41 txd->offset_eop = NFD3_DESC_TX_EOP;
42 txd->dma_len = cpu_to_le16(pkt_len);
43 nfp_desc_set_dma_addr_40b(txd, xrxbuf->dma_addr + pkt_off);
44 txd->data_len = cpu_to_le16(pkt_len);
45
46 txd->flags = 0;
47 txd->mss = 0;
48 txd->lso_hdrlen = 0;
49
50 tx_ring->wr_ptr_add++;
51 tx_ring->wr_p++;
52
53 return true;
54 }
55
nfp_nfd3_xsk_rx_skb(struct nfp_net_rx_ring * rx_ring,const struct nfp_net_rx_desc * rxd,struct nfp_net_xsk_rx_buf * xrxbuf,const struct nfp_meta_parsed * meta,unsigned int pkt_len,bool meta_xdp,unsigned int * skbs_polled)56 static void nfp_nfd3_xsk_rx_skb(struct nfp_net_rx_ring *rx_ring,
57 const struct nfp_net_rx_desc *rxd,
58 struct nfp_net_xsk_rx_buf *xrxbuf,
59 const struct nfp_meta_parsed *meta,
60 unsigned int pkt_len,
61 bool meta_xdp,
62 unsigned int *skbs_polled)
63 {
64 struct nfp_net_r_vector *r_vec = rx_ring->r_vec;
65 struct nfp_net_dp *dp = &r_vec->nfp_net->dp;
66 struct net_device *netdev;
67 struct sk_buff *skb;
68
69 if (likely(!meta->portid)) {
70 netdev = dp->netdev;
71 } else {
72 struct nfp_net *nn = netdev_priv(dp->netdev);
73
74 netdev = nfp_app_dev_get(nn->app, meta->portid, NULL);
75 if (unlikely(!netdev)) {
76 nfp_net_xsk_rx_drop(r_vec, xrxbuf);
77 return;
78 }
79 nfp_repr_inc_rx_stats(netdev, pkt_len);
80 }
81
82 skb = napi_alloc_skb(&r_vec->napi, pkt_len);
83 if (!skb) {
84 nfp_net_xsk_rx_drop(r_vec, xrxbuf);
85 return;
86 }
87 skb_put_data(skb, xrxbuf->xdp->data, pkt_len);
88
89 skb->mark = meta->mark;
90 skb_set_hash(skb, meta->hash, meta->hash_type);
91
92 skb_record_rx_queue(skb, rx_ring->idx);
93 skb->protocol = eth_type_trans(skb, netdev);
94
95 nfp_nfd3_rx_csum(dp, r_vec, rxd, meta, skb);
96
97 if (unlikely(!nfp_net_vlan_strip(skb, rxd, meta))) {
98 dev_kfree_skb_any(skb);
99 nfp_net_xsk_rx_drop(r_vec, xrxbuf);
100 return;
101 }
102
103 if (meta_xdp)
104 skb_metadata_set(skb,
105 xrxbuf->xdp->data - xrxbuf->xdp->data_meta);
106
107 napi_gro_receive(&rx_ring->r_vec->napi, skb);
108
109 nfp_net_xsk_rx_free(xrxbuf);
110
111 (*skbs_polled)++;
112 }
113
114 static unsigned int
nfp_nfd3_xsk_rx(struct nfp_net_rx_ring * rx_ring,int budget,unsigned int * skbs_polled)115 nfp_nfd3_xsk_rx(struct nfp_net_rx_ring *rx_ring, int budget,
116 unsigned int *skbs_polled)
117 {
118 struct nfp_net_r_vector *r_vec = rx_ring->r_vec;
119 struct nfp_net_dp *dp = &r_vec->nfp_net->dp;
120 struct nfp_net_tx_ring *tx_ring;
121 struct bpf_prog *xdp_prog;
122 bool xdp_redir = false;
123 int pkts_polled = 0;
124
125 xdp_prog = READ_ONCE(dp->xdp_prog);
126 tx_ring = r_vec->xdp_ring;
127
128 while (pkts_polled < budget) {
129 unsigned int meta_len, data_len, pkt_len, pkt_off;
130 struct nfp_net_xsk_rx_buf *xrxbuf;
131 struct nfp_net_rx_desc *rxd;
132 struct nfp_meta_parsed meta;
133 int idx, act;
134
135 idx = D_IDX(rx_ring, rx_ring->rd_p);
136
137 rxd = &rx_ring->rxds[idx];
138 if (!(rxd->rxd.meta_len_dd & PCIE_DESC_RX_DD))
139 break;
140
141 rx_ring->rd_p++;
142 pkts_polled++;
143
144 xrxbuf = &rx_ring->xsk_rxbufs[idx];
145
146 /* If starved of buffers "drop" it and scream. */
147 if (rx_ring->rd_p >= rx_ring->wr_p) {
148 nn_dp_warn(dp, "Starved of RX buffers\n");
149 nfp_net_xsk_rx_drop(r_vec, xrxbuf);
150 break;
151 }
152
153 /* Memory barrier to ensure that we won't do other reads
154 * before the DD bit.
155 */
156 dma_rmb();
157
158 memset(&meta, 0, sizeof(meta));
159
160 /* Only supporting AF_XDP with dynamic metadata so buffer layout
161 * is always:
162 *
163 * ---------------------------------------------------------
164 * | off | metadata | packet | XXXX |
165 * ---------------------------------------------------------
166 */
167 meta_len = rxd->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK;
168 data_len = le16_to_cpu(rxd->rxd.data_len);
169 pkt_len = data_len - meta_len;
170
171 if (unlikely(meta_len > NFP_NET_MAX_PREPEND)) {
172 nn_dp_warn(dp, "Oversized RX packet metadata %u\n",
173 meta_len);
174 nfp_net_xsk_rx_drop(r_vec, xrxbuf);
175 continue;
176 }
177
178 /* Stats update. */
179 u64_stats_update_begin(&r_vec->rx_sync);
180 r_vec->rx_pkts++;
181 r_vec->rx_bytes += pkt_len;
182 u64_stats_update_end(&r_vec->rx_sync);
183
184 xrxbuf->xdp->data += meta_len;
185 xrxbuf->xdp->data_end = xrxbuf->xdp->data + pkt_len;
186 xdp_set_data_meta_invalid(xrxbuf->xdp);
187 xsk_buff_dma_sync_for_cpu(xrxbuf->xdp, r_vec->xsk_pool);
188 net_prefetch(xrxbuf->xdp->data);
189
190 if (meta_len) {
191 if (unlikely(nfp_nfd3_parse_meta(dp->netdev, &meta,
192 xrxbuf->xdp->data -
193 meta_len,
194 xrxbuf->xdp->data,
195 pkt_len, meta_len))) {
196 nn_dp_warn(dp, "Invalid RX packet metadata\n");
197 nfp_net_xsk_rx_drop(r_vec, xrxbuf);
198 continue;
199 }
200
201 if (unlikely(meta.portid)) {
202 struct nfp_net *nn = netdev_priv(dp->netdev);
203
204 if (meta.portid != NFP_META_PORT_ID_CTRL) {
205 nfp_nfd3_xsk_rx_skb(rx_ring, rxd,
206 xrxbuf, &meta,
207 pkt_len, false,
208 skbs_polled);
209 continue;
210 }
211
212 nfp_app_ctrl_rx_raw(nn->app, xrxbuf->xdp->data,
213 pkt_len);
214 nfp_net_xsk_rx_free(xrxbuf);
215 continue;
216 }
217 }
218
219 act = bpf_prog_run_xdp(xdp_prog, xrxbuf->xdp);
220
221 pkt_len = xrxbuf->xdp->data_end - xrxbuf->xdp->data;
222 pkt_off = xrxbuf->xdp->data - xrxbuf->xdp->data_hard_start;
223
224 switch (act) {
225 case XDP_PASS:
226 nfp_nfd3_xsk_rx_skb(rx_ring, rxd, xrxbuf, &meta, pkt_len,
227 true, skbs_polled);
228 break;
229 case XDP_TX:
230 if (!nfp_nfd3_xsk_tx_xdp(dp, r_vec, rx_ring, tx_ring,
231 xrxbuf, pkt_len, pkt_off))
232 nfp_net_xsk_rx_drop(r_vec, xrxbuf);
233 else
234 nfp_net_xsk_rx_unstash(xrxbuf);
235 break;
236 case XDP_REDIRECT:
237 if (xdp_do_redirect(dp->netdev, xrxbuf->xdp, xdp_prog)) {
238 nfp_net_xsk_rx_drop(r_vec, xrxbuf);
239 } else {
240 nfp_net_xsk_rx_unstash(xrxbuf);
241 xdp_redir = true;
242 }
243 break;
244 default:
245 bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act);
246 fallthrough;
247 case XDP_ABORTED:
248 trace_xdp_exception(dp->netdev, xdp_prog, act);
249 fallthrough;
250 case XDP_DROP:
251 nfp_net_xsk_rx_drop(r_vec, xrxbuf);
252 break;
253 }
254 }
255
256 nfp_net_xsk_rx_ring_fill_freelist(r_vec->rx_ring);
257
258 if (xdp_redir)
259 xdp_do_flush_map();
260
261 if (tx_ring->wr_ptr_add)
262 nfp_net_tx_xmit_more_flush(tx_ring);
263
264 return pkts_polled;
265 }
266
nfp_nfd3_xsk_tx_free(struct nfp_nfd3_tx_buf * txbuf)267 void nfp_nfd3_xsk_tx_free(struct nfp_nfd3_tx_buf *txbuf)
268 {
269 xsk_buff_free(txbuf->xdp);
270
271 txbuf->dma_addr = 0;
272 txbuf->xdp = NULL;
273 }
274
nfp_nfd3_xsk_complete(struct nfp_net_tx_ring * tx_ring)275 static bool nfp_nfd3_xsk_complete(struct nfp_net_tx_ring *tx_ring)
276 {
277 struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
278 u32 done_pkts = 0, done_bytes = 0, reused = 0;
279 bool done_all;
280 int idx, todo;
281 u32 qcp_rd_p;
282
283 if (tx_ring->wr_p == tx_ring->rd_p)
284 return true;
285
286 /* Work out how many descriptors have been transmitted. */
287 qcp_rd_p = nfp_qcp_rd_ptr_read(tx_ring->qcp_q);
288
289 if (qcp_rd_p == tx_ring->qcp_rd_p)
290 return true;
291
292 todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p);
293
294 done_all = todo <= NFP_NET_XDP_MAX_COMPLETE;
295 todo = min(todo, NFP_NET_XDP_MAX_COMPLETE);
296
297 tx_ring->qcp_rd_p = D_IDX(tx_ring, tx_ring->qcp_rd_p + todo);
298
299 done_pkts = todo;
300 while (todo--) {
301 struct nfp_nfd3_tx_buf *txbuf;
302
303 idx = D_IDX(tx_ring, tx_ring->rd_p);
304 tx_ring->rd_p++;
305
306 txbuf = &tx_ring->txbufs[idx];
307 if (unlikely(!txbuf->real_len))
308 continue;
309
310 done_bytes += txbuf->real_len;
311 txbuf->real_len = 0;
312
313 if (txbuf->is_xsk_tx) {
314 nfp_nfd3_xsk_tx_free(txbuf);
315 reused++;
316 }
317 }
318
319 u64_stats_update_begin(&r_vec->tx_sync);
320 r_vec->tx_bytes += done_bytes;
321 r_vec->tx_pkts += done_pkts;
322 u64_stats_update_end(&r_vec->tx_sync);
323
324 xsk_tx_completed(r_vec->xsk_pool, done_pkts - reused);
325
326 WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt,
327 "XDP TX ring corruption rd_p=%u wr_p=%u cnt=%u\n",
328 tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt);
329
330 return done_all;
331 }
332
nfp_nfd3_xsk_tx(struct nfp_net_tx_ring * tx_ring)333 static void nfp_nfd3_xsk_tx(struct nfp_net_tx_ring *tx_ring)
334 {
335 struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
336 struct xdp_desc desc[NFP_NET_XSK_TX_BATCH];
337 struct xsk_buff_pool *xsk_pool;
338 struct nfp_nfd3_tx_desc *txd;
339 u32 pkts = 0, wr_idx;
340 u32 i, got;
341
342 xsk_pool = r_vec->xsk_pool;
343
344 while (nfp_net_tx_space(tx_ring) >= NFP_NET_XSK_TX_BATCH) {
345 for (i = 0; i < NFP_NET_XSK_TX_BATCH; i++)
346 if (!xsk_tx_peek_desc(xsk_pool, &desc[i]))
347 break;
348 got = i;
349 if (!got)
350 break;
351
352 wr_idx = D_IDX(tx_ring, tx_ring->wr_p + i);
353 prefetchw(&tx_ring->txds[wr_idx]);
354
355 for (i = 0; i < got; i++)
356 xsk_buff_raw_dma_sync_for_device(xsk_pool, desc[i].addr,
357 desc[i].len);
358
359 for (i = 0; i < got; i++) {
360 wr_idx = D_IDX(tx_ring, tx_ring->wr_p + i);
361
362 tx_ring->txbufs[wr_idx].real_len = desc[i].len;
363 tx_ring->txbufs[wr_idx].is_xsk_tx = false;
364
365 /* Build TX descriptor. */
366 txd = &tx_ring->txds[wr_idx];
367 nfp_desc_set_dma_addr_40b(txd,
368 xsk_buff_raw_get_dma(xsk_pool, desc[i].addr));
369 txd->offset_eop = NFD3_DESC_TX_EOP;
370 txd->dma_len = cpu_to_le16(desc[i].len);
371 txd->data_len = cpu_to_le16(desc[i].len);
372 }
373
374 tx_ring->wr_p += got;
375 pkts += got;
376 }
377
378 if (!pkts)
379 return;
380
381 xsk_tx_release(xsk_pool);
382 /* Ensure all records are visible before incrementing write counter. */
383 wmb();
384 nfp_qcp_wr_ptr_add(tx_ring->qcp_q, pkts);
385 }
386
nfp_nfd3_xsk_poll(struct napi_struct * napi,int budget)387 int nfp_nfd3_xsk_poll(struct napi_struct *napi, int budget)
388 {
389 struct nfp_net_r_vector *r_vec =
390 container_of(napi, struct nfp_net_r_vector, napi);
391 unsigned int pkts_polled, skbs = 0;
392
393 pkts_polled = nfp_nfd3_xsk_rx(r_vec->rx_ring, budget, &skbs);
394
395 if (pkts_polled < budget) {
396 if (r_vec->tx_ring)
397 nfp_nfd3_tx_complete(r_vec->tx_ring, budget);
398
399 if (!nfp_nfd3_xsk_complete(r_vec->xdp_ring))
400 pkts_polled = budget;
401
402 nfp_nfd3_xsk_tx(r_vec->xdp_ring);
403
404 if (pkts_polled < budget && napi_complete_done(napi, skbs))
405 nfp_net_irq_unmask(r_vec->nfp_net, r_vec->irq_entry);
406 }
407
408 return pkts_polled;
409 }
410