1159d2131STariq Toukan /* 2159d2131STariq Toukan * Copyright (c) 2018, Mellanox Technologies. All rights reserved. 3159d2131STariq Toukan * 4159d2131STariq Toukan * This software is available to you under a choice of one of two 5159d2131STariq Toukan * licenses. You may choose to be licensed under the terms of the GNU 6159d2131STariq Toukan * General Public License (GPL) Version 2, available from the file 7159d2131STariq Toukan * COPYING in the main directory of this source tree, or the 8159d2131STariq Toukan * OpenIB.org BSD license below: 9159d2131STariq Toukan * 10159d2131STariq Toukan * Redistribution and use in source and binary forms, with or 11159d2131STariq Toukan * without modification, are permitted provided that the following 12159d2131STariq Toukan * conditions are met: 13159d2131STariq Toukan * 14159d2131STariq Toukan * - Redistributions of source code must retain the above 15159d2131STariq Toukan * copyright notice, this list of conditions and the following 16159d2131STariq Toukan * disclaimer. 17159d2131STariq Toukan * 18159d2131STariq Toukan * - Redistributions in binary form must reproduce the above 19159d2131STariq Toukan * copyright notice, this list of conditions and the following 20159d2131STariq Toukan * disclaimer in the documentation and/or other materials 21159d2131STariq Toukan * provided with the distribution. 22159d2131STariq Toukan * 23159d2131STariq Toukan * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24159d2131STariq Toukan * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25159d2131STariq Toukan * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26159d2131STariq Toukan * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27159d2131STariq Toukan * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28159d2131STariq Toukan * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29159d2131STariq Toukan * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30159d2131STariq Toukan * SOFTWARE. 31159d2131STariq Toukan */ 32159d2131STariq Toukan 33159d2131STariq Toukan #include <linux/bpf_trace.h> 34159d2131STariq Toukan #include "en/xdp.h" 35159d2131STariq Toukan 36c94e4f11STariq Toukan static inline bool 37c94e4f11STariq Toukan mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_dma_info *di, 38c94e4f11STariq Toukan struct xdp_buff *xdp) 39c94e4f11STariq Toukan { 40c94e4f11STariq Toukan struct mlx5e_xdp_info xdpi; 41c94e4f11STariq Toukan 42c94e4f11STariq Toukan xdpi.xdpf = convert_to_xdp_frame(xdp); 43c94e4f11STariq Toukan if (unlikely(!xdpi.xdpf)) 44c94e4f11STariq Toukan return false; 45c94e4f11STariq Toukan xdpi.dma_addr = di->addr + (xdpi.xdpf->data - (void *)xdpi.xdpf); 46c94e4f11STariq Toukan dma_sync_single_for_device(sq->pdev, xdpi.dma_addr, 47c94e4f11STariq Toukan xdpi.xdpf->len, PCI_DMA_TODEVICE); 48c94e4f11STariq Toukan xdpi.di = *di; 49c94e4f11STariq Toukan 505e0d2eefSTariq Toukan return sq->xmit_xdp_frame(sq, &xdpi); 51c94e4f11STariq Toukan } 52c94e4f11STariq Toukan 53159d2131STariq Toukan /* returns true if packet was consumed by xdp */ 54159d2131STariq Toukan bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di, 55159d2131STariq Toukan void *va, u16 *rx_headroom, u32 *len) 56159d2131STariq Toukan { 57159d2131STariq Toukan struct bpf_prog *prog = READ_ONCE(rq->xdp_prog); 58159d2131STariq Toukan struct xdp_buff xdp; 59159d2131STariq Toukan u32 act; 60159d2131STariq Toukan int err; 61159d2131STariq Toukan 62159d2131STariq Toukan if (!prog) 63159d2131STariq Toukan return false; 64159d2131STariq Toukan 65159d2131STariq Toukan xdp.data = va + *rx_headroom; 66159d2131STariq Toukan xdp_set_data_meta_invalid(&xdp); 67159d2131STariq Toukan xdp.data_end = xdp.data + *len; 68159d2131STariq Toukan xdp.data_hard_start = va; 69159d2131STariq Toukan xdp.rxq = &rq->xdp_rxq; 70159d2131STariq Toukan 71159d2131STariq Toukan act = bpf_prog_run_xdp(prog, &xdp); 72159d2131STariq Toukan switch (act) { 73159d2131STariq Toukan case XDP_PASS: 74159d2131STariq Toukan *rx_headroom = xdp.data - xdp.data_hard_start; 75159d2131STariq Toukan *len = xdp.data_end - xdp.data; 76159d2131STariq Toukan return false; 77159d2131STariq Toukan case XDP_TX: 78c94e4f11STariq Toukan if (unlikely(!mlx5e_xmit_xdp_buff(&rq->xdpsq, di, &xdp))) 79c94e4f11STariq Toukan goto xdp_abort; 80c94e4f11STariq Toukan __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */ 81159d2131STariq Toukan return true; 82159d2131STariq Toukan case XDP_REDIRECT: 83159d2131STariq Toukan /* When XDP enabled then page-refcnt==1 here */ 84159d2131STariq Toukan err = xdp_do_redirect(rq->netdev, &xdp, prog); 85c94e4f11STariq Toukan if (unlikely(err)) 86c94e4f11STariq Toukan goto xdp_abort; 87159d2131STariq Toukan __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); 8815143bf5STariq Toukan __set_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags); 89159d2131STariq Toukan mlx5e_page_dma_unmap(rq, di); 9086690b4bSTariq Toukan rq->stats->xdp_redirect++; 91159d2131STariq Toukan return true; 92159d2131STariq Toukan default: 93159d2131STariq Toukan bpf_warn_invalid_xdp_action(act); 94e77f02b8SGustavo A. R. Silva /* fall through */ 95159d2131STariq Toukan case XDP_ABORTED: 96c94e4f11STariq Toukan xdp_abort: 97159d2131STariq Toukan trace_xdp_exception(rq->netdev, prog, act); 98e77f02b8SGustavo A. R. Silva /* fall through */ 99159d2131STariq Toukan case XDP_DROP: 100159d2131STariq Toukan rq->stats->xdp_drop++; 101159d2131STariq Toukan return true; 102159d2131STariq Toukan } 103159d2131STariq Toukan } 104159d2131STariq Toukan 1055e0d2eefSTariq Toukan static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq) 1065e0d2eefSTariq Toukan { 1075e0d2eefSTariq Toukan struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; 10873cab880SShay Agroskin struct mlx5e_xdpsq_stats *stats = sq->stats; 1095e0d2eefSTariq Toukan struct mlx5_wq_cyc *wq = &sq->wq; 1105e0d2eefSTariq Toukan u8 wqebbs; 1115e0d2eefSTariq Toukan u16 pi; 1125e0d2eefSTariq Toukan 1135e0d2eefSTariq Toukan mlx5e_xdpsq_fetch_wqe(sq, &session->wqe); 1145e0d2eefSTariq Toukan 1155e0d2eefSTariq Toukan prefetchw(session->wqe->data); 1165e0d2eefSTariq Toukan session->ds_count = MLX5E_XDP_TX_EMPTY_DS_COUNT; 1175e0d2eefSTariq Toukan 1185e0d2eefSTariq Toukan pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1195e0d2eefSTariq Toukan 1205e0d2eefSTariq Toukan /* The mult of MLX5_SEND_WQE_MAX_WQEBBS * MLX5_SEND_WQEBB_NUM_DS 1215e0d2eefSTariq Toukan * (16 * 4 == 64) does not fit in the 6-bit DS field of Ctrl Segment. 1225e0d2eefSTariq Toukan * We use a bound lower that MLX5_SEND_WQE_MAX_WQEBBS to let a 1235e0d2eefSTariq Toukan * full-session WQE be cache-aligned. 1245e0d2eefSTariq Toukan */ 1255e0d2eefSTariq Toukan #if L1_CACHE_BYTES < 128 1265e0d2eefSTariq Toukan #define MLX5E_XDP_MPW_MAX_WQEBBS (MLX5_SEND_WQE_MAX_WQEBBS - 1) 1275e0d2eefSTariq Toukan #else 1285e0d2eefSTariq Toukan #define MLX5E_XDP_MPW_MAX_WQEBBS (MLX5_SEND_WQE_MAX_WQEBBS - 2) 1295e0d2eefSTariq Toukan #endif 1305e0d2eefSTariq Toukan 1315e0d2eefSTariq Toukan wqebbs = min_t(u16, mlx5_wq_cyc_get_contig_wqebbs(wq, pi), 1325e0d2eefSTariq Toukan MLX5E_XDP_MPW_MAX_WQEBBS); 1335e0d2eefSTariq Toukan 1345e0d2eefSTariq Toukan session->max_ds_count = MLX5_SEND_WQEBB_NUM_DS * wqebbs; 13573cab880SShay Agroskin stats->mpwqe++; 1365e0d2eefSTariq Toukan } 1375e0d2eefSTariq Toukan 1385e0d2eefSTariq Toukan static void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq) 1395e0d2eefSTariq Toukan { 1405e0d2eefSTariq Toukan struct mlx5_wq_cyc *wq = &sq->wq; 1415e0d2eefSTariq Toukan struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; 1425e0d2eefSTariq Toukan struct mlx5_wqe_ctrl_seg *cseg = &session->wqe->ctrl; 1435e0d2eefSTariq Toukan u16 ds_count = session->ds_count; 1445e0d2eefSTariq Toukan u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1455e0d2eefSTariq Toukan struct mlx5e_xdp_wqe_info *wi = &sq->db.wqe_info[pi]; 1465e0d2eefSTariq Toukan 1475e0d2eefSTariq Toukan cseg->opmod_idx_opcode = 1485e0d2eefSTariq Toukan cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_ENHANCED_MPSW); 1495e0d2eefSTariq Toukan cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_count); 1505e0d2eefSTariq Toukan 1515e0d2eefSTariq Toukan wi->num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS); 1525e0d2eefSTariq Toukan wi->num_ds = ds_count - MLX5E_XDP_TX_EMPTY_DS_COUNT; 1535e0d2eefSTariq Toukan 1545e0d2eefSTariq Toukan sq->pc += wi->num_wqebbs; 1555e0d2eefSTariq Toukan 1565e0d2eefSTariq Toukan sq->doorbell_cseg = cseg; 1575e0d2eefSTariq Toukan 1585e0d2eefSTariq Toukan session->wqe = NULL; /* Close session */ 1595e0d2eefSTariq Toukan } 1605e0d2eefSTariq Toukan 1615e0d2eefSTariq Toukan static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, 1625e0d2eefSTariq Toukan struct mlx5e_xdp_info *xdpi) 1635e0d2eefSTariq Toukan { 1645e0d2eefSTariq Toukan struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; 1655e0d2eefSTariq Toukan struct mlx5e_xdpsq_stats *stats = sq->stats; 1665e0d2eefSTariq Toukan 1675e0d2eefSTariq Toukan dma_addr_t dma_addr = xdpi->dma_addr; 1685e0d2eefSTariq Toukan struct xdp_frame *xdpf = xdpi->xdpf; 1695e0d2eefSTariq Toukan unsigned int dma_len = xdpf->len; 1705e0d2eefSTariq Toukan 1715e0d2eefSTariq Toukan if (unlikely(sq->hw_mtu < dma_len)) { 1725e0d2eefSTariq Toukan stats->err++; 1735e0d2eefSTariq Toukan return false; 1745e0d2eefSTariq Toukan } 1755e0d2eefSTariq Toukan 1765e0d2eefSTariq Toukan if (unlikely(!session->wqe)) { 1775e0d2eefSTariq Toukan if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1785e0d2eefSTariq Toukan MLX5_SEND_WQE_MAX_WQEBBS))) { 1795e0d2eefSTariq Toukan /* SQ is full, ring doorbell */ 1805e0d2eefSTariq Toukan mlx5e_xmit_xdp_doorbell(sq); 1815e0d2eefSTariq Toukan stats->full++; 1825e0d2eefSTariq Toukan return false; 1835e0d2eefSTariq Toukan } 1845e0d2eefSTariq Toukan 1855e0d2eefSTariq Toukan mlx5e_xdp_mpwqe_session_start(sq); 1865e0d2eefSTariq Toukan } 1875e0d2eefSTariq Toukan 1885e0d2eefSTariq Toukan mlx5e_xdp_mpwqe_add_dseg(sq, dma_addr, dma_len); 1895e0d2eefSTariq Toukan 1905e0d2eefSTariq Toukan if (unlikely(session->ds_count == session->max_ds_count)) 1915e0d2eefSTariq Toukan mlx5e_xdp_mpwqe_complete(sq); 1925e0d2eefSTariq Toukan 1935e0d2eefSTariq Toukan mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi); 1945e0d2eefSTariq Toukan stats->xmit++; 1955e0d2eefSTariq Toukan return true; 1965e0d2eefSTariq Toukan } 1975e0d2eefSTariq Toukan 1985e0d2eefSTariq Toukan static bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_info *xdpi) 199159d2131STariq Toukan { 200159d2131STariq Toukan struct mlx5_wq_cyc *wq = &sq->wq; 201159d2131STariq Toukan u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 202159d2131STariq Toukan struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); 203159d2131STariq Toukan 204159d2131STariq Toukan struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 205159d2131STariq Toukan struct mlx5_wqe_eth_seg *eseg = &wqe->eth; 206c94e4f11STariq Toukan struct mlx5_wqe_data_seg *dseg = wqe->data; 207159d2131STariq Toukan 208c94e4f11STariq Toukan struct xdp_frame *xdpf = xdpi->xdpf; 209c94e4f11STariq Toukan dma_addr_t dma_addr = xdpi->dma_addr; 210c94e4f11STariq Toukan unsigned int dma_len = xdpf->len; 211159d2131STariq Toukan 212890388adSTariq Toukan struct mlx5e_xdpsq_stats *stats = sq->stats; 213159d2131STariq Toukan 214159d2131STariq Toukan prefetchw(wqe); 215159d2131STariq Toukan 216c94e4f11STariq Toukan if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE || sq->hw_mtu < dma_len)) { 217890388adSTariq Toukan stats->err++; 218159d2131STariq Toukan return false; 219159d2131STariq Toukan } 220159d2131STariq Toukan 221159d2131STariq Toukan if (unlikely(!mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1))) { 222159d2131STariq Toukan /* SQ is full, ring doorbell */ 223159d2131STariq Toukan mlx5e_xmit_xdp_doorbell(sq); 224890388adSTariq Toukan stats->full++; 225159d2131STariq Toukan return false; 226159d2131STariq Toukan } 227159d2131STariq Toukan 228159d2131STariq Toukan cseg->fm_ce_se = 0; 229159d2131STariq Toukan 230159d2131STariq Toukan /* copy the inline part if required */ 231159d2131STariq Toukan if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) { 232c94e4f11STariq Toukan memcpy(eseg->inline_hdr.start, xdpf->data, MLX5E_XDP_MIN_INLINE); 233159d2131STariq Toukan eseg->inline_hdr.sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE); 234159d2131STariq Toukan dma_len -= MLX5E_XDP_MIN_INLINE; 235159d2131STariq Toukan dma_addr += MLX5E_XDP_MIN_INLINE; 236159d2131STariq Toukan dseg++; 237159d2131STariq Toukan } 238159d2131STariq Toukan 239159d2131STariq Toukan /* write the dma part */ 240159d2131STariq Toukan dseg->addr = cpu_to_be64(dma_addr); 241159d2131STariq Toukan dseg->byte_count = cpu_to_be32(dma_len); 242159d2131STariq Toukan 243159d2131STariq Toukan cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_SEND); 244159d2131STariq Toukan 245159d2131STariq Toukan sq->pc++; 246159d2131STariq Toukan 247b8180392STariq Toukan sq->doorbell_cseg = cseg; 248159d2131STariq Toukan 249fea28dd6STariq Toukan mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi); 250890388adSTariq Toukan stats->xmit++; 251159d2131STariq Toukan return true; 252159d2131STariq Toukan } 253159d2131STariq Toukan 254feb2ff9dSTariq Toukan bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) 255159d2131STariq Toukan { 256fea28dd6STariq Toukan struct mlx5e_xdp_info_fifo *xdpi_fifo; 257159d2131STariq Toukan struct mlx5e_xdpsq *sq; 258159d2131STariq Toukan struct mlx5_cqe64 *cqe; 25958b99ee3STariq Toukan bool is_redirect; 260159d2131STariq Toukan u16 sqcc; 261159d2131STariq Toukan int i; 262159d2131STariq Toukan 263159d2131STariq Toukan sq = container_of(cq, struct mlx5e_xdpsq, cq); 264159d2131STariq Toukan 265159d2131STariq Toukan if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) 266159d2131STariq Toukan return false; 267159d2131STariq Toukan 268159d2131STariq Toukan cqe = mlx5_cqwq_get_cqe(&cq->wq); 269159d2131STariq Toukan if (!cqe) 270159d2131STariq Toukan return false; 271159d2131STariq Toukan 272feb2ff9dSTariq Toukan is_redirect = !rq; 273fea28dd6STariq Toukan xdpi_fifo = &sq->db.xdpi_fifo; 274159d2131STariq Toukan 275159d2131STariq Toukan /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), 276159d2131STariq Toukan * otherwise a cq overrun may occur 277159d2131STariq Toukan */ 278159d2131STariq Toukan sqcc = sq->cc; 279159d2131STariq Toukan 280159d2131STariq Toukan i = 0; 281159d2131STariq Toukan do { 282159d2131STariq Toukan u16 wqe_counter; 283159d2131STariq Toukan bool last_wqe; 284159d2131STariq Toukan 285159d2131STariq Toukan mlx5_cqwq_pop(&cq->wq); 286159d2131STariq Toukan 287159d2131STariq Toukan wqe_counter = be16_to_cpu(cqe->wqe_counter); 288159d2131STariq Toukan 289db02a308STariq Toukan if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) 290db02a308STariq Toukan netdev_WARN_ONCE(sq->channel->netdev, 291db02a308STariq Toukan "Bad OP in XDPSQ CQE: 0x%x\n", 292db02a308STariq Toukan get_cqe_opcode(cqe)); 293db02a308STariq Toukan 294159d2131STariq Toukan do { 2951feeab80STariq Toukan struct mlx5e_xdp_wqe_info *wi; 2961feeab80STariq Toukan u16 ci, j; 297159d2131STariq Toukan 298159d2131STariq Toukan last_wqe = (sqcc == wqe_counter); 2991feeab80STariq Toukan ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); 3001feeab80STariq Toukan wi = &sq->db.wqe_info[ci]; 3011feeab80STariq Toukan 3021feeab80STariq Toukan sqcc += wi->num_wqebbs; 3031feeab80STariq Toukan 3041feeab80STariq Toukan for (j = 0; j < wi->num_ds; j++) { 3051feeab80STariq Toukan struct mlx5e_xdp_info xdpi = 3061feeab80STariq Toukan mlx5e_xdpi_fifo_pop(xdpi_fifo); 30758b99ee3STariq Toukan 30858b99ee3STariq Toukan if (is_redirect) { 309fea28dd6STariq Toukan xdp_return_frame(xdpi.xdpf); 310fea28dd6STariq Toukan dma_unmap_single(sq->pdev, xdpi.dma_addr, 311fea28dd6STariq Toukan xdpi.xdpf->len, DMA_TO_DEVICE); 31258b99ee3STariq Toukan } else { 313159d2131STariq Toukan /* Recycle RX page */ 314fea28dd6STariq Toukan mlx5e_page_release(rq, &xdpi.di, true); 31558b99ee3STariq Toukan } 3161feeab80STariq Toukan } 317159d2131STariq Toukan } while (!last_wqe); 318159d2131STariq Toukan } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq))); 319159d2131STariq Toukan 320890388adSTariq Toukan sq->stats->cqes += i; 321159d2131STariq Toukan 322159d2131STariq Toukan mlx5_cqwq_update_db_record(&cq->wq); 323159d2131STariq Toukan 324159d2131STariq Toukan /* ensure cq space is freed before enabling more cqes */ 325159d2131STariq Toukan wmb(); 326159d2131STariq Toukan 327159d2131STariq Toukan sq->cc = sqcc; 328159d2131STariq Toukan return (i == MLX5E_TX_CQ_POLL_BUDGET); 329159d2131STariq Toukan } 330159d2131STariq Toukan 331feb2ff9dSTariq Toukan void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq) 332159d2131STariq Toukan { 333fea28dd6STariq Toukan struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo; 334feb2ff9dSTariq Toukan bool is_redirect = !rq; 335159d2131STariq Toukan 336159d2131STariq Toukan while (sq->cc != sq->pc) { 3371feeab80STariq Toukan struct mlx5e_xdp_wqe_info *wi; 3381feeab80STariq Toukan u16 ci, i; 3391feeab80STariq Toukan 3401feeab80STariq Toukan ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc); 3411feeab80STariq Toukan wi = &sq->db.wqe_info[ci]; 3421feeab80STariq Toukan 3431feeab80STariq Toukan sq->cc += wi->num_wqebbs; 3441feeab80STariq Toukan 3451feeab80STariq Toukan for (i = 0; i < wi->num_ds; i++) { 346fea28dd6STariq Toukan struct mlx5e_xdp_info xdpi = 347fea28dd6STariq Toukan mlx5e_xdpi_fifo_pop(xdpi_fifo); 34858b99ee3STariq Toukan 34958b99ee3STariq Toukan if (is_redirect) { 350fea28dd6STariq Toukan xdp_return_frame(xdpi.xdpf); 351fea28dd6STariq Toukan dma_unmap_single(sq->pdev, xdpi.dma_addr, 352fea28dd6STariq Toukan xdpi.xdpf->len, DMA_TO_DEVICE); 35358b99ee3STariq Toukan } else { 35458b99ee3STariq Toukan /* Recycle RX page */ 355fea28dd6STariq Toukan mlx5e_page_release(rq, &xdpi.di, false); 356159d2131STariq Toukan } 357159d2131STariq Toukan } 35858b99ee3STariq Toukan } 3591feeab80STariq Toukan } 360159d2131STariq Toukan 36158b99ee3STariq Toukan int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 36258b99ee3STariq Toukan u32 flags) 36358b99ee3STariq Toukan { 36458b99ee3STariq Toukan struct mlx5e_priv *priv = netdev_priv(dev); 36558b99ee3STariq Toukan struct mlx5e_xdpsq *sq; 36658b99ee3STariq Toukan int drops = 0; 36758b99ee3STariq Toukan int sq_num; 36858b99ee3STariq Toukan int i; 36958b99ee3STariq Toukan 370407e17b1SSaeed Mahameed /* this flag is sufficient, no need to test internal sq state */ 371407e17b1SSaeed Mahameed if (unlikely(!mlx5e_xdp_tx_is_enabled(priv))) 37258b99ee3STariq Toukan return -ENETDOWN; 37358b99ee3STariq Toukan 37458b99ee3STariq Toukan if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 37558b99ee3STariq Toukan return -EINVAL; 37658b99ee3STariq Toukan 37758b99ee3STariq Toukan sq_num = smp_processor_id(); 37858b99ee3STariq Toukan 37958b99ee3STariq Toukan if (unlikely(sq_num >= priv->channels.num)) 38058b99ee3STariq Toukan return -ENXIO; 38158b99ee3STariq Toukan 38258b99ee3STariq Toukan sq = &priv->channels.c[sq_num]->xdpsq; 38358b99ee3STariq Toukan 38458b99ee3STariq Toukan for (i = 0; i < n; i++) { 38558b99ee3STariq Toukan struct xdp_frame *xdpf = frames[i]; 38658b99ee3STariq Toukan struct mlx5e_xdp_info xdpi; 38758b99ee3STariq Toukan 38858b99ee3STariq Toukan xdpi.dma_addr = dma_map_single(sq->pdev, xdpf->data, xdpf->len, 38958b99ee3STariq Toukan DMA_TO_DEVICE); 39058b99ee3STariq Toukan if (unlikely(dma_mapping_error(sq->pdev, xdpi.dma_addr))) { 39139c64d8cSJesper Dangaard Brouer xdp_return_frame_rx_napi(xdpf); 39258b99ee3STariq Toukan drops++; 39358b99ee3STariq Toukan continue; 39458b99ee3STariq Toukan } 39558b99ee3STariq Toukan 39658b99ee3STariq Toukan xdpi.xdpf = xdpf; 39758b99ee3STariq Toukan 3985e0d2eefSTariq Toukan if (unlikely(!sq->xmit_xdp_frame(sq, &xdpi))) { 39939c64d8cSJesper Dangaard Brouer dma_unmap_single(sq->pdev, xdpi.dma_addr, 40039c64d8cSJesper Dangaard Brouer xdpf->len, DMA_TO_DEVICE); 40158b99ee3STariq Toukan xdp_return_frame_rx_napi(xdpf); 40258b99ee3STariq Toukan drops++; 40358b99ee3STariq Toukan } 40458b99ee3STariq Toukan } 40558b99ee3STariq Toukan 4065e0d2eefSTariq Toukan if (flags & XDP_XMIT_FLUSH) { 4075e0d2eefSTariq Toukan if (sq->mpwqe.wqe) 4085e0d2eefSTariq Toukan mlx5e_xdp_mpwqe_complete(sq); 40958b99ee3STariq Toukan mlx5e_xmit_xdp_doorbell(sq); 4105e0d2eefSTariq Toukan } 41158b99ee3STariq Toukan 41258b99ee3STariq Toukan return n - drops; 41358b99ee3STariq Toukan } 4144fb2f516STariq Toukan 4154fb2f516STariq Toukan void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq) 4164fb2f516STariq Toukan { 4174fb2f516STariq Toukan struct mlx5e_xdpsq *xdpsq = &rq->xdpsq; 4184fb2f516STariq Toukan 4195e0d2eefSTariq Toukan if (xdpsq->mpwqe.wqe) 4205e0d2eefSTariq Toukan mlx5e_xdp_mpwqe_complete(xdpsq); 4215e0d2eefSTariq Toukan 4224fb2f516STariq Toukan mlx5e_xmit_xdp_doorbell(xdpsq); 4234fb2f516STariq Toukan 42415143bf5STariq Toukan if (test_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags)) { 4254fb2f516STariq Toukan xdp_do_flush_map(); 42615143bf5STariq Toukan __clear_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags); 4274fb2f516STariq Toukan } 4284fb2f516STariq Toukan } 4295e0d2eefSTariq Toukan 4305e0d2eefSTariq Toukan void mlx5e_set_xmit_fp(struct mlx5e_xdpsq *sq, bool is_mpw) 4315e0d2eefSTariq Toukan { 4325e0d2eefSTariq Toukan sq->xmit_xdp_frame = is_mpw ? 4335e0d2eefSTariq Toukan mlx5e_xmit_xdp_frame_mpwqe : mlx5e_xmit_xdp_frame; 4345e0d2eefSTariq Toukan } 4355e0d2eefSTariq Toukan 436