1159d2131STariq Toukan /* 2159d2131STariq Toukan * Copyright (c) 2018, Mellanox Technologies. All rights reserved. 3159d2131STariq Toukan * 4159d2131STariq Toukan * This software is available to you under a choice of one of two 5159d2131STariq Toukan * licenses. You may choose to be licensed under the terms of the GNU 6159d2131STariq Toukan * General Public License (GPL) Version 2, available from the file 7159d2131STariq Toukan * COPYING in the main directory of this source tree, or the 8159d2131STariq Toukan * OpenIB.org BSD license below: 9159d2131STariq Toukan * 10159d2131STariq Toukan * Redistribution and use in source and binary forms, with or 11159d2131STariq Toukan * without modification, are permitted provided that the following 12159d2131STariq Toukan * conditions are met: 13159d2131STariq Toukan * 14159d2131STariq Toukan * - Redistributions of source code must retain the above 15159d2131STariq Toukan * copyright notice, this list of conditions and the following 16159d2131STariq Toukan * disclaimer. 17159d2131STariq Toukan * 18159d2131STariq Toukan * - Redistributions in binary form must reproduce the above 19159d2131STariq Toukan * copyright notice, this list of conditions and the following 20159d2131STariq Toukan * disclaimer in the documentation and/or other materials 21159d2131STariq Toukan * provided with the distribution. 22159d2131STariq Toukan * 23159d2131STariq Toukan * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24159d2131STariq Toukan * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25159d2131STariq Toukan * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26159d2131STariq Toukan * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27159d2131STariq Toukan * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28159d2131STariq Toukan * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29159d2131STariq Toukan * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30159d2131STariq Toukan * SOFTWARE. 31159d2131STariq Toukan */ 32159d2131STariq Toukan 33159d2131STariq Toukan #include <linux/bpf_trace.h> 34159d2131STariq Toukan #include "en/xdp.h" 35159d2131STariq Toukan 36c94e4f11STariq Toukan static inline bool 37c94e4f11STariq Toukan mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_dma_info *di, 38c94e4f11STariq Toukan struct xdp_buff *xdp) 39c94e4f11STariq Toukan { 40c94e4f11STariq Toukan struct mlx5e_xdp_info xdpi; 41c94e4f11STariq Toukan 42c94e4f11STariq Toukan xdpi.xdpf = convert_to_xdp_frame(xdp); 43c94e4f11STariq Toukan if (unlikely(!xdpi.xdpf)) 44c94e4f11STariq Toukan return false; 45c94e4f11STariq Toukan xdpi.dma_addr = di->addr + (xdpi.xdpf->data - (void *)xdpi.xdpf); 46c94e4f11STariq Toukan dma_sync_single_for_device(sq->pdev, xdpi.dma_addr, 47c94e4f11STariq Toukan xdpi.xdpf->len, PCI_DMA_TODEVICE); 48c94e4f11STariq Toukan xdpi.di = *di; 49c94e4f11STariq Toukan 505e0d2eefSTariq Toukan return sq->xmit_xdp_frame(sq, &xdpi); 51c94e4f11STariq Toukan } 52c94e4f11STariq Toukan 53159d2131STariq Toukan /* returns true if packet was consumed by xdp */ 54159d2131STariq Toukan bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di, 55159d2131STariq Toukan void *va, u16 *rx_headroom, u32 *len) 56159d2131STariq Toukan { 57159d2131STariq Toukan struct bpf_prog *prog = READ_ONCE(rq->xdp_prog); 58159d2131STariq Toukan struct xdp_buff xdp; 59159d2131STariq Toukan u32 act; 60159d2131STariq Toukan int err; 61159d2131STariq Toukan 62159d2131STariq Toukan if (!prog) 63159d2131STariq Toukan return false; 64159d2131STariq Toukan 65159d2131STariq Toukan xdp.data = va + *rx_headroom; 66159d2131STariq Toukan xdp_set_data_meta_invalid(&xdp); 67159d2131STariq Toukan xdp.data_end = xdp.data + *len; 68159d2131STariq Toukan xdp.data_hard_start = va; 69159d2131STariq Toukan xdp.rxq = &rq->xdp_rxq; 70159d2131STariq Toukan 71159d2131STariq Toukan act = bpf_prog_run_xdp(prog, &xdp); 72159d2131STariq Toukan switch (act) { 73159d2131STariq Toukan case XDP_PASS: 74159d2131STariq Toukan *rx_headroom = xdp.data - xdp.data_hard_start; 75159d2131STariq Toukan *len = xdp.data_end - xdp.data; 76159d2131STariq Toukan return false; 77159d2131STariq Toukan case XDP_TX: 78c94e4f11STariq Toukan if (unlikely(!mlx5e_xmit_xdp_buff(&rq->xdpsq, di, &xdp))) 79c94e4f11STariq Toukan goto xdp_abort; 80c94e4f11STariq Toukan __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */ 81159d2131STariq Toukan return true; 82159d2131STariq Toukan case XDP_REDIRECT: 83159d2131STariq Toukan /* When XDP enabled then page-refcnt==1 here */ 84159d2131STariq Toukan err = xdp_do_redirect(rq->netdev, &xdp, prog); 85c94e4f11STariq Toukan if (unlikely(err)) 86c94e4f11STariq Toukan goto xdp_abort; 87159d2131STariq Toukan __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); 88dac0d15fSTariq Toukan rq->xdpsq.redirect_flush = true; 89159d2131STariq Toukan mlx5e_page_dma_unmap(rq, di); 9086690b4bSTariq Toukan rq->stats->xdp_redirect++; 91159d2131STariq Toukan return true; 92159d2131STariq Toukan default: 93159d2131STariq Toukan bpf_warn_invalid_xdp_action(act); 94e77f02b8SGustavo A. R. Silva /* fall through */ 95159d2131STariq Toukan case XDP_ABORTED: 96c94e4f11STariq Toukan xdp_abort: 97159d2131STariq Toukan trace_xdp_exception(rq->netdev, prog, act); 98e77f02b8SGustavo A. R. Silva /* fall through */ 99159d2131STariq Toukan case XDP_DROP: 100159d2131STariq Toukan rq->stats->xdp_drop++; 101159d2131STariq Toukan return true; 102159d2131STariq Toukan } 103159d2131STariq Toukan } 104159d2131STariq Toukan 1055e0d2eefSTariq Toukan static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq) 1065e0d2eefSTariq Toukan { 1075e0d2eefSTariq Toukan struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; 1085e0d2eefSTariq Toukan struct mlx5_wq_cyc *wq = &sq->wq; 1095e0d2eefSTariq Toukan u8 wqebbs; 1105e0d2eefSTariq Toukan u16 pi; 1115e0d2eefSTariq Toukan 1125e0d2eefSTariq Toukan mlx5e_xdpsq_fetch_wqe(sq, &session->wqe); 1135e0d2eefSTariq Toukan 1145e0d2eefSTariq Toukan prefetchw(session->wqe->data); 1155e0d2eefSTariq Toukan session->ds_count = MLX5E_XDP_TX_EMPTY_DS_COUNT; 1165e0d2eefSTariq Toukan 1175e0d2eefSTariq Toukan pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1185e0d2eefSTariq Toukan 1195e0d2eefSTariq Toukan /* The mult of MLX5_SEND_WQE_MAX_WQEBBS * MLX5_SEND_WQEBB_NUM_DS 1205e0d2eefSTariq Toukan * (16 * 4 == 64) does not fit in the 6-bit DS field of Ctrl Segment. 1215e0d2eefSTariq Toukan * We use a bound lower that MLX5_SEND_WQE_MAX_WQEBBS to let a 1225e0d2eefSTariq Toukan * full-session WQE be cache-aligned. 1235e0d2eefSTariq Toukan */ 1245e0d2eefSTariq Toukan #if L1_CACHE_BYTES < 128 1255e0d2eefSTariq Toukan #define MLX5E_XDP_MPW_MAX_WQEBBS (MLX5_SEND_WQE_MAX_WQEBBS - 1) 1265e0d2eefSTariq Toukan #else 1275e0d2eefSTariq Toukan #define MLX5E_XDP_MPW_MAX_WQEBBS (MLX5_SEND_WQE_MAX_WQEBBS - 2) 1285e0d2eefSTariq Toukan #endif 1295e0d2eefSTariq Toukan 1305e0d2eefSTariq Toukan wqebbs = min_t(u16, mlx5_wq_cyc_get_contig_wqebbs(wq, pi), 1315e0d2eefSTariq Toukan MLX5E_XDP_MPW_MAX_WQEBBS); 1325e0d2eefSTariq Toukan 1335e0d2eefSTariq Toukan session->max_ds_count = MLX5_SEND_WQEBB_NUM_DS * wqebbs; 1345e0d2eefSTariq Toukan } 1355e0d2eefSTariq Toukan 1365e0d2eefSTariq Toukan static void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq) 1375e0d2eefSTariq Toukan { 1385e0d2eefSTariq Toukan struct mlx5_wq_cyc *wq = &sq->wq; 1395e0d2eefSTariq Toukan struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; 1405e0d2eefSTariq Toukan struct mlx5_wqe_ctrl_seg *cseg = &session->wqe->ctrl; 1415e0d2eefSTariq Toukan u16 ds_count = session->ds_count; 1425e0d2eefSTariq Toukan u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1435e0d2eefSTariq Toukan struct mlx5e_xdp_wqe_info *wi = &sq->db.wqe_info[pi]; 1445e0d2eefSTariq Toukan 1455e0d2eefSTariq Toukan cseg->opmod_idx_opcode = 1465e0d2eefSTariq Toukan cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_ENHANCED_MPSW); 1475e0d2eefSTariq Toukan cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_count); 1485e0d2eefSTariq Toukan 1495e0d2eefSTariq Toukan wi->num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS); 1505e0d2eefSTariq Toukan wi->num_ds = ds_count - MLX5E_XDP_TX_EMPTY_DS_COUNT; 1515e0d2eefSTariq Toukan 1525e0d2eefSTariq Toukan sq->pc += wi->num_wqebbs; 1535e0d2eefSTariq Toukan 1545e0d2eefSTariq Toukan sq->doorbell_cseg = cseg; 1555e0d2eefSTariq Toukan 1565e0d2eefSTariq Toukan session->wqe = NULL; /* Close session */ 1575e0d2eefSTariq Toukan } 1585e0d2eefSTariq Toukan 1595e0d2eefSTariq Toukan static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, 1605e0d2eefSTariq Toukan struct mlx5e_xdp_info *xdpi) 1615e0d2eefSTariq Toukan { 1625e0d2eefSTariq Toukan struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; 1635e0d2eefSTariq Toukan struct mlx5e_xdpsq_stats *stats = sq->stats; 1645e0d2eefSTariq Toukan 1655e0d2eefSTariq Toukan dma_addr_t dma_addr = xdpi->dma_addr; 1665e0d2eefSTariq Toukan struct xdp_frame *xdpf = xdpi->xdpf; 1675e0d2eefSTariq Toukan unsigned int dma_len = xdpf->len; 1685e0d2eefSTariq Toukan 1695e0d2eefSTariq Toukan if (unlikely(sq->hw_mtu < dma_len)) { 1705e0d2eefSTariq Toukan stats->err++; 1715e0d2eefSTariq Toukan return false; 1725e0d2eefSTariq Toukan } 1735e0d2eefSTariq Toukan 1745e0d2eefSTariq Toukan if (unlikely(!session->wqe)) { 1755e0d2eefSTariq Toukan if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1765e0d2eefSTariq Toukan MLX5_SEND_WQE_MAX_WQEBBS))) { 1775e0d2eefSTariq Toukan /* SQ is full, ring doorbell */ 1785e0d2eefSTariq Toukan mlx5e_xmit_xdp_doorbell(sq); 1795e0d2eefSTariq Toukan stats->full++; 1805e0d2eefSTariq Toukan return false; 1815e0d2eefSTariq Toukan } 1825e0d2eefSTariq Toukan 1835e0d2eefSTariq Toukan mlx5e_xdp_mpwqe_session_start(sq); 1845e0d2eefSTariq Toukan } 1855e0d2eefSTariq Toukan 1865e0d2eefSTariq Toukan mlx5e_xdp_mpwqe_add_dseg(sq, dma_addr, dma_len); 1875e0d2eefSTariq Toukan 1885e0d2eefSTariq Toukan if (unlikely(session->ds_count == session->max_ds_count)) 1895e0d2eefSTariq Toukan mlx5e_xdp_mpwqe_complete(sq); 1905e0d2eefSTariq Toukan 1915e0d2eefSTariq Toukan mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi); 1925e0d2eefSTariq Toukan stats->xmit++; 1935e0d2eefSTariq Toukan return true; 1945e0d2eefSTariq Toukan } 1955e0d2eefSTariq Toukan 1965e0d2eefSTariq Toukan static bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_info *xdpi) 197159d2131STariq Toukan { 198159d2131STariq Toukan struct mlx5_wq_cyc *wq = &sq->wq; 199159d2131STariq Toukan u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 200159d2131STariq Toukan struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); 201159d2131STariq Toukan 202159d2131STariq Toukan struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 203159d2131STariq Toukan struct mlx5_wqe_eth_seg *eseg = &wqe->eth; 204c94e4f11STariq Toukan struct mlx5_wqe_data_seg *dseg = wqe->data; 205159d2131STariq Toukan 206c94e4f11STariq Toukan struct xdp_frame *xdpf = xdpi->xdpf; 207c94e4f11STariq Toukan dma_addr_t dma_addr = xdpi->dma_addr; 208c94e4f11STariq Toukan unsigned int dma_len = xdpf->len; 209159d2131STariq Toukan 210890388adSTariq Toukan struct mlx5e_xdpsq_stats *stats = sq->stats; 211159d2131STariq Toukan 212159d2131STariq Toukan prefetchw(wqe); 213159d2131STariq Toukan 214c94e4f11STariq Toukan if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE || sq->hw_mtu < dma_len)) { 215890388adSTariq Toukan stats->err++; 216159d2131STariq Toukan return false; 217159d2131STariq Toukan } 218159d2131STariq Toukan 219159d2131STariq Toukan if (unlikely(!mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1))) { 220159d2131STariq Toukan /* SQ is full, ring doorbell */ 221159d2131STariq Toukan mlx5e_xmit_xdp_doorbell(sq); 222890388adSTariq Toukan stats->full++; 223159d2131STariq Toukan return false; 224159d2131STariq Toukan } 225159d2131STariq Toukan 226159d2131STariq Toukan cseg->fm_ce_se = 0; 227159d2131STariq Toukan 228159d2131STariq Toukan /* copy the inline part if required */ 229159d2131STariq Toukan if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) { 230c94e4f11STariq Toukan memcpy(eseg->inline_hdr.start, xdpf->data, MLX5E_XDP_MIN_INLINE); 231159d2131STariq Toukan eseg->inline_hdr.sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE); 232159d2131STariq Toukan dma_len -= MLX5E_XDP_MIN_INLINE; 233159d2131STariq Toukan dma_addr += MLX5E_XDP_MIN_INLINE; 234159d2131STariq Toukan dseg++; 235159d2131STariq Toukan } 236159d2131STariq Toukan 237159d2131STariq Toukan /* write the dma part */ 238159d2131STariq Toukan dseg->addr = cpu_to_be64(dma_addr); 239159d2131STariq Toukan dseg->byte_count = cpu_to_be32(dma_len); 240159d2131STariq Toukan 241159d2131STariq Toukan cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_SEND); 242159d2131STariq Toukan 243159d2131STariq Toukan sq->pc++; 244159d2131STariq Toukan 245b8180392STariq Toukan sq->doorbell_cseg = cseg; 246159d2131STariq Toukan 247fea28dd6STariq Toukan mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi); 248890388adSTariq Toukan stats->xmit++; 249159d2131STariq Toukan return true; 250159d2131STariq Toukan } 251159d2131STariq Toukan 252feb2ff9dSTariq Toukan bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) 253159d2131STariq Toukan { 254fea28dd6STariq Toukan struct mlx5e_xdp_info_fifo *xdpi_fifo; 255159d2131STariq Toukan struct mlx5e_xdpsq *sq; 256159d2131STariq Toukan struct mlx5_cqe64 *cqe; 25758b99ee3STariq Toukan bool is_redirect; 258159d2131STariq Toukan u16 sqcc; 259159d2131STariq Toukan int i; 260159d2131STariq Toukan 261159d2131STariq Toukan sq = container_of(cq, struct mlx5e_xdpsq, cq); 262159d2131STariq Toukan 263159d2131STariq Toukan if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) 264159d2131STariq Toukan return false; 265159d2131STariq Toukan 266159d2131STariq Toukan cqe = mlx5_cqwq_get_cqe(&cq->wq); 267159d2131STariq Toukan if (!cqe) 268159d2131STariq Toukan return false; 269159d2131STariq Toukan 270feb2ff9dSTariq Toukan is_redirect = !rq; 271fea28dd6STariq Toukan xdpi_fifo = &sq->db.xdpi_fifo; 272159d2131STariq Toukan 273159d2131STariq Toukan /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), 274159d2131STariq Toukan * otherwise a cq overrun may occur 275159d2131STariq Toukan */ 276159d2131STariq Toukan sqcc = sq->cc; 277159d2131STariq Toukan 278159d2131STariq Toukan i = 0; 279159d2131STariq Toukan do { 280159d2131STariq Toukan u16 wqe_counter; 281159d2131STariq Toukan bool last_wqe; 282159d2131STariq Toukan 283159d2131STariq Toukan mlx5_cqwq_pop(&cq->wq); 284159d2131STariq Toukan 285159d2131STariq Toukan wqe_counter = be16_to_cpu(cqe->wqe_counter); 286159d2131STariq Toukan 287db02a308STariq Toukan if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) 288db02a308STariq Toukan netdev_WARN_ONCE(sq->channel->netdev, 289db02a308STariq Toukan "Bad OP in XDPSQ CQE: 0x%x\n", 290db02a308STariq Toukan get_cqe_opcode(cqe)); 291db02a308STariq Toukan 292159d2131STariq Toukan do { 2931feeab80STariq Toukan struct mlx5e_xdp_wqe_info *wi; 2941feeab80STariq Toukan u16 ci, j; 295159d2131STariq Toukan 296159d2131STariq Toukan last_wqe = (sqcc == wqe_counter); 2971feeab80STariq Toukan ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); 2981feeab80STariq Toukan wi = &sq->db.wqe_info[ci]; 2991feeab80STariq Toukan 3001feeab80STariq Toukan sqcc += wi->num_wqebbs; 3011feeab80STariq Toukan 3021feeab80STariq Toukan for (j = 0; j < wi->num_ds; j++) { 3031feeab80STariq Toukan struct mlx5e_xdp_info xdpi = 3041feeab80STariq Toukan mlx5e_xdpi_fifo_pop(xdpi_fifo); 30558b99ee3STariq Toukan 30658b99ee3STariq Toukan if (is_redirect) { 307fea28dd6STariq Toukan xdp_return_frame(xdpi.xdpf); 308fea28dd6STariq Toukan dma_unmap_single(sq->pdev, xdpi.dma_addr, 309fea28dd6STariq Toukan xdpi.xdpf->len, DMA_TO_DEVICE); 31058b99ee3STariq Toukan } else { 311159d2131STariq Toukan /* Recycle RX page */ 312fea28dd6STariq Toukan mlx5e_page_release(rq, &xdpi.di, true); 31358b99ee3STariq Toukan } 3141feeab80STariq Toukan } 315159d2131STariq Toukan } while (!last_wqe); 316159d2131STariq Toukan } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq))); 317159d2131STariq Toukan 318890388adSTariq Toukan sq->stats->cqes += i; 319159d2131STariq Toukan 320159d2131STariq Toukan mlx5_cqwq_update_db_record(&cq->wq); 321159d2131STariq Toukan 322159d2131STariq Toukan /* ensure cq space is freed before enabling more cqes */ 323159d2131STariq Toukan wmb(); 324159d2131STariq Toukan 325159d2131STariq Toukan sq->cc = sqcc; 326159d2131STariq Toukan return (i == MLX5E_TX_CQ_POLL_BUDGET); 327159d2131STariq Toukan } 328159d2131STariq Toukan 329feb2ff9dSTariq Toukan void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq) 330159d2131STariq Toukan { 331fea28dd6STariq Toukan struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo; 332feb2ff9dSTariq Toukan bool is_redirect = !rq; 333159d2131STariq Toukan 334159d2131STariq Toukan while (sq->cc != sq->pc) { 3351feeab80STariq Toukan struct mlx5e_xdp_wqe_info *wi; 3361feeab80STariq Toukan u16 ci, i; 3371feeab80STariq Toukan 3381feeab80STariq Toukan ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc); 3391feeab80STariq Toukan wi = &sq->db.wqe_info[ci]; 3401feeab80STariq Toukan 3411feeab80STariq Toukan sq->cc += wi->num_wqebbs; 3421feeab80STariq Toukan 3431feeab80STariq Toukan for (i = 0; i < wi->num_ds; i++) { 344fea28dd6STariq Toukan struct mlx5e_xdp_info xdpi = 345fea28dd6STariq Toukan mlx5e_xdpi_fifo_pop(xdpi_fifo); 34658b99ee3STariq Toukan 34758b99ee3STariq Toukan if (is_redirect) { 348fea28dd6STariq Toukan xdp_return_frame(xdpi.xdpf); 349fea28dd6STariq Toukan dma_unmap_single(sq->pdev, xdpi.dma_addr, 350fea28dd6STariq Toukan xdpi.xdpf->len, DMA_TO_DEVICE); 35158b99ee3STariq Toukan } else { 35258b99ee3STariq Toukan /* Recycle RX page */ 353fea28dd6STariq Toukan mlx5e_page_release(rq, &xdpi.di, false); 354159d2131STariq Toukan } 355159d2131STariq Toukan } 35658b99ee3STariq Toukan } 3571feeab80STariq Toukan } 358159d2131STariq Toukan 35958b99ee3STariq Toukan int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 36058b99ee3STariq Toukan u32 flags) 36158b99ee3STariq Toukan { 36258b99ee3STariq Toukan struct mlx5e_priv *priv = netdev_priv(dev); 36358b99ee3STariq Toukan struct mlx5e_xdpsq *sq; 36458b99ee3STariq Toukan int drops = 0; 36558b99ee3STariq Toukan int sq_num; 36658b99ee3STariq Toukan int i; 36758b99ee3STariq Toukan 368407e17b1SSaeed Mahameed /* this flag is sufficient, no need to test internal sq state */ 369407e17b1SSaeed Mahameed if (unlikely(!mlx5e_xdp_tx_is_enabled(priv))) 37058b99ee3STariq Toukan return -ENETDOWN; 37158b99ee3STariq Toukan 37258b99ee3STariq Toukan if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 37358b99ee3STariq Toukan return -EINVAL; 37458b99ee3STariq Toukan 37558b99ee3STariq Toukan sq_num = smp_processor_id(); 37658b99ee3STariq Toukan 37758b99ee3STariq Toukan if (unlikely(sq_num >= priv->channels.num)) 37858b99ee3STariq Toukan return -ENXIO; 37958b99ee3STariq Toukan 38058b99ee3STariq Toukan sq = &priv->channels.c[sq_num]->xdpsq; 38158b99ee3STariq Toukan 38258b99ee3STariq Toukan for (i = 0; i < n; i++) { 38358b99ee3STariq Toukan struct xdp_frame *xdpf = frames[i]; 38458b99ee3STariq Toukan struct mlx5e_xdp_info xdpi; 38558b99ee3STariq Toukan 38658b99ee3STariq Toukan xdpi.dma_addr = dma_map_single(sq->pdev, xdpf->data, xdpf->len, 38758b99ee3STariq Toukan DMA_TO_DEVICE); 38858b99ee3STariq Toukan if (unlikely(dma_mapping_error(sq->pdev, xdpi.dma_addr))) { 38939c64d8cSJesper Dangaard Brouer xdp_return_frame_rx_napi(xdpf); 39058b99ee3STariq Toukan drops++; 39158b99ee3STariq Toukan continue; 39258b99ee3STariq Toukan } 39358b99ee3STariq Toukan 39458b99ee3STariq Toukan xdpi.xdpf = xdpf; 39558b99ee3STariq Toukan 3965e0d2eefSTariq Toukan if (unlikely(!sq->xmit_xdp_frame(sq, &xdpi))) { 39739c64d8cSJesper Dangaard Brouer dma_unmap_single(sq->pdev, xdpi.dma_addr, 39839c64d8cSJesper Dangaard Brouer xdpf->len, DMA_TO_DEVICE); 39958b99ee3STariq Toukan xdp_return_frame_rx_napi(xdpf); 40058b99ee3STariq Toukan drops++; 40158b99ee3STariq Toukan } 40258b99ee3STariq Toukan } 40358b99ee3STariq Toukan 4045e0d2eefSTariq Toukan if (flags & XDP_XMIT_FLUSH) { 4055e0d2eefSTariq Toukan if (sq->mpwqe.wqe) 4065e0d2eefSTariq Toukan mlx5e_xdp_mpwqe_complete(sq); 40758b99ee3STariq Toukan mlx5e_xmit_xdp_doorbell(sq); 4085e0d2eefSTariq Toukan } 40958b99ee3STariq Toukan 41058b99ee3STariq Toukan return n - drops; 41158b99ee3STariq Toukan } 4124fb2f516STariq Toukan 4134fb2f516STariq Toukan void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq) 4144fb2f516STariq Toukan { 4154fb2f516STariq Toukan struct mlx5e_xdpsq *xdpsq = &rq->xdpsq; 4164fb2f516STariq Toukan 4175e0d2eefSTariq Toukan if (xdpsq->mpwqe.wqe) 4185e0d2eefSTariq Toukan mlx5e_xdp_mpwqe_complete(xdpsq); 4195e0d2eefSTariq Toukan 4204fb2f516STariq Toukan mlx5e_xmit_xdp_doorbell(xdpsq); 4214fb2f516STariq Toukan 4224fb2f516STariq Toukan if (xdpsq->redirect_flush) { 4234fb2f516STariq Toukan xdp_do_flush_map(); 4244fb2f516STariq Toukan xdpsq->redirect_flush = false; 4254fb2f516STariq Toukan } 4264fb2f516STariq Toukan } 4275e0d2eefSTariq Toukan 4285e0d2eefSTariq Toukan void mlx5e_set_xmit_fp(struct mlx5e_xdpsq *sq, bool is_mpw) 4295e0d2eefSTariq Toukan { 4305e0d2eefSTariq Toukan sq->xmit_xdp_frame = is_mpw ? 4315e0d2eefSTariq Toukan mlx5e_xmit_xdp_frame_mpwqe : mlx5e_xmit_xdp_frame; 4325e0d2eefSTariq Toukan } 4335e0d2eefSTariq Toukan 434