1159d2131STariq Toukan /* 2159d2131STariq Toukan * Copyright (c) 2018, Mellanox Technologies. All rights reserved. 3159d2131STariq Toukan * 4159d2131STariq Toukan * This software is available to you under a choice of one of two 5159d2131STariq Toukan * licenses. You may choose to be licensed under the terms of the GNU 6159d2131STariq Toukan * General Public License (GPL) Version 2, available from the file 7159d2131STariq Toukan * COPYING in the main directory of this source tree, or the 8159d2131STariq Toukan * OpenIB.org BSD license below: 9159d2131STariq Toukan * 10159d2131STariq Toukan * Redistribution and use in source and binary forms, with or 11159d2131STariq Toukan * without modification, are permitted provided that the following 12159d2131STariq Toukan * conditions are met: 13159d2131STariq Toukan * 14159d2131STariq Toukan * - Redistributions of source code must retain the above 15159d2131STariq Toukan * copyright notice, this list of conditions and the following 16159d2131STariq Toukan * disclaimer. 17159d2131STariq Toukan * 18159d2131STariq Toukan * - Redistributions in binary form must reproduce the above 19159d2131STariq Toukan * copyright notice, this list of conditions and the following 20159d2131STariq Toukan * disclaimer in the documentation and/or other materials 21159d2131STariq Toukan * provided with the distribution. 22159d2131STariq Toukan * 23159d2131STariq Toukan * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24159d2131STariq Toukan * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25159d2131STariq Toukan * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26159d2131STariq Toukan * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27159d2131STariq Toukan * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28159d2131STariq Toukan * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29159d2131STariq Toukan * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30159d2131STariq Toukan * SOFTWARE. 31159d2131STariq Toukan */ 32159d2131STariq Toukan #ifndef __MLX5_EN_XDP_H__ 33159d2131STariq Toukan #define __MLX5_EN_XDP_H__ 34159d2131STariq Toukan 35159d2131STariq Toukan #include "en.h" 36159d2131STariq Toukan 37159d2131STariq Toukan #define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN) 385e0d2eefSTariq Toukan #define MLX5E_XDP_TX_EMPTY_DS_COUNT \ 395e0d2eefSTariq Toukan (sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS) 405e0d2eefSTariq Toukan #define MLX5E_XDP_TX_DS_COUNT (MLX5E_XDP_TX_EMPTY_DS_COUNT + 1 /* SG DS */) 41159d2131STariq Toukan 42d460c271SMaxim Mikityanskiy int mlx5e_xdp_max_mtu(struct mlx5e_params *params); 43159d2131STariq Toukan bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di, 44159d2131STariq Toukan void *va, u16 *rx_headroom, u32 *len); 45feb2ff9dSTariq Toukan bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq); 46feb2ff9dSTariq Toukan void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq); 475e0d2eefSTariq Toukan void mlx5e_set_xmit_fp(struct mlx5e_xdpsq *sq, bool is_mpw); 484fb2f516STariq Toukan void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq); 4958b99ee3STariq Toukan int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 5058b99ee3STariq Toukan u32 flags); 51159d2131STariq Toukan 52407e17b1SSaeed Mahameed static inline void mlx5e_xdp_tx_enable(struct mlx5e_priv *priv) 53407e17b1SSaeed Mahameed { 54407e17b1SSaeed Mahameed set_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state); 55407e17b1SSaeed Mahameed } 56407e17b1SSaeed Mahameed 57407e17b1SSaeed Mahameed static inline void mlx5e_xdp_tx_disable(struct mlx5e_priv *priv) 58407e17b1SSaeed Mahameed { 59407e17b1SSaeed Mahameed clear_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state); 60407e17b1SSaeed Mahameed /* let other device's napi(s) see our new state */ 61407e17b1SSaeed Mahameed synchronize_rcu(); 62407e17b1SSaeed Mahameed } 63407e17b1SSaeed Mahameed 64407e17b1SSaeed Mahameed static inline bool mlx5e_xdp_tx_is_enabled(struct mlx5e_priv *priv) 65407e17b1SSaeed Mahameed { 66407e17b1SSaeed Mahameed return test_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state); 67407e17b1SSaeed Mahameed } 68407e17b1SSaeed Mahameed 69159d2131STariq Toukan static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq) 70159d2131STariq Toukan { 71b8180392STariq Toukan if (sq->doorbell_cseg) { 72b8180392STariq Toukan mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, sq->doorbell_cseg); 73b8180392STariq Toukan sq->doorbell_cseg = NULL; 74b8180392STariq Toukan } 75159d2131STariq Toukan } 76159d2131STariq Toukan 77c2273219SShay Agroskin /* Enable inline WQEs to shift some load from a congested HCA (HW) to 78c2273219SShay Agroskin * a less congested cpu (SW). 79c2273219SShay Agroskin */ 80c2273219SShay Agroskin static inline void mlx5e_xdp_update_inline_state(struct mlx5e_xdpsq *sq) 81c2273219SShay Agroskin { 82c2273219SShay Agroskin u16 outstanding = sq->xdpi_fifo_pc - sq->xdpi_fifo_cc; 83c2273219SShay Agroskin struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; 84c2273219SShay Agroskin 85c2273219SShay Agroskin #define MLX5E_XDP_INLINE_WATERMARK_LOW 10 86c2273219SShay Agroskin #define MLX5E_XDP_INLINE_WATERMARK_HIGH 128 87c2273219SShay Agroskin 88c2273219SShay Agroskin if (session->inline_on) { 89c2273219SShay Agroskin if (outstanding <= MLX5E_XDP_INLINE_WATERMARK_LOW) 90c2273219SShay Agroskin session->inline_on = 0; 91c2273219SShay Agroskin return; 92c2273219SShay Agroskin } 93c2273219SShay Agroskin 94c2273219SShay Agroskin /* inline is false */ 95c2273219SShay Agroskin if (outstanding >= MLX5E_XDP_INLINE_WATERMARK_HIGH) 96c2273219SShay Agroskin session->inline_on = 1; 97c2273219SShay Agroskin } 98c2273219SShay Agroskin 99fea28dd6STariq Toukan static inline void 100d963fa15SMaxim Mikityanskiy mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq, 101d963fa15SMaxim Mikityanskiy struct mlx5e_xdp_xmit_data *xdptxd, 102c2273219SShay Agroskin struct mlx5e_xdpsq_stats *stats) 1035e0d2eefSTariq Toukan { 1045e0d2eefSTariq Toukan struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; 1055e0d2eefSTariq Toukan struct mlx5_wqe_data_seg *dseg = 106c2273219SShay Agroskin (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count; 107d963fa15SMaxim Mikityanskiy u32 dma_len = xdptxd->len; 1085e0d2eefSTariq Toukan 109c2273219SShay Agroskin session->pkt_count++; 110c2273219SShay Agroskin 111c2273219SShay Agroskin #define MLX5E_XDP_INLINE_WQE_SZ_THRSD (256 - sizeof(struct mlx5_wqe_inline_seg)) 112c2273219SShay Agroskin 113c2273219SShay Agroskin if (session->inline_on && dma_len <= MLX5E_XDP_INLINE_WQE_SZ_THRSD) { 114c2273219SShay Agroskin struct mlx5_wqe_inline_seg *inline_dseg = 115c2273219SShay Agroskin (struct mlx5_wqe_inline_seg *)dseg; 116c2273219SShay Agroskin u16 ds_len = sizeof(*inline_dseg) + dma_len; 117c2273219SShay Agroskin u16 ds_cnt = DIV_ROUND_UP(ds_len, MLX5_SEND_WQE_DS); 118c2273219SShay Agroskin 119c2273219SShay Agroskin if (unlikely(session->ds_count + ds_cnt > session->max_ds_count)) { 120c2273219SShay Agroskin /* Not enough space for inline wqe, send with memory pointer */ 121c2273219SShay Agroskin session->complete = true; 122c2273219SShay Agroskin goto no_inline; 123c2273219SShay Agroskin } 124c2273219SShay Agroskin 125c2273219SShay Agroskin inline_dseg->byte_count = cpu_to_be32(dma_len | MLX5_INLINE_SEG); 126d963fa15SMaxim Mikityanskiy memcpy(inline_dseg->data, xdptxd->data, dma_len); 127c2273219SShay Agroskin 128c2273219SShay Agroskin session->ds_count += ds_cnt; 129c2273219SShay Agroskin stats->inlnw++; 130c2273219SShay Agroskin return; 131c2273219SShay Agroskin } 132c2273219SShay Agroskin 133c2273219SShay Agroskin no_inline: 134d963fa15SMaxim Mikityanskiy dseg->addr = cpu_to_be64(xdptxd->dma_addr); 1355e0d2eefSTariq Toukan dseg->byte_count = cpu_to_be32(dma_len); 1365e0d2eefSTariq Toukan dseg->lkey = sq->mkey_be; 137c2273219SShay Agroskin session->ds_count++; 1385e0d2eefSTariq Toukan } 1395e0d2eefSTariq Toukan 1405e0d2eefSTariq Toukan static inline void mlx5e_xdpsq_fetch_wqe(struct mlx5e_xdpsq *sq, 1415e0d2eefSTariq Toukan struct mlx5e_tx_wqe **wqe) 1425e0d2eefSTariq Toukan { 1435e0d2eefSTariq Toukan struct mlx5_wq_cyc *wq = &sq->wq; 1445e0d2eefSTariq Toukan u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1455e0d2eefSTariq Toukan 1465e0d2eefSTariq Toukan *wqe = mlx5_wq_cyc_get_wqe(wq, pi); 1475e0d2eefSTariq Toukan memset(*wqe, 0, sizeof(**wqe)); 1485e0d2eefSTariq Toukan } 1495e0d2eefSTariq Toukan 1505e0d2eefSTariq Toukan static inline void 151fea28dd6STariq Toukan mlx5e_xdpi_fifo_push(struct mlx5e_xdp_info_fifo *fifo, 152fea28dd6STariq Toukan struct mlx5e_xdp_info *xi) 153fea28dd6STariq Toukan { 154fea28dd6STariq Toukan u32 i = (*fifo->pc)++ & fifo->mask; 155fea28dd6STariq Toukan 156fea28dd6STariq Toukan fifo->xi[i] = *xi; 157fea28dd6STariq Toukan } 158fea28dd6STariq Toukan 159fea28dd6STariq Toukan static inline struct mlx5e_xdp_info 160fea28dd6STariq Toukan mlx5e_xdpi_fifo_pop(struct mlx5e_xdp_info_fifo *fifo) 161fea28dd6STariq Toukan { 162fea28dd6STariq Toukan return fifo->xi[(*fifo->cc)++ & fifo->mask]; 163fea28dd6STariq Toukan } 164159d2131STariq Toukan #endif 165