1159d2131STariq Toukan /*
2159d2131STariq Toukan * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
3159d2131STariq Toukan *
4159d2131STariq Toukan * This software is available to you under a choice of one of two
5159d2131STariq Toukan * licenses. You may choose to be licensed under the terms of the GNU
6159d2131STariq Toukan * General Public License (GPL) Version 2, available from the file
7159d2131STariq Toukan * COPYING in the main directory of this source tree, or the
8159d2131STariq Toukan * OpenIB.org BSD license below:
9159d2131STariq Toukan *
10159d2131STariq Toukan * Redistribution and use in source and binary forms, with or
11159d2131STariq Toukan * without modification, are permitted provided that the following
12159d2131STariq Toukan * conditions are met:
13159d2131STariq Toukan *
14159d2131STariq Toukan * - Redistributions of source code must retain the above
15159d2131STariq Toukan * copyright notice, this list of conditions and the following
16159d2131STariq Toukan * disclaimer.
17159d2131STariq Toukan *
18159d2131STariq Toukan * - Redistributions in binary form must reproduce the above
19159d2131STariq Toukan * copyright notice, this list of conditions and the following
20159d2131STariq Toukan * disclaimer in the documentation and/or other materials
21159d2131STariq Toukan * provided with the distribution.
22159d2131STariq Toukan *
23159d2131STariq Toukan * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24159d2131STariq Toukan * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25159d2131STariq Toukan * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26159d2131STariq Toukan * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27159d2131STariq Toukan * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28159d2131STariq Toukan * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29159d2131STariq Toukan * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30159d2131STariq Toukan * SOFTWARE.
31159d2131STariq Toukan */
32159d2131STariq Toukan #ifndef __MLX5_EN_XDP_H__
33159d2131STariq Toukan #define __MLX5_EN_XDP_H__
34159d2131STariq Toukan
3593761ca1STariq Toukan #include <linux/indirect_call_wrapper.h>
3693761ca1STariq Toukan
37159d2131STariq Toukan #include "en.h"
38542578c6STariq Toukan #include "en/txrx.h"
39159d2131STariq Toukan
40159d2131STariq Toukan #define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
41159d2131STariq Toukan
42388a2b56SMaxim Mikityanskiy #define MLX5E_XDP_INLINE_WQE_MAX_DS_CNT 16
43388a2b56SMaxim Mikityanskiy #define MLX5E_XDP_INLINE_WQE_SZ_THRSD \
44388a2b56SMaxim Mikityanskiy (MLX5E_XDP_INLINE_WQE_MAX_DS_CNT * MLX5_SEND_WQE_DS - \
45388a2b56SMaxim Mikityanskiy sizeof(struct mlx5_wqe_inline_seg))
466c085a8aSShay Agroskin
47384a13caSToke Høiland-Jørgensen struct mlx5e_xdp_buff {
48384a13caSToke Høiland-Jørgensen struct xdp_buff xdp;
49bc8d405bSToke Høiland-Jørgensen struct mlx5_cqe64 *cqe;
50bc8d405bSToke Høiland-Jørgensen struct mlx5e_rq *rq;
51384a13caSToke Høiland-Jørgensen };
52384a13caSToke Høiland-Jørgensen
53aebc62d3STariq Toukan /* XDP packets can be transmitted in different ways. On completion, we need to
54aebc62d3STariq Toukan * distinguish between them to clean up things in a proper way.
55aebc62d3STariq Toukan */
56aebc62d3STariq Toukan enum mlx5e_xdp_xmit_mode {
57aebc62d3STariq Toukan /* An xdp_frame was transmitted due to either XDP_REDIRECT from another
58aebc62d3STariq Toukan * device or XDP_TX from an XSK RQ. The frame has to be unmapped and
59aebc62d3STariq Toukan * returned.
60aebc62d3STariq Toukan */
61aebc62d3STariq Toukan MLX5E_XDP_XMIT_MODE_FRAME,
62aebc62d3STariq Toukan
63aebc62d3STariq Toukan /* The xdp_frame was created in place as a result of XDP_TX from a
64aebc62d3STariq Toukan * regular RQ. No DMA remapping happened, and the page belongs to us.
65aebc62d3STariq Toukan */
66aebc62d3STariq Toukan MLX5E_XDP_XMIT_MODE_PAGE,
67aebc62d3STariq Toukan
68aebc62d3STariq Toukan /* No xdp_frame was created at all, the transmit happened from a UMEM
69aebc62d3STariq Toukan * page. The UMEM Completion Ring producer pointer has to be increased.
70aebc62d3STariq Toukan */
71aebc62d3STariq Toukan MLX5E_XDP_XMIT_MODE_XSK,
72aebc62d3STariq Toukan };
73aebc62d3STariq Toukan
743f734b8cSTariq Toukan /* xmit_mode entry is pushed to the fifo per packet, followed by multiple
753f734b8cSTariq Toukan * entries, as follows:
763f734b8cSTariq Toukan *
773f734b8cSTariq Toukan * MLX5E_XDP_XMIT_MODE_FRAME:
783f734b8cSTariq Toukan * xdpf, dma_addr_1, dma_addr_2, ... , dma_addr_num.
793f734b8cSTariq Toukan * 'num' is derived from xdpf.
803f734b8cSTariq Toukan *
813f734b8cSTariq Toukan * MLX5E_XDP_XMIT_MODE_PAGE:
823f734b8cSTariq Toukan * num, page_1, page_2, ... , page_num.
833f734b8cSTariq Toukan *
843f734b8cSTariq Toukan * MLX5E_XDP_XMIT_MODE_XSK:
853f734b8cSTariq Toukan * none.
863f734b8cSTariq Toukan */
87*34a79876SDragos Tatulea #define MLX5E_XDP_FIFO_ENTRIES2DS_MAX_RATIO 4
88*34a79876SDragos Tatulea
893f734b8cSTariq Toukan union mlx5e_xdp_info {
90aebc62d3STariq Toukan enum mlx5e_xdp_xmit_mode mode;
91aebc62d3STariq Toukan union {
92aebc62d3STariq Toukan struct xdp_frame *xdpf;
93aebc62d3STariq Toukan dma_addr_t dma_addr;
94aebc62d3STariq Toukan } frame;
953f734b8cSTariq Toukan union {
96aebc62d3STariq Toukan struct mlx5e_rq *rq;
973f734b8cSTariq Toukan u8 num;
98aebc62d3STariq Toukan struct page *page;
99aebc62d3STariq Toukan } page;
100aebc62d3STariq Toukan };
101aebc62d3STariq Toukan
102a011b49fSMaxim Mikityanskiy struct mlx5e_xsk_param;
103a011b49fSMaxim Mikityanskiy int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk);
1049da5294eSTariq Toukan bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
105384a13caSToke Høiland-Jørgensen struct bpf_prog *prog, struct mlx5e_xdp_buff *mlctx);
106db05815bSMaxim Mikityanskiy void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq);
107b9673cf5SMaxim Mikityanskiy bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
108b9673cf5SMaxim Mikityanskiy void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);
1095e0d2eefSTariq Toukan void mlx5e_set_xmit_fp(struct mlx5e_xdpsq *sq, bool is_mpw);
1104fb2f516STariq Toukan void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq);
11158b99ee3STariq Toukan int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
11258b99ee3STariq Toukan u32 flags);
113159d2131STariq Toukan
114bc8d405bSToke Høiland-Jørgensen extern const struct xdp_metadata_ops mlx5e_xdp_metadata_ops;
115bc8d405bSToke Høiland-Jørgensen
11693761ca1STariq Toukan INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,
117b39fe61eSMaxim Mikityanskiy struct mlx5e_xmit_data *xdptxd,
11893761ca1STariq Toukan int check_result));
11993761ca1STariq Toukan INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,
120b39fe61eSMaxim Mikityanskiy struct mlx5e_xmit_data *xdptxd,
12193761ca1STariq Toukan int check_result));
12293761ca1STariq Toukan INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq));
12393761ca1STariq Toukan INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq));
12493761ca1STariq Toukan
mlx5e_xdp_tx_enable(struct mlx5e_priv * priv)125407e17b1SSaeed Mahameed static inline void mlx5e_xdp_tx_enable(struct mlx5e_priv *priv)
126407e17b1SSaeed Mahameed {
127407e17b1SSaeed Mahameed set_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state);
1289cf88808SMaxim Mikityanskiy
1299cf88808SMaxim Mikityanskiy if (priv->channels.params.xdp_prog)
1309cf88808SMaxim Mikityanskiy set_bit(MLX5E_STATE_XDP_ACTIVE, &priv->state);
131407e17b1SSaeed Mahameed }
132407e17b1SSaeed Mahameed
mlx5e_xdp_tx_disable(struct mlx5e_priv * priv)133407e17b1SSaeed Mahameed static inline void mlx5e_xdp_tx_disable(struct mlx5e_priv *priv)
134407e17b1SSaeed Mahameed {
1359cf88808SMaxim Mikityanskiy if (priv->channels.params.xdp_prog)
1369cf88808SMaxim Mikityanskiy clear_bit(MLX5E_STATE_XDP_ACTIVE, &priv->state);
1379cf88808SMaxim Mikityanskiy
138407e17b1SSaeed Mahameed clear_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state);
1399cf88808SMaxim Mikityanskiy /* Let other device's napi(s) and XSK wakeups see our new state. */
1404d6e6b0cSMaxim Mikityanskiy synchronize_net();
141407e17b1SSaeed Mahameed }
142407e17b1SSaeed Mahameed
mlx5e_xdp_tx_is_enabled(struct mlx5e_priv * priv)143407e17b1SSaeed Mahameed static inline bool mlx5e_xdp_tx_is_enabled(struct mlx5e_priv *priv)
144407e17b1SSaeed Mahameed {
145407e17b1SSaeed Mahameed return test_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state);
146407e17b1SSaeed Mahameed }
147407e17b1SSaeed Mahameed
mlx5e_xdp_is_active(struct mlx5e_priv * priv)1489cf88808SMaxim Mikityanskiy static inline bool mlx5e_xdp_is_active(struct mlx5e_priv *priv)
149db05815bSMaxim Mikityanskiy {
1509cf88808SMaxim Mikityanskiy return test_bit(MLX5E_STATE_XDP_ACTIVE, &priv->state);
151db05815bSMaxim Mikityanskiy }
152db05815bSMaxim Mikityanskiy
mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq * sq)153159d2131STariq Toukan static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq)
154159d2131STariq Toukan {
155b8180392STariq Toukan if (sq->doorbell_cseg) {
156b8180392STariq Toukan mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, sq->doorbell_cseg);
157b8180392STariq Toukan sq->doorbell_cseg = NULL;
158b8180392STariq Toukan }
159159d2131STariq Toukan }
160159d2131STariq Toukan
161c2273219SShay Agroskin /* Enable inline WQEs to shift some load from a congested HCA (HW) to
162c2273219SShay Agroskin * a less congested cpu (SW).
163c2273219SShay Agroskin */
mlx5e_xdp_get_inline_state(struct mlx5e_xdpsq * sq,bool cur)16497e3afd6SMaxim Mikityanskiy static inline bool mlx5e_xdp_get_inline_state(struct mlx5e_xdpsq *sq, bool cur)
165c2273219SShay Agroskin {
166c2273219SShay Agroskin u16 outstanding = sq->xdpi_fifo_pc - sq->xdpi_fifo_cc;
167c2273219SShay Agroskin
168c2273219SShay Agroskin #define MLX5E_XDP_INLINE_WATERMARK_LOW 10
169c2273219SShay Agroskin #define MLX5E_XDP_INLINE_WATERMARK_HIGH 128
170c2273219SShay Agroskin
17197e3afd6SMaxim Mikityanskiy if (cur && outstanding <= MLX5E_XDP_INLINE_WATERMARK_LOW)
17297e3afd6SMaxim Mikityanskiy return false;
173c2273219SShay Agroskin
17497e3afd6SMaxim Mikityanskiy if (!cur && outstanding >= MLX5E_XDP_INLINE_WATERMARK_HIGH)
17597e3afd6SMaxim Mikityanskiy return true;
17697e3afd6SMaxim Mikityanskiy
17797e3afd6SMaxim Mikityanskiy return cur;
178c2273219SShay Agroskin }
179c2273219SShay Agroskin
mlx5e_xdp_mpwqe_is_full(struct mlx5e_tx_mpwqe * session,u8 max_sq_mpw_wqebbs)180e3c4c496SMaxim Mikityanskiy static inline bool mlx5e_xdp_mpwqe_is_full(struct mlx5e_tx_mpwqe *session, u8 max_sq_mpw_wqebbs)
1816c085a8aSShay Agroskin {
182388a2b56SMaxim Mikityanskiy if (session->inline_on)
183388a2b56SMaxim Mikityanskiy return session->ds_count + MLX5E_XDP_INLINE_WQE_MAX_DS_CNT >
18476c31e5fSAya Levin max_sq_mpw_wqebbs * MLX5_SEND_WQEBB_NUM_DS;
18576c31e5fSAya Levin
18676c31e5fSAya Levin return mlx5e_tx_mpwqe_is_full(session, max_sq_mpw_wqebbs);
1876c085a8aSShay Agroskin }
1886c085a8aSShay Agroskin
18905dfd570STariq Toukan struct mlx5e_xdp_wqe_info {
19005dfd570STariq Toukan u8 num_wqebbs;
19105dfd570STariq Toukan u8 num_pkts;
19205dfd570STariq Toukan };
19305dfd570STariq Toukan
1946c085a8aSShay Agroskin static inline void
mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq * sq,struct mlx5e_xmit_data * xdptxd,struct mlx5e_xdpsq_stats * stats)195d963fa15SMaxim Mikityanskiy mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq,
196b39fe61eSMaxim Mikityanskiy struct mlx5e_xmit_data *xdptxd,
197c2273219SShay Agroskin struct mlx5e_xdpsq_stats *stats)
1985e0d2eefSTariq Toukan {
199b39fe61eSMaxim Mikityanskiy struct mlx5e_tx_mpwqe *session = &sq->mpwqe;
2005e0d2eefSTariq Toukan struct mlx5_wqe_data_seg *dseg =
201c2273219SShay Agroskin (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count;
202d963fa15SMaxim Mikityanskiy u32 dma_len = xdptxd->len;
2035e0d2eefSTariq Toukan
204c2273219SShay Agroskin session->pkt_count++;
2055af75c74SMaxim Mikityanskiy session->bytes_count += dma_len;
206c2273219SShay Agroskin
207c2273219SShay Agroskin if (session->inline_on && dma_len <= MLX5E_XDP_INLINE_WQE_SZ_THRSD) {
208c2273219SShay Agroskin struct mlx5_wqe_inline_seg *inline_dseg =
209c2273219SShay Agroskin (struct mlx5_wqe_inline_seg *)dseg;
210c2273219SShay Agroskin u16 ds_len = sizeof(*inline_dseg) + dma_len;
211c2273219SShay Agroskin u16 ds_cnt = DIV_ROUND_UP(ds_len, MLX5_SEND_WQE_DS);
212c2273219SShay Agroskin
213c2273219SShay Agroskin inline_dseg->byte_count = cpu_to_be32(dma_len | MLX5_INLINE_SEG);
214d963fa15SMaxim Mikityanskiy memcpy(inline_dseg->data, xdptxd->data, dma_len);
215c2273219SShay Agroskin
216c2273219SShay Agroskin session->ds_count += ds_cnt;
217c2273219SShay Agroskin stats->inlnw++;
218c2273219SShay Agroskin return;
219c2273219SShay Agroskin }
220c2273219SShay Agroskin
221d963fa15SMaxim Mikityanskiy dseg->addr = cpu_to_be64(xdptxd->dma_addr);
2225e0d2eefSTariq Toukan dseg->byte_count = cpu_to_be32(dma_len);
2235e0d2eefSTariq Toukan dseg->lkey = sq->mkey_be;
224c2273219SShay Agroskin session->ds_count++;
2255e0d2eefSTariq Toukan }
2265e0d2eefSTariq Toukan
2275e0d2eefSTariq Toukan static inline void
mlx5e_xdpi_fifo_push(struct mlx5e_xdp_info_fifo * fifo,union mlx5e_xdp_info xi)228fea28dd6STariq Toukan mlx5e_xdpi_fifo_push(struct mlx5e_xdp_info_fifo *fifo,
2293f734b8cSTariq Toukan union mlx5e_xdp_info xi)
230fea28dd6STariq Toukan {
231fea28dd6STariq Toukan u32 i = (*fifo->pc)++ & fifo->mask;
232fea28dd6STariq Toukan
2333f734b8cSTariq Toukan fifo->xi[i] = xi;
234fea28dd6STariq Toukan }
235fea28dd6STariq Toukan
2363f734b8cSTariq Toukan static inline union mlx5e_xdp_info
mlx5e_xdpi_fifo_pop(struct mlx5e_xdp_info_fifo * fifo)237fea28dd6STariq Toukan mlx5e_xdpi_fifo_pop(struct mlx5e_xdp_info_fifo *fifo)
238fea28dd6STariq Toukan {
239fea28dd6STariq Toukan return fifo->xi[(*fifo->cc)++ & fifo->mask];
240fea28dd6STariq Toukan }
241159d2131STariq Toukan #endif
242