1542578c6STariq Toukan /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2542578c6STariq Toukan /* Copyright (c) 2019 Mellanox Technologies. */
3542578c6STariq Toukan
4542578c6STariq Toukan #ifndef __MLX5_EN_TXRX_H___
5542578c6STariq Toukan #define __MLX5_EN_TXRX_H___
6542578c6STariq Toukan
7542578c6STariq Toukan #include "en.h"
85d0b8476STariq Toukan #include <linux/indirect_call_wrapper.h>
9542578c6STariq Toukan
1097e3afd6SMaxim Mikityanskiy #define MLX5E_TX_WQE_EMPTY_DS_COUNT (sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS)
1197e3afd6SMaxim Mikityanskiy
12542578c6STariq Toukan #define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start))
13542578c6STariq Toukan
14f9c955b4SMaxim Mikityanskiy /* IPSEC inline data includes:
15f9c955b4SMaxim Mikityanskiy * 1. ESP trailer: up to 255 bytes of padding, 1 byte for pad length, 1 byte for
16f9c955b4SMaxim Mikityanskiy * next header.
17f9c955b4SMaxim Mikityanskiy * 2. ESP authentication data: 16 bytes for ICV.
18f9c955b4SMaxim Mikityanskiy */
19f9c955b4SMaxim Mikityanskiy #define MLX5E_MAX_TX_IPSEC_DS DIV_ROUND_UP(sizeof(struct mlx5_wqe_inline_seg) + \
20f9c955b4SMaxim Mikityanskiy 255 + 1 + 1 + 16, MLX5_SEND_WQE_DS)
21f9c955b4SMaxim Mikityanskiy
22f9c955b4SMaxim Mikityanskiy /* 366 should be big enough to cover all L2, L3 and L4 headers with possible
23f9c955b4SMaxim Mikityanskiy * encapsulations.
24f9c955b4SMaxim Mikityanskiy */
25f9c955b4SMaxim Mikityanskiy #define MLX5E_MAX_TX_INLINE_DS DIV_ROUND_UP(366 - INL_HDR_START_SZ + VLAN_HLEN, \
26f9c955b4SMaxim Mikityanskiy MLX5_SEND_WQE_DS)
27f9c955b4SMaxim Mikityanskiy
28f9c955b4SMaxim Mikityanskiy /* Sync the calculation with mlx5e_sq_calc_wqe_attr. */
29f9c955b4SMaxim Mikityanskiy #define MLX5E_MAX_TX_WQEBBS DIV_ROUND_UP(MLX5E_TX_WQE_EMPTY_DS_COUNT + \
30f9c955b4SMaxim Mikityanskiy MLX5E_MAX_TX_INLINE_DS + \
31f9c955b4SMaxim Mikityanskiy MLX5E_MAX_TX_IPSEC_DS + \
32f9c955b4SMaxim Mikityanskiy MAX_SKB_FRAGS + 1, \
33f9c955b4SMaxim Mikityanskiy MLX5_SEND_WQEBB_NUM_DS)
34f9c955b4SMaxim Mikityanskiy
35cecaa6a7SEran Ben Elisha #define MLX5E_RX_ERR_CQE(cqe) (get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)
36cecaa6a7SEran Ben Elisha
37432119deSAya Levin static inline
mlx5e_cqe_ts_to_ns(cqe_ts_to_ns func,struct mlx5_clock * clock,u64 cqe_ts)38432119deSAya Levin ktime_t mlx5e_cqe_ts_to_ns(cqe_ts_to_ns func, struct mlx5_clock *clock, u64 cqe_ts)
39432119deSAya Levin {
40432119deSAya Levin return INDIRECT_CALL_2(func, mlx5_real_time_cyc2time, mlx5_timecounter_cyc2time,
41432119deSAya Levin clock, cqe_ts);
42432119deSAya Levin }
43432119deSAya Levin
4428bff095STariq Toukan enum mlx5e_icosq_wqe_type {
4528bff095STariq Toukan MLX5E_ICOSQ_WQE_NOP,
4628bff095STariq Toukan MLX5E_ICOSQ_WQE_UMR_RX,
4764509b05SBen Ben-Ishay MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR,
481182f365STariq Toukan #ifdef CONFIG_MLX5_EN_TLS
491182f365STariq Toukan MLX5E_ICOSQ_WQE_UMR_TLS,
501182f365STariq Toukan MLX5E_ICOSQ_WQE_SET_PSV_TLS,
510419d8c9STariq Toukan MLX5E_ICOSQ_WQE_GET_PSV_TLS,
521182f365STariq Toukan #endif
5328bff095STariq Toukan };
5428bff095STariq Toukan
55b307f7f1STariq Toukan /* General */
mlx5e_skb_is_multicast(struct sk_buff * skb)5647c97e6bSRon Diskin static inline bool mlx5e_skb_is_multicast(struct sk_buff *skb)
5747c97e6bSRon Diskin {
5847c97e6bSRon Diskin return skb->pkt_type == PACKET_MULTICAST || skb->pkt_type == PACKET_BROADCAST;
5947c97e6bSRon Diskin }
6047c97e6bSRon Diskin
61b307f7f1STariq Toukan void mlx5e_trigger_irq(struct mlx5e_icosq *sq);
62b307f7f1STariq Toukan void mlx5e_completion_event(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe);
63b307f7f1STariq Toukan void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
64b307f7f1STariq Toukan int mlx5e_napi_poll(struct napi_struct *napi, int budget);
65b307f7f1STariq Toukan int mlx5e_poll_ico_cq(struct mlx5e_cq *cq);
66b307f7f1STariq Toukan
67b307f7f1STariq Toukan /* RX */
685d0b8476STariq Toukan INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq));
695d0b8476STariq Toukan INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq));
70b307f7f1STariq Toukan int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);
71b307f7f1STariq Toukan void mlx5e_free_rx_descs(struct mlx5e_rq *rq);
724c2a1323SDragos Tatulea void mlx5e_free_rx_missing_descs(struct mlx5e_rq *rq);
73b307f7f1STariq Toukan
mlx5e_rx_hw_stamp(struct hwtstamp_config * config)74bc8d405bSToke Høiland-Jørgensen static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
75bc8d405bSToke Høiland-Jørgensen {
76bc8d405bSToke Høiland-Jørgensen return config->rx_filter == HWTSTAMP_FILTER_ALL;
77bc8d405bSToke Høiland-Jørgensen }
78bc8d405bSToke Høiland-Jørgensen
79b307f7f1STariq Toukan /* TX */
80e32654f1STariq Toukan struct mlx5e_xmit_data {
81e32654f1STariq Toukan dma_addr_t dma_addr;
82e32654f1STariq Toukan void *data;
83eb9b9fdcSTariq Toukan u32 len : 31;
84eb9b9fdcSTariq Toukan u32 has_frags : 1;
85eb9b9fdcSTariq Toukan };
86eb9b9fdcSTariq Toukan
87eb9b9fdcSTariq Toukan struct mlx5e_xmit_data_frags {
88eb9b9fdcSTariq Toukan struct mlx5e_xmit_data xd;
89eb9b9fdcSTariq Toukan struct skb_shared_info *sinfo;
90c1783e74STariq Toukan dma_addr_t *dma_arr;
91e32654f1STariq Toukan };
92e32654f1STariq Toukan
93b307f7f1STariq Toukan netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev);
94b307f7f1STariq Toukan bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);
95b307f7f1STariq Toukan void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq);
96b307f7f1STariq Toukan
97542578c6STariq Toukan static inline bool
mlx5e_skb_fifo_has_room(struct mlx5e_skb_fifo * fifo)9819b43a43SAya Levin mlx5e_skb_fifo_has_room(struct mlx5e_skb_fifo *fifo)
9919b43a43SAya Levin {
100aa98d15eSRahul Rameshbabu return (u16)(*fifo->pc - *fifo->cc) <= fifo->mask;
10119b43a43SAya Levin }
10219b43a43SAya Levin
10319b43a43SAya Levin static inline bool
mlx5e_wqc_has_room_for(struct mlx5_wq_cyc * wq,u16 cc,u16 pc,u16 n)104542578c6STariq Toukan mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n)
105542578c6STariq Toukan {
106542578c6STariq Toukan return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc);
107542578c6STariq Toukan }
108542578c6STariq Toukan
mlx5e_fetch_wqe(struct mlx5_wq_cyc * wq,u16 pi,size_t wqe_size)109fed0c6cfSMaxim Mikityanskiy static inline void *mlx5e_fetch_wqe(struct mlx5_wq_cyc *wq, u16 pi, size_t wqe_size)
110542578c6STariq Toukan {
111fd1b2259STariq Toukan void *wqe;
112542578c6STariq Toukan
113fed0c6cfSMaxim Mikityanskiy wqe = mlx5_wq_cyc_get_wqe(wq, pi);
114fed0c6cfSMaxim Mikityanskiy memset(wqe, 0, wqe_size);
115fd1b2259STariq Toukan
116fd1b2259STariq Toukan return wqe;
117542578c6STariq Toukan }
118542578c6STariq Toukan
119fed0c6cfSMaxim Mikityanskiy #define MLX5E_TX_FETCH_WQE(sq, pi) \
120fed0c6cfSMaxim Mikityanskiy ((struct mlx5e_tx_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_tx_wqe)))
121fed0c6cfSMaxim Mikityanskiy
122542578c6STariq Toukan static inline struct mlx5e_tx_wqe *
mlx5e_post_nop(struct mlx5_wq_cyc * wq,u32 sqn,u16 * pc)123542578c6STariq Toukan mlx5e_post_nop(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
124542578c6STariq Toukan {
125542578c6STariq Toukan u16 pi = mlx5_wq_cyc_ctr2ix(wq, *pc);
126542578c6STariq Toukan struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
127542578c6STariq Toukan struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
128542578c6STariq Toukan
129542578c6STariq Toukan memset(cseg, 0, sizeof(*cseg));
130542578c6STariq Toukan
131542578c6STariq Toukan cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP);
132542578c6STariq Toukan cseg->qpn_ds = cpu_to_be32((sqn << 8) | 0x01);
133542578c6STariq Toukan
134542578c6STariq Toukan (*pc)++;
135542578c6STariq Toukan
136542578c6STariq Toukan return wqe;
137542578c6STariq Toukan }
138542578c6STariq Toukan
13937badd15STariq Toukan static inline struct mlx5e_tx_wqe *
mlx5e_post_nop_fence(struct mlx5_wq_cyc * wq,u32 sqn,u16 * pc)14037badd15STariq Toukan mlx5e_post_nop_fence(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
14137badd15STariq Toukan {
14237badd15STariq Toukan u16 pi = mlx5_wq_cyc_ctr2ix(wq, *pc);
14337badd15STariq Toukan struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
14437badd15STariq Toukan struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
14537badd15STariq Toukan
14637badd15STariq Toukan memset(cseg, 0, sizeof(*cseg));
14737badd15STariq Toukan
14837badd15STariq Toukan cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP);
14937badd15STariq Toukan cseg->qpn_ds = cpu_to_be32((sqn << 8) | 0x01);
15037badd15STariq Toukan cseg->fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL;
15137badd15STariq Toukan
15237badd15STariq Toukan (*pc)++;
15337badd15STariq Toukan
15437badd15STariq Toukan return wqe;
15537badd15STariq Toukan }
15637badd15STariq Toukan
15705dfd570STariq Toukan struct mlx5e_tx_wqe_info {
15805dfd570STariq Toukan struct sk_buff *skb;
15905dfd570STariq Toukan u32 num_bytes;
16005dfd570STariq Toukan u8 num_wqebbs;
16105dfd570STariq Toukan u8 num_dma;
162338c46c6SMaxim Mikityanskiy u8 num_fifo_pkts;
16305dfd570STariq Toukan #ifdef CONFIG_MLX5_EN_TLS
16405dfd570STariq Toukan struct page *resync_dump_frag_page;
16505dfd570STariq Toukan #endif
16605dfd570STariq Toukan };
16705dfd570STariq Toukan
mlx5e_txqsq_get_next_pi(struct mlx5e_txqsq * sq,u16 size)168ec9cdca0SMaxim Mikityanskiy static inline u16 mlx5e_txqsq_get_next_pi(struct mlx5e_txqsq *sq, u16 size)
169ec9cdca0SMaxim Mikityanskiy {
170ec9cdca0SMaxim Mikityanskiy struct mlx5_wq_cyc *wq = &sq->wq;
171ec9cdca0SMaxim Mikityanskiy u16 pi, contig_wqebbs;
172ec9cdca0SMaxim Mikityanskiy
173ec9cdca0SMaxim Mikityanskiy pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
174ec9cdca0SMaxim Mikityanskiy contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
175ec9cdca0SMaxim Mikityanskiy if (unlikely(contig_wqebbs < size)) {
176ec9cdca0SMaxim Mikityanskiy struct mlx5e_tx_wqe_info *wi, *edge_wi;
177ec9cdca0SMaxim Mikityanskiy
178ec9cdca0SMaxim Mikityanskiy wi = &sq->db.wqe_info[pi];
179ec9cdca0SMaxim Mikityanskiy edge_wi = wi + contig_wqebbs;
180ec9cdca0SMaxim Mikityanskiy
181ec9cdca0SMaxim Mikityanskiy /* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */
182ec9cdca0SMaxim Mikityanskiy for (; wi < edge_wi; wi++) {
183ec9cdca0SMaxim Mikityanskiy *wi = (struct mlx5e_tx_wqe_info) {
184ec9cdca0SMaxim Mikityanskiy .num_wqebbs = 1,
185ec9cdca0SMaxim Mikityanskiy };
186ec9cdca0SMaxim Mikityanskiy mlx5e_post_nop(wq, sq->sqn, &sq->pc);
187ec9cdca0SMaxim Mikityanskiy }
188ec9cdca0SMaxim Mikityanskiy sq->stats->nop += contig_wqebbs;
189ec9cdca0SMaxim Mikityanskiy
190ec9cdca0SMaxim Mikityanskiy pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
191ec9cdca0SMaxim Mikityanskiy }
192ec9cdca0SMaxim Mikityanskiy
193ec9cdca0SMaxim Mikityanskiy return pi;
194ec9cdca0SMaxim Mikityanskiy }
195ec9cdca0SMaxim Mikityanskiy
196*7aa50380SRahul Rameshbabu void mlx5e_txqsq_wake(struct mlx5e_txqsq *sq);
197*7aa50380SRahul Rameshbabu
mlx5e_shampo_get_cqe_header_index(struct mlx5e_rq * rq,struct mlx5_cqe64 * cqe)198b8d91145SKhalid Manaa static inline u16 mlx5e_shampo_get_cqe_header_index(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
199b8d91145SKhalid Manaa {
200b8d91145SKhalid Manaa return be16_to_cpu(cqe->shampo.header_entry_index) & (rq->mpwqe.shampo->hd_per_wq - 1);
201b8d91145SKhalid Manaa }
202b8d91145SKhalid Manaa
20364509b05SBen Ben-Ishay struct mlx5e_shampo_umr {
20464509b05SBen Ben-Ishay u16 len;
20564509b05SBen Ben-Ishay };
20664509b05SBen Ben-Ishay
20705dfd570STariq Toukan struct mlx5e_icosq_wqe_info {
20828bff095STariq Toukan u8 wqe_type;
20905dfd570STariq Toukan u8 num_wqebbs;
21005dfd570STariq Toukan
21128bff095STariq Toukan /* Auxiliary data for different wqe types. */
21205dfd570STariq Toukan union {
21305dfd570STariq Toukan struct {
21405dfd570STariq Toukan struct mlx5e_rq *rq;
21505dfd570STariq Toukan } umr;
21664509b05SBen Ben-Ishay struct mlx5e_shampo_umr shampo;
2171182f365STariq Toukan #ifdef CONFIG_MLX5_EN_TLS
2181182f365STariq Toukan struct {
2191182f365STariq Toukan struct mlx5e_ktls_offload_context_rx *priv_rx;
2201182f365STariq Toukan } tls_set_params;
2210419d8c9STariq Toukan struct {
2220419d8c9STariq Toukan struct mlx5e_ktls_rx_resync_buf *buf;
2230419d8c9STariq Toukan } tls_get_params;
2241182f365STariq Toukan #endif
22505dfd570STariq Toukan };
22605dfd570STariq Toukan };
22705dfd570STariq Toukan
2281182f365STariq Toukan void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq);
2291182f365STariq Toukan
mlx5e_icosq_get_next_pi(struct mlx5e_icosq * sq,u16 size)230ec9cdca0SMaxim Mikityanskiy static inline u16 mlx5e_icosq_get_next_pi(struct mlx5e_icosq *sq, u16 size)
231ec9cdca0SMaxim Mikityanskiy {
232ec9cdca0SMaxim Mikityanskiy struct mlx5_wq_cyc *wq = &sq->wq;
233ec9cdca0SMaxim Mikityanskiy u16 pi, contig_wqebbs;
234ec9cdca0SMaxim Mikityanskiy
235ec9cdca0SMaxim Mikityanskiy pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
236ec9cdca0SMaxim Mikityanskiy contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
237ec9cdca0SMaxim Mikityanskiy if (unlikely(contig_wqebbs < size)) {
238ec9cdca0SMaxim Mikityanskiy struct mlx5e_icosq_wqe_info *wi, *edge_wi;
239ec9cdca0SMaxim Mikityanskiy
240ec9cdca0SMaxim Mikityanskiy wi = &sq->db.wqe_info[pi];
241ec9cdca0SMaxim Mikityanskiy edge_wi = wi + contig_wqebbs;
242ec9cdca0SMaxim Mikityanskiy
243ec9cdca0SMaxim Mikityanskiy /* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */
244ec9cdca0SMaxim Mikityanskiy for (; wi < edge_wi; wi++) {
245ec9cdca0SMaxim Mikityanskiy *wi = (struct mlx5e_icosq_wqe_info) {
24628bff095STariq Toukan .wqe_type = MLX5E_ICOSQ_WQE_NOP,
247ec9cdca0SMaxim Mikityanskiy .num_wqebbs = 1,
248ec9cdca0SMaxim Mikityanskiy };
249ec9cdca0SMaxim Mikityanskiy mlx5e_post_nop(wq, sq->sqn, &sq->pc);
250ec9cdca0SMaxim Mikityanskiy }
251ec9cdca0SMaxim Mikityanskiy
252ec9cdca0SMaxim Mikityanskiy pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
253ec9cdca0SMaxim Mikityanskiy }
254ec9cdca0SMaxim Mikityanskiy
255ec9cdca0SMaxim Mikityanskiy return pi;
256ec9cdca0SMaxim Mikityanskiy }
257ec9cdca0SMaxim Mikityanskiy
258542578c6STariq Toukan static inline void
mlx5e_notify_hw(struct mlx5_wq_cyc * wq,u16 pc,void __iomem * uar_map,struct mlx5_wqe_ctrl_seg * ctrl)259542578c6STariq Toukan mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, void __iomem *uar_map,
260542578c6STariq Toukan struct mlx5_wqe_ctrl_seg *ctrl)
261542578c6STariq Toukan {
26282fe2996STariq Toukan ctrl->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE;
263542578c6STariq Toukan /* ensure wqe is visible to device before updating doorbell record */
264542578c6STariq Toukan dma_wmb();
265542578c6STariq Toukan
266542578c6STariq Toukan *wq->db = cpu_to_be32(pc);
267542578c6STariq Toukan
268542578c6STariq Toukan /* ensure doorbell record is visible to device before ringing the
269542578c6STariq Toukan * doorbell
270542578c6STariq Toukan */
271542578c6STariq Toukan wmb();
272542578c6STariq Toukan
273542578c6STariq Toukan mlx5_write64((__be32 *)ctrl, uar_map);
274542578c6STariq Toukan }
275542578c6STariq Toukan
mlx5e_cq_arm(struct mlx5e_cq * cq)276542578c6STariq Toukan static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
277542578c6STariq Toukan {
278542578c6STariq Toukan struct mlx5_core_cq *mcq;
279542578c6STariq Toukan
280542578c6STariq Toukan mcq = &cq->mcq;
281542578c6STariq Toukan mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc);
282542578c6STariq Toukan }
283542578c6STariq Toukan
284542578c6STariq Toukan static inline struct mlx5e_sq_dma *
mlx5e_dma_get(struct mlx5e_txqsq * sq,u32 i)285542578c6STariq Toukan mlx5e_dma_get(struct mlx5e_txqsq *sq, u32 i)
286542578c6STariq Toukan {
287542578c6STariq Toukan return &sq->db.dma_fifo[i & sq->dma_fifo_mask];
288542578c6STariq Toukan }
289542578c6STariq Toukan
290542578c6STariq Toukan static inline void
mlx5e_dma_push(struct mlx5e_txqsq * sq,dma_addr_t addr,u32 size,enum mlx5e_dma_map_type map_type)291542578c6STariq Toukan mlx5e_dma_push(struct mlx5e_txqsq *sq, dma_addr_t addr, u32 size,
292542578c6STariq Toukan enum mlx5e_dma_map_type map_type)
293542578c6STariq Toukan {
294542578c6STariq Toukan struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, sq->dma_fifo_pc++);
295542578c6STariq Toukan
296542578c6STariq Toukan dma->addr = addr;
297542578c6STariq Toukan dma->size = size;
298542578c6STariq Toukan dma->type = map_type;
299542578c6STariq Toukan }
300542578c6STariq Toukan
3010b676aaeSEran Ben Elisha static inline
mlx5e_skb_fifo_get(struct mlx5e_skb_fifo * fifo,u16 i)3020b676aaeSEran Ben Elisha struct sk_buff **mlx5e_skb_fifo_get(struct mlx5e_skb_fifo *fifo, u16 i)
303338c46c6SMaxim Mikityanskiy {
3040b676aaeSEran Ben Elisha return &fifo->fifo[i & fifo->mask];
305338c46c6SMaxim Mikityanskiy }
306338c46c6SMaxim Mikityanskiy
3070b676aaeSEran Ben Elisha static inline
mlx5e_skb_fifo_push(struct mlx5e_skb_fifo * fifo,struct sk_buff * skb)3080b676aaeSEran Ben Elisha void mlx5e_skb_fifo_push(struct mlx5e_skb_fifo *fifo, struct sk_buff *skb)
309338c46c6SMaxim Mikityanskiy {
3100b676aaeSEran Ben Elisha struct sk_buff **skb_item = mlx5e_skb_fifo_get(fifo, (*fifo->pc)++);
311338c46c6SMaxim Mikityanskiy
312338c46c6SMaxim Mikityanskiy *skb_item = skb;
313338c46c6SMaxim Mikityanskiy }
314338c46c6SMaxim Mikityanskiy
3150b676aaeSEran Ben Elisha static inline
mlx5e_skb_fifo_pop(struct mlx5e_skb_fifo * fifo)3160b676aaeSEran Ben Elisha struct sk_buff *mlx5e_skb_fifo_pop(struct mlx5e_skb_fifo *fifo)
317338c46c6SMaxim Mikityanskiy {
3183a50cf1eSVadim Fedorenko WARN_ON_ONCE(*fifo->pc == *fifo->cc);
3193a50cf1eSVadim Fedorenko
3200b676aaeSEran Ben Elisha return *mlx5e_skb_fifo_get(fifo, (*fifo->cc)++);
321338c46c6SMaxim Mikityanskiy }
322338c46c6SMaxim Mikityanskiy
323542578c6STariq Toukan static inline void
mlx5e_tx_dma_unmap(struct device * pdev,struct mlx5e_sq_dma * dma)324542578c6STariq Toukan mlx5e_tx_dma_unmap(struct device *pdev, struct mlx5e_sq_dma *dma)
325542578c6STariq Toukan {
326542578c6STariq Toukan switch (dma->type) {
327542578c6STariq Toukan case MLX5E_DMA_MAP_SINGLE:
328542578c6STariq Toukan dma_unmap_single(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
329542578c6STariq Toukan break;
330542578c6STariq Toukan case MLX5E_DMA_MAP_PAGE:
331542578c6STariq Toukan dma_unmap_page(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
332542578c6STariq Toukan break;
333542578c6STariq Toukan default:
334542578c6STariq Toukan WARN_ONCE(true, "mlx5e_tx_dma_unmap unknown DMA type!\n");
335542578c6STariq Toukan }
336542578c6STariq Toukan }
337542578c6STariq Toukan
3385af75c74SMaxim Mikityanskiy void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq);
3398e4b53f6SMaxim Mikityanskiy
mlx5e_tx_mpwqe_is_full(struct mlx5e_tx_mpwqe * session,u8 max_sq_mpw_wqebbs)34076c31e5fSAya Levin static inline bool mlx5e_tx_mpwqe_is_full(struct mlx5e_tx_mpwqe *session, u8 max_sq_mpw_wqebbs)
341530d5ce2SMaxim Mikityanskiy {
34276c31e5fSAya Levin return session->ds_count == max_sq_mpw_wqebbs * MLX5_SEND_WQEBB_NUM_DS;
343530d5ce2SMaxim Mikityanskiy }
344530d5ce2SMaxim Mikityanskiy
mlx5e_rqwq_reset(struct mlx5e_rq * rq)3455ee090edSAya Levin static inline void mlx5e_rqwq_reset(struct mlx5e_rq *rq)
3465ee090edSAya Levin {
34739369fd5SAya Levin if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
3485ee090edSAya Levin mlx5_wq_ll_reset(&rq->mpwqe.wq);
34939369fd5SAya Levin rq->mpwqe.actual_wq_head = 0;
35039369fd5SAya Levin } else {
3515ee090edSAya Levin mlx5_wq_cyc_reset(&rq->wqe.wq);
3525ee090edSAya Levin }
35339369fd5SAya Levin }
3545ee090edSAya Levin
mlx5e_dump_error_cqe(struct mlx5e_cq * cq,u32 qn,struct mlx5_err_cqe * err_cqe)355b9961af7SAya Levin static inline void mlx5e_dump_error_cqe(struct mlx5e_cq *cq, u32 qn,
356f1b95753STariq Toukan struct mlx5_err_cqe *err_cqe)
357f1b95753STariq Toukan {
358f1b95753STariq Toukan struct mlx5_cqwq *wq = &cq->wq;
359f1b95753STariq Toukan u32 ci;
360f1b95753STariq Toukan
361f1b95753STariq Toukan ci = mlx5_cqwq_ctr2ix(wq, wq->cc - 1);
362f1b95753STariq Toukan
3634d0b7ef9SAya Levin netdev_err(cq->netdev,
364b9961af7SAya Levin "Error cqe on cqn 0x%x, ci 0x%x, qn 0x%x, opcode 0x%x, syndrome 0x%x, vendor syndrome 0x%x\n",
365b9961af7SAya Levin cq->mcq.cqn, ci, qn,
366f1b95753STariq Toukan get_cqe_opcode((struct mlx5_cqe64 *)err_cqe),
367f1b95753STariq Toukan err_cqe->syndrome, err_cqe->vendor_err_synd);
368f1b95753STariq Toukan mlx5_dump_err_cqe(cq->mdev, err_cqe);
369f1b95753STariq Toukan }
370f1b95753STariq Toukan
mlx5e_rqwq_get_size(struct mlx5e_rq * rq)3715d95c816SAya Levin static inline u32 mlx5e_rqwq_get_size(struct mlx5e_rq *rq)
3725d95c816SAya Levin {
3735d95c816SAya Levin switch (rq->wq_type) {
3745d95c816SAya Levin case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
3755d95c816SAya Levin return mlx5_wq_ll_get_size(&rq->mpwqe.wq);
3765d95c816SAya Levin default:
3775d95c816SAya Levin return mlx5_wq_cyc_get_size(&rq->wqe.wq);
3785d95c816SAya Levin }
3795d95c816SAya Levin }
3805d95c816SAya Levin
mlx5e_rqwq_get_cur_sz(struct mlx5e_rq * rq)3815d95c816SAya Levin static inline u32 mlx5e_rqwq_get_cur_sz(struct mlx5e_rq *rq)
3825d95c816SAya Levin {
3835d95c816SAya Levin switch (rq->wq_type) {
3845d95c816SAya Levin case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
3855d95c816SAya Levin return rq->mpwqe.wq.cur_sz;
3865d95c816SAya Levin default:
3875d95c816SAya Levin return rq->wqe.wq.cur_sz;
3885d95c816SAya Levin }
3895d95c816SAya Levin }
3905d95c816SAya Levin
mlx5e_rqwq_get_head(struct mlx5e_rq * rq)391fc42d0deSAya Levin static inline u16 mlx5e_rqwq_get_head(struct mlx5e_rq *rq)
392fc42d0deSAya Levin {
393fc42d0deSAya Levin switch (rq->wq_type) {
394fc42d0deSAya Levin case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
395fc42d0deSAya Levin return mlx5_wq_ll_get_head(&rq->mpwqe.wq);
396fc42d0deSAya Levin default:
397fc42d0deSAya Levin return mlx5_wq_cyc_get_head(&rq->wqe.wq);
398fc42d0deSAya Levin }
399fc42d0deSAya Levin }
400fc42d0deSAya Levin
mlx5e_rqwq_get_wqe_counter(struct mlx5e_rq * rq)401de6c6ab7SAya Levin static inline u16 mlx5e_rqwq_get_wqe_counter(struct mlx5e_rq *rq)
402de6c6ab7SAya Levin {
403de6c6ab7SAya Levin switch (rq->wq_type) {
404de6c6ab7SAya Levin case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
405de6c6ab7SAya Levin return mlx5_wq_ll_get_counter(&rq->mpwqe.wq);
406de6c6ab7SAya Levin default:
407de6c6ab7SAya Levin return mlx5_wq_cyc_get_counter(&rq->wqe.wq);
408de6c6ab7SAya Levin }
409de6c6ab7SAya Levin }
410de6c6ab7SAya Levin
411542578c6STariq Toukan /* SW parser related functions */
412542578c6STariq Toukan
413542578c6STariq Toukan struct mlx5e_swp_spec {
414542578c6STariq Toukan __be16 l3_proto;
415542578c6STariq Toukan u8 l4_proto;
416542578c6STariq Toukan u8 is_tun;
417542578c6STariq Toukan __be16 tun_l3_proto;
418542578c6STariq Toukan u8 tun_l4_proto;
419542578c6STariq Toukan };
420542578c6STariq Toukan
mlx5e_eseg_swp_offsets_add_vlan(struct mlx5_wqe_eth_seg * eseg)421b544011fSMoshe Shemesh static inline void mlx5e_eseg_swp_offsets_add_vlan(struct mlx5_wqe_eth_seg *eseg)
422b544011fSMoshe Shemesh {
423b544011fSMoshe Shemesh /* SWP offsets are in 2-bytes words */
424b544011fSMoshe Shemesh eseg->swp_outer_l3_offset += VLAN_HLEN / 2;
425b544011fSMoshe Shemesh eseg->swp_outer_l4_offset += VLAN_HLEN / 2;
426b544011fSMoshe Shemesh eseg->swp_inner_l3_offset += VLAN_HLEN / 2;
427b544011fSMoshe Shemesh eseg->swp_inner_l4_offset += VLAN_HLEN / 2;
428b544011fSMoshe Shemesh }
429b544011fSMoshe Shemesh
430542578c6STariq Toukan static inline void
mlx5e_set_eseg_swp(struct sk_buff * skb,struct mlx5_wqe_eth_seg * eseg,struct mlx5e_swp_spec * swp_spec)431542578c6STariq Toukan mlx5e_set_eseg_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg,
432542578c6STariq Toukan struct mlx5e_swp_spec *swp_spec)
433542578c6STariq Toukan {
434542578c6STariq Toukan /* SWP offsets are in 2-bytes words */
435542578c6STariq Toukan eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2;
436542578c6STariq Toukan if (swp_spec->l3_proto == htons(ETH_P_IPV6))
437542578c6STariq Toukan eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6;
438542578c6STariq Toukan if (swp_spec->l4_proto) {
439542578c6STariq Toukan eseg->swp_outer_l4_offset = skb_transport_offset(skb) / 2;
440542578c6STariq Toukan if (swp_spec->l4_proto == IPPROTO_UDP)
441542578c6STariq Toukan eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_UDP;
442542578c6STariq Toukan }
443542578c6STariq Toukan
444542578c6STariq Toukan if (swp_spec->is_tun) {
445542578c6STariq Toukan eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2;
446542578c6STariq Toukan if (swp_spec->tun_l3_proto == htons(ETH_P_IPV6))
447542578c6STariq Toukan eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
448542578c6STariq Toukan } else { /* typically for ipsec when xfrm mode != XFRM_MODE_TUNNEL */
449542578c6STariq Toukan eseg->swp_inner_l3_offset = skb_network_offset(skb) / 2;
450542578c6STariq Toukan if (swp_spec->l3_proto == htons(ETH_P_IPV6))
451542578c6STariq Toukan eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
452542578c6STariq Toukan }
453542578c6STariq Toukan switch (swp_spec->tun_l4_proto) {
454542578c6STariq Toukan case IPPROTO_UDP:
455542578c6STariq Toukan eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP;
456c8b838d1SGustavo A. R. Silva fallthrough;
457542578c6STariq Toukan case IPPROTO_TCP:
458542578c6STariq Toukan eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2;
459542578c6STariq Toukan break;
460542578c6STariq Toukan }
461542578c6STariq Toukan }
462542578c6STariq Toukan
463c27bd171SAya Levin #define MLX5E_STOP_ROOM(wqebbs) ((wqebbs) * 2 - 1)
4645ffb4d85SMaxim Mikityanskiy
mlx5e_stop_room_for_wqe(struct mlx5_core_dev * mdev,u16 wqe_size)465c27bd171SAya Levin static inline u16 mlx5e_stop_room_for_wqe(struct mlx5_core_dev *mdev, u16 wqe_size)
466c27bd171SAya Levin {
467022dbea0SRahul Rameshbabu WARN_ON_ONCE(PAGE_SIZE / MLX5_SEND_WQE_BB < (u16)mlx5e_get_max_sq_wqebbs(mdev));
468f9c955b4SMaxim Mikityanskiy
4695ffb4d85SMaxim Mikityanskiy /* A WQE must not cross the page boundary, hence two conditions:
4705ffb4d85SMaxim Mikityanskiy * 1. Its size must not exceed the page size.
4715ffb4d85SMaxim Mikityanskiy * 2. If the WQE size is X, and the space remaining in a page is less
4725ffb4d85SMaxim Mikityanskiy * than X, this space needs to be padded with NOPs. So, one WQE of
4735ffb4d85SMaxim Mikityanskiy * size X may require up to X-1 WQEBBs of padding, which makes the
4745ffb4d85SMaxim Mikityanskiy * stop room of X-1 + X.
4755ffb4d85SMaxim Mikityanskiy * WQE size is also limited by the hardware limit.
4765ffb4d85SMaxim Mikityanskiy */
477c27bd171SAya Levin WARN_ONCE(wqe_size > mlx5e_get_max_sq_wqebbs(mdev),
478c27bd171SAya Levin "wqe_size %u is greater than max SQ WQEBBs %u",
479c27bd171SAya Levin wqe_size, mlx5e_get_max_sq_wqebbs(mdev));
4805ffb4d85SMaxim Mikityanskiy
481c27bd171SAya Levin return MLX5E_STOP_ROOM(wqe_size);
482c27bd171SAya Levin }
483c27bd171SAya Levin
mlx5e_stop_room_for_max_wqe(struct mlx5_core_dev * mdev)484c27bd171SAya Levin static inline u16 mlx5e_stop_room_for_max_wqe(struct mlx5_core_dev *mdev)
485c27bd171SAya Levin {
486c27bd171SAya Levin return MLX5E_STOP_ROOM(mlx5e_get_max_sq_wqebbs(mdev));
4875ffb4d85SMaxim Mikityanskiy }
4885ffb4d85SMaxim Mikityanskiy
mlx5e_stop_room_for_mpwqe(struct mlx5_core_dev * mdev)48921a0502dSMaxim Mikityanskiy static inline u16 mlx5e_stop_room_for_mpwqe(struct mlx5_core_dev *mdev)
49021a0502dSMaxim Mikityanskiy {
49121a0502dSMaxim Mikityanskiy u8 mpwqe_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
49221a0502dSMaxim Mikityanskiy
49321a0502dSMaxim Mikityanskiy return mlx5e_stop_room_for_wqe(mdev, mpwqe_wqebbs);
49421a0502dSMaxim Mikityanskiy }
49521a0502dSMaxim Mikityanskiy
mlx5e_icosq_can_post_wqe(struct mlx5e_icosq * sq,u16 wqe_size)4963ff3874fSTariq Toukan static inline bool mlx5e_icosq_can_post_wqe(struct mlx5e_icosq *sq, u16 wqe_size)
4973ff3874fSTariq Toukan {
4984c78782eSMaxim Mikityanskiy u16 room = sq->reserved_room + MLX5E_STOP_ROOM(wqe_size);
4993ff3874fSTariq Toukan
5003ff3874fSTariq Toukan return mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room);
5013ff3874fSTariq Toukan }
502cf544517SMaxim Mikityanskiy
mlx5e_get_mpw_info(struct mlx5e_rq * rq,int i)503cf544517SMaxim Mikityanskiy static inline struct mlx5e_mpw_info *mlx5e_get_mpw_info(struct mlx5e_rq *rq, int i)
504cf544517SMaxim Mikityanskiy {
5056f574284SDragos Tatulea size_t isz = struct_size(rq->mpwqe.info, alloc_units.frag_pages, rq->mpwqe.pages_per_wqe);
506cf544517SMaxim Mikityanskiy
507cf544517SMaxim Mikityanskiy return (struct mlx5e_mpw_info *)((char *)rq->mpwqe.info + array_size(i, isz));
508cf544517SMaxim Mikityanskiy }
509542578c6STariq Toukan #endif
510