1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2019 Mellanox Technologies. */
3 
4 #ifndef __MLX5_EN_TXRX_H___
5 #define __MLX5_EN_TXRX_H___
6 
7 #include "en.h"
8 #include <linux/indirect_call_wrapper.h>
9 
10 #define MLX5E_TX_WQE_EMPTY_DS_COUNT (sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS)
11 
12 /* The mult of MLX5_SEND_WQE_MAX_WQEBBS * MLX5_SEND_WQEBB_NUM_DS
13  * (16 * 4 == 64) does not fit in the 6-bit DS field of Ctrl Segment.
14  * We use a bound lower that MLX5_SEND_WQE_MAX_WQEBBS to let a
15  * full-session WQE be cache-aligned.
16  */
17 #if L1_CACHE_BYTES < 128
18 #define MLX5E_TX_MPW_MAX_WQEBBS (MLX5_SEND_WQE_MAX_WQEBBS - 1)
19 #else
20 #define MLX5E_TX_MPW_MAX_WQEBBS (MLX5_SEND_WQE_MAX_WQEBBS - 2)
21 #endif
22 
23 #define MLX5E_TX_MPW_MAX_NUM_DS (MLX5E_TX_MPW_MAX_WQEBBS * MLX5_SEND_WQEBB_NUM_DS)
24 
25 #define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start))
26 
27 #define MLX5E_RX_ERR_CQE(cqe) (get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)
28 
29 static inline
30 ktime_t mlx5e_cqe_ts_to_ns(cqe_ts_to_ns func, struct mlx5_clock *clock, u64 cqe_ts)
31 {
32 	return INDIRECT_CALL_2(func, mlx5_real_time_cyc2time, mlx5_timecounter_cyc2time,
33 			       clock, cqe_ts);
34 }
35 
36 enum mlx5e_icosq_wqe_type {
37 	MLX5E_ICOSQ_WQE_NOP,
38 	MLX5E_ICOSQ_WQE_UMR_RX,
39 	MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR,
40 #ifdef CONFIG_MLX5_EN_TLS
41 	MLX5E_ICOSQ_WQE_UMR_TLS,
42 	MLX5E_ICOSQ_WQE_SET_PSV_TLS,
43 	MLX5E_ICOSQ_WQE_GET_PSV_TLS,
44 #endif
45 };
46 
47 /* General */
48 static inline bool mlx5e_skb_is_multicast(struct sk_buff *skb)
49 {
50 	return skb->pkt_type == PACKET_MULTICAST || skb->pkt_type == PACKET_BROADCAST;
51 }
52 
53 void mlx5e_trigger_irq(struct mlx5e_icosq *sq);
54 void mlx5e_completion_event(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe);
55 void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
56 int mlx5e_napi_poll(struct napi_struct *napi, int budget);
57 int mlx5e_poll_ico_cq(struct mlx5e_cq *cq);
58 
59 /* RX */
60 void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info);
61 void mlx5e_page_release_dynamic(struct mlx5e_rq *rq,
62 				struct mlx5e_dma_info *dma_info,
63 				bool recycle);
64 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq));
65 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq));
66 int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);
67 void mlx5e_free_rx_descs(struct mlx5e_rq *rq);
68 void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq);
69 
70 /* TX */
71 u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
72 		       struct net_device *sb_dev);
73 netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev);
74 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);
75 void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq);
76 
77 static inline bool
78 mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n)
79 {
80 	return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc);
81 }
82 
83 static inline void *mlx5e_fetch_wqe(struct mlx5_wq_cyc *wq, u16 pi, size_t wqe_size)
84 {
85 	void *wqe;
86 
87 	wqe = mlx5_wq_cyc_get_wqe(wq, pi);
88 	memset(wqe, 0, wqe_size);
89 
90 	return wqe;
91 }
92 
93 #define MLX5E_TX_FETCH_WQE(sq, pi) \
94 	((struct mlx5e_tx_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_tx_wqe)))
95 
96 static inline struct mlx5e_tx_wqe *
97 mlx5e_post_nop(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
98 {
99 	u16                         pi   = mlx5_wq_cyc_ctr2ix(wq, *pc);
100 	struct mlx5e_tx_wqe        *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
101 	struct mlx5_wqe_ctrl_seg   *cseg = &wqe->ctrl;
102 
103 	memset(cseg, 0, sizeof(*cseg));
104 
105 	cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP);
106 	cseg->qpn_ds           = cpu_to_be32((sqn << 8) | 0x01);
107 
108 	(*pc)++;
109 
110 	return wqe;
111 }
112 
113 static inline struct mlx5e_tx_wqe *
114 mlx5e_post_nop_fence(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
115 {
116 	u16                         pi   = mlx5_wq_cyc_ctr2ix(wq, *pc);
117 	struct mlx5e_tx_wqe        *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
118 	struct mlx5_wqe_ctrl_seg   *cseg = &wqe->ctrl;
119 
120 	memset(cseg, 0, sizeof(*cseg));
121 
122 	cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP);
123 	cseg->qpn_ds           = cpu_to_be32((sqn << 8) | 0x01);
124 	cseg->fm_ce_se         = MLX5_FENCE_MODE_INITIATOR_SMALL;
125 
126 	(*pc)++;
127 
128 	return wqe;
129 }
130 
131 struct mlx5e_tx_wqe_info {
132 	struct sk_buff *skb;
133 	u32 num_bytes;
134 	u8 num_wqebbs;
135 	u8 num_dma;
136 	u8 num_fifo_pkts;
137 #ifdef CONFIG_MLX5_EN_TLS
138 	struct page *resync_dump_frag_page;
139 #endif
140 };
141 
142 static inline u16 mlx5e_txqsq_get_next_pi(struct mlx5e_txqsq *sq, u16 size)
143 {
144 	struct mlx5_wq_cyc *wq = &sq->wq;
145 	u16 pi, contig_wqebbs;
146 
147 	pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
148 	contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
149 	if (unlikely(contig_wqebbs < size)) {
150 		struct mlx5e_tx_wqe_info *wi, *edge_wi;
151 
152 		wi = &sq->db.wqe_info[pi];
153 		edge_wi = wi + contig_wqebbs;
154 
155 		/* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */
156 		for (; wi < edge_wi; wi++) {
157 			*wi = (struct mlx5e_tx_wqe_info) {
158 				.num_wqebbs = 1,
159 			};
160 			mlx5e_post_nop(wq, sq->sqn, &sq->pc);
161 		}
162 		sq->stats->nop += contig_wqebbs;
163 
164 		pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
165 	}
166 
167 	return pi;
168 }
169 
170 struct mlx5e_shampo_umr {
171 	u16 len;
172 };
173 
174 struct mlx5e_icosq_wqe_info {
175 	u8 wqe_type;
176 	u8 num_wqebbs;
177 
178 	/* Auxiliary data for different wqe types. */
179 	union {
180 		struct {
181 			struct mlx5e_rq *rq;
182 		} umr;
183 		struct mlx5e_shampo_umr shampo;
184 #ifdef CONFIG_MLX5_EN_TLS
185 		struct {
186 			struct mlx5e_ktls_offload_context_rx *priv_rx;
187 		} tls_set_params;
188 		struct {
189 			struct mlx5e_ktls_rx_resync_buf *buf;
190 		} tls_get_params;
191 #endif
192 	};
193 };
194 
195 void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq);
196 
197 static inline u16 mlx5e_icosq_get_next_pi(struct mlx5e_icosq *sq, u16 size)
198 {
199 	struct mlx5_wq_cyc *wq = &sq->wq;
200 	u16 pi, contig_wqebbs;
201 
202 	pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
203 	contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
204 	if (unlikely(contig_wqebbs < size)) {
205 		struct mlx5e_icosq_wqe_info *wi, *edge_wi;
206 
207 		wi = &sq->db.wqe_info[pi];
208 		edge_wi = wi + contig_wqebbs;
209 
210 		/* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */
211 		for (; wi < edge_wi; wi++) {
212 			*wi = (struct mlx5e_icosq_wqe_info) {
213 				.wqe_type   = MLX5E_ICOSQ_WQE_NOP,
214 				.num_wqebbs = 1,
215 			};
216 			mlx5e_post_nop(wq, sq->sqn, &sq->pc);
217 		}
218 
219 		pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
220 	}
221 
222 	return pi;
223 }
224 
225 static inline void
226 mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, void __iomem *uar_map,
227 		struct mlx5_wqe_ctrl_seg *ctrl)
228 {
229 	ctrl->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE;
230 	/* ensure wqe is visible to device before updating doorbell record */
231 	dma_wmb();
232 
233 	*wq->db = cpu_to_be32(pc);
234 
235 	/* ensure doorbell record is visible to device before ringing the
236 	 * doorbell
237 	 */
238 	wmb();
239 
240 	mlx5_write64((__be32 *)ctrl, uar_map);
241 }
242 
243 static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
244 {
245 	struct mlx5_core_cq *mcq;
246 
247 	mcq = &cq->mcq;
248 	mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc);
249 }
250 
251 static inline struct mlx5e_sq_dma *
252 mlx5e_dma_get(struct mlx5e_txqsq *sq, u32 i)
253 {
254 	return &sq->db.dma_fifo[i & sq->dma_fifo_mask];
255 }
256 
257 static inline void
258 mlx5e_dma_push(struct mlx5e_txqsq *sq, dma_addr_t addr, u32 size,
259 	       enum mlx5e_dma_map_type map_type)
260 {
261 	struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, sq->dma_fifo_pc++);
262 
263 	dma->addr = addr;
264 	dma->size = size;
265 	dma->type = map_type;
266 }
267 
268 static inline
269 struct sk_buff **mlx5e_skb_fifo_get(struct mlx5e_skb_fifo *fifo, u16 i)
270 {
271 	return &fifo->fifo[i & fifo->mask];
272 }
273 
274 static inline
275 void mlx5e_skb_fifo_push(struct mlx5e_skb_fifo *fifo, struct sk_buff *skb)
276 {
277 	struct sk_buff **skb_item = mlx5e_skb_fifo_get(fifo, (*fifo->pc)++);
278 
279 	*skb_item = skb;
280 }
281 
282 static inline
283 struct sk_buff *mlx5e_skb_fifo_pop(struct mlx5e_skb_fifo *fifo)
284 {
285 	return *mlx5e_skb_fifo_get(fifo, (*fifo->cc)++);
286 }
287 
288 static inline void
289 mlx5e_tx_dma_unmap(struct device *pdev, struct mlx5e_sq_dma *dma)
290 {
291 	switch (dma->type) {
292 	case MLX5E_DMA_MAP_SINGLE:
293 		dma_unmap_single(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
294 		break;
295 	case MLX5E_DMA_MAP_PAGE:
296 		dma_unmap_page(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
297 		break;
298 	default:
299 		WARN_ONCE(true, "mlx5e_tx_dma_unmap unknown DMA type!\n");
300 	}
301 }
302 
303 void mlx5e_sq_xmit_simple(struct mlx5e_txqsq *sq, struct sk_buff *skb, bool xmit_more);
304 void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq);
305 
306 static inline bool mlx5e_tx_mpwqe_is_full(struct mlx5e_tx_mpwqe *session)
307 {
308 	return session->ds_count == MLX5E_TX_MPW_MAX_NUM_DS;
309 }
310 
311 static inline void mlx5e_rqwq_reset(struct mlx5e_rq *rq)
312 {
313 	if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
314 		mlx5_wq_ll_reset(&rq->mpwqe.wq);
315 		rq->mpwqe.actual_wq_head = 0;
316 	} else {
317 		mlx5_wq_cyc_reset(&rq->wqe.wq);
318 	}
319 }
320 
321 static inline void mlx5e_dump_error_cqe(struct mlx5e_cq *cq, u32 qn,
322 					struct mlx5_err_cqe *err_cqe)
323 {
324 	struct mlx5_cqwq *wq = &cq->wq;
325 	u32 ci;
326 
327 	ci = mlx5_cqwq_ctr2ix(wq, wq->cc - 1);
328 
329 	netdev_err(cq->netdev,
330 		   "Error cqe on cqn 0x%x, ci 0x%x, qn 0x%x, opcode 0x%x, syndrome 0x%x, vendor syndrome 0x%x\n",
331 		   cq->mcq.cqn, ci, qn,
332 		   get_cqe_opcode((struct mlx5_cqe64 *)err_cqe),
333 		   err_cqe->syndrome, err_cqe->vendor_err_synd);
334 	mlx5_dump_err_cqe(cq->mdev, err_cqe);
335 }
336 
337 static inline u32 mlx5e_rqwq_get_size(struct mlx5e_rq *rq)
338 {
339 	switch (rq->wq_type) {
340 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
341 		return mlx5_wq_ll_get_size(&rq->mpwqe.wq);
342 	default:
343 		return mlx5_wq_cyc_get_size(&rq->wqe.wq);
344 	}
345 }
346 
347 static inline u32 mlx5e_rqwq_get_cur_sz(struct mlx5e_rq *rq)
348 {
349 	switch (rq->wq_type) {
350 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
351 		return rq->mpwqe.wq.cur_sz;
352 	default:
353 		return rq->wqe.wq.cur_sz;
354 	}
355 }
356 
357 static inline u16 mlx5e_rqwq_get_head(struct mlx5e_rq *rq)
358 {
359 	switch (rq->wq_type) {
360 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
361 		return mlx5_wq_ll_get_head(&rq->mpwqe.wq);
362 	default:
363 		return mlx5_wq_cyc_get_head(&rq->wqe.wq);
364 	}
365 }
366 
367 static inline u16 mlx5e_rqwq_get_wqe_counter(struct mlx5e_rq *rq)
368 {
369 	switch (rq->wq_type) {
370 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
371 		return mlx5_wq_ll_get_counter(&rq->mpwqe.wq);
372 	default:
373 		return mlx5_wq_cyc_get_counter(&rq->wqe.wq);
374 	}
375 }
376 
377 /* SW parser related functions */
378 
379 struct mlx5e_swp_spec {
380 	__be16 l3_proto;
381 	u8 l4_proto;
382 	u8 is_tun;
383 	__be16 tun_l3_proto;
384 	u8 tun_l4_proto;
385 };
386 
387 static inline void mlx5e_eseg_swp_offsets_add_vlan(struct mlx5_wqe_eth_seg *eseg)
388 {
389 	/* SWP offsets are in 2-bytes words */
390 	eseg->swp_outer_l3_offset += VLAN_HLEN / 2;
391 	eseg->swp_outer_l4_offset += VLAN_HLEN / 2;
392 	eseg->swp_inner_l3_offset += VLAN_HLEN / 2;
393 	eseg->swp_inner_l4_offset += VLAN_HLEN / 2;
394 }
395 
396 static inline void
397 mlx5e_set_eseg_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg,
398 		   struct mlx5e_swp_spec *swp_spec)
399 {
400 	/* SWP offsets are in 2-bytes words */
401 	eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2;
402 	if (swp_spec->l3_proto == htons(ETH_P_IPV6))
403 		eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6;
404 	if (swp_spec->l4_proto) {
405 		eseg->swp_outer_l4_offset = skb_transport_offset(skb) / 2;
406 		if (swp_spec->l4_proto == IPPROTO_UDP)
407 			eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_UDP;
408 	}
409 
410 	if (swp_spec->is_tun) {
411 		eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2;
412 		if (swp_spec->tun_l3_proto == htons(ETH_P_IPV6))
413 			eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
414 	} else { /* typically for ipsec when xfrm mode != XFRM_MODE_TUNNEL */
415 		eseg->swp_inner_l3_offset = skb_network_offset(skb) / 2;
416 		if (swp_spec->l3_proto == htons(ETH_P_IPV6))
417 			eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
418 	}
419 	switch (swp_spec->tun_l4_proto) {
420 	case IPPROTO_UDP:
421 		eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP;
422 		fallthrough;
423 	case IPPROTO_TCP:
424 		eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2;
425 		break;
426 	}
427 }
428 
429 static inline u16 mlx5e_stop_room_for_wqe(u16 wqe_size)
430 {
431 	BUILD_BUG_ON(PAGE_SIZE / MLX5_SEND_WQE_BB < MLX5_SEND_WQE_MAX_WQEBBS);
432 
433 	/* A WQE must not cross the page boundary, hence two conditions:
434 	 * 1. Its size must not exceed the page size.
435 	 * 2. If the WQE size is X, and the space remaining in a page is less
436 	 *    than X, this space needs to be padded with NOPs. So, one WQE of
437 	 *    size X may require up to X-1 WQEBBs of padding, which makes the
438 	 *    stop room of X-1 + X.
439 	 * WQE size is also limited by the hardware limit.
440 	 */
441 
442 	if (__builtin_constant_p(wqe_size))
443 		BUILD_BUG_ON(wqe_size > MLX5_SEND_WQE_MAX_WQEBBS);
444 	else
445 		WARN_ON_ONCE(wqe_size > MLX5_SEND_WQE_MAX_WQEBBS);
446 
447 	return wqe_size * 2 - 1;
448 }
449 
450 static inline bool mlx5e_icosq_can_post_wqe(struct mlx5e_icosq *sq, u16 wqe_size)
451 {
452 	u16 room = sq->reserved_room + mlx5e_stop_room_for_wqe(wqe_size);
453 
454 	return mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room);
455 }
456 #endif
457