1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2019 Mellanox Technologies. */
3 
4 #ifndef __MLX5_EN_TXRX_H___
5 #define __MLX5_EN_TXRX_H___
6 
7 #include "en.h"
8 #include <linux/indirect_call_wrapper.h>
9 
10 #define MLX5E_TX_WQE_EMPTY_DS_COUNT (sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS)
11 
12 #define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start))
13 
14 #define MLX5E_RX_ERR_CQE(cqe) (get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)
15 
16 static inline
17 ktime_t mlx5e_cqe_ts_to_ns(cqe_ts_to_ns func, struct mlx5_clock *clock, u64 cqe_ts)
18 {
19 	return INDIRECT_CALL_2(func, mlx5_real_time_cyc2time, mlx5_timecounter_cyc2time,
20 			       clock, cqe_ts);
21 }
22 
23 enum mlx5e_icosq_wqe_type {
24 	MLX5E_ICOSQ_WQE_NOP,
25 	MLX5E_ICOSQ_WQE_UMR_RX,
26 	MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR,
27 #ifdef CONFIG_MLX5_EN_TLS
28 	MLX5E_ICOSQ_WQE_UMR_TLS,
29 	MLX5E_ICOSQ_WQE_SET_PSV_TLS,
30 	MLX5E_ICOSQ_WQE_GET_PSV_TLS,
31 #endif
32 };
33 
34 /* General */
35 static inline bool mlx5e_skb_is_multicast(struct sk_buff *skb)
36 {
37 	return skb->pkt_type == PACKET_MULTICAST || skb->pkt_type == PACKET_BROADCAST;
38 }
39 
40 void mlx5e_trigger_irq(struct mlx5e_icosq *sq);
41 void mlx5e_completion_event(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe);
42 void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
43 int mlx5e_napi_poll(struct napi_struct *napi, int budget);
44 int mlx5e_poll_ico_cq(struct mlx5e_cq *cq);
45 
46 /* RX */
47 void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info);
48 void mlx5e_page_release_dynamic(struct mlx5e_rq *rq,
49 				struct mlx5e_dma_info *dma_info,
50 				bool recycle);
51 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq));
52 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq));
53 int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);
54 void mlx5e_free_rx_descs(struct mlx5e_rq *rq);
55 void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq);
56 
57 /* TX */
58 netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev);
59 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);
60 void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq);
61 
62 static inline bool
63 mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n)
64 {
65 	return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc);
66 }
67 
68 static inline void *mlx5e_fetch_wqe(struct mlx5_wq_cyc *wq, u16 pi, size_t wqe_size)
69 {
70 	void *wqe;
71 
72 	wqe = mlx5_wq_cyc_get_wqe(wq, pi);
73 	memset(wqe, 0, wqe_size);
74 
75 	return wqe;
76 }
77 
78 #define MLX5E_TX_FETCH_WQE(sq, pi) \
79 	((struct mlx5e_tx_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_tx_wqe)))
80 
81 static inline struct mlx5e_tx_wqe *
82 mlx5e_post_nop(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
83 {
84 	u16                         pi   = mlx5_wq_cyc_ctr2ix(wq, *pc);
85 	struct mlx5e_tx_wqe        *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
86 	struct mlx5_wqe_ctrl_seg   *cseg = &wqe->ctrl;
87 
88 	memset(cseg, 0, sizeof(*cseg));
89 
90 	cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP);
91 	cseg->qpn_ds           = cpu_to_be32((sqn << 8) | 0x01);
92 
93 	(*pc)++;
94 
95 	return wqe;
96 }
97 
98 static inline struct mlx5e_tx_wqe *
99 mlx5e_post_nop_fence(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
100 {
101 	u16                         pi   = mlx5_wq_cyc_ctr2ix(wq, *pc);
102 	struct mlx5e_tx_wqe        *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
103 	struct mlx5_wqe_ctrl_seg   *cseg = &wqe->ctrl;
104 
105 	memset(cseg, 0, sizeof(*cseg));
106 
107 	cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP);
108 	cseg->qpn_ds           = cpu_to_be32((sqn << 8) | 0x01);
109 	cseg->fm_ce_se         = MLX5_FENCE_MODE_INITIATOR_SMALL;
110 
111 	(*pc)++;
112 
113 	return wqe;
114 }
115 
116 struct mlx5e_tx_wqe_info {
117 	struct sk_buff *skb;
118 	u32 num_bytes;
119 	u8 num_wqebbs;
120 	u8 num_dma;
121 	u8 num_fifo_pkts;
122 #ifdef CONFIG_MLX5_EN_TLS
123 	struct page *resync_dump_frag_page;
124 #endif
125 };
126 
127 static inline u16 mlx5e_txqsq_get_next_pi(struct mlx5e_txqsq *sq, u16 size)
128 {
129 	struct mlx5_wq_cyc *wq = &sq->wq;
130 	u16 pi, contig_wqebbs;
131 
132 	pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
133 	contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
134 	if (unlikely(contig_wqebbs < size)) {
135 		struct mlx5e_tx_wqe_info *wi, *edge_wi;
136 
137 		wi = &sq->db.wqe_info[pi];
138 		edge_wi = wi + contig_wqebbs;
139 
140 		/* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */
141 		for (; wi < edge_wi; wi++) {
142 			*wi = (struct mlx5e_tx_wqe_info) {
143 				.num_wqebbs = 1,
144 			};
145 			mlx5e_post_nop(wq, sq->sqn, &sq->pc);
146 		}
147 		sq->stats->nop += contig_wqebbs;
148 
149 		pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
150 	}
151 
152 	return pi;
153 }
154 
155 static inline u16 mlx5e_shampo_get_cqe_header_index(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
156 {
157 	return be16_to_cpu(cqe->shampo.header_entry_index) & (rq->mpwqe.shampo->hd_per_wq - 1);
158 }
159 
160 struct mlx5e_shampo_umr {
161 	u16 len;
162 };
163 
164 struct mlx5e_icosq_wqe_info {
165 	u8 wqe_type;
166 	u8 num_wqebbs;
167 
168 	/* Auxiliary data for different wqe types. */
169 	union {
170 		struct {
171 			struct mlx5e_rq *rq;
172 		} umr;
173 		struct mlx5e_shampo_umr shampo;
174 #ifdef CONFIG_MLX5_EN_TLS
175 		struct {
176 			struct mlx5e_ktls_offload_context_rx *priv_rx;
177 		} tls_set_params;
178 		struct {
179 			struct mlx5e_ktls_rx_resync_buf *buf;
180 		} tls_get_params;
181 #endif
182 	};
183 };
184 
185 void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq);
186 
187 static inline u16 mlx5e_icosq_get_next_pi(struct mlx5e_icosq *sq, u16 size)
188 {
189 	struct mlx5_wq_cyc *wq = &sq->wq;
190 	u16 pi, contig_wqebbs;
191 
192 	pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
193 	contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
194 	if (unlikely(contig_wqebbs < size)) {
195 		struct mlx5e_icosq_wqe_info *wi, *edge_wi;
196 
197 		wi = &sq->db.wqe_info[pi];
198 		edge_wi = wi + contig_wqebbs;
199 
200 		/* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */
201 		for (; wi < edge_wi; wi++) {
202 			*wi = (struct mlx5e_icosq_wqe_info) {
203 				.wqe_type   = MLX5E_ICOSQ_WQE_NOP,
204 				.num_wqebbs = 1,
205 			};
206 			mlx5e_post_nop(wq, sq->sqn, &sq->pc);
207 		}
208 
209 		pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
210 	}
211 
212 	return pi;
213 }
214 
215 static inline void
216 mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, void __iomem *uar_map,
217 		struct mlx5_wqe_ctrl_seg *ctrl)
218 {
219 	ctrl->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE;
220 	/* ensure wqe is visible to device before updating doorbell record */
221 	dma_wmb();
222 
223 	*wq->db = cpu_to_be32(pc);
224 
225 	/* ensure doorbell record is visible to device before ringing the
226 	 * doorbell
227 	 */
228 	wmb();
229 
230 	mlx5_write64((__be32 *)ctrl, uar_map);
231 }
232 
233 static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
234 {
235 	struct mlx5_core_cq *mcq;
236 
237 	mcq = &cq->mcq;
238 	mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc);
239 }
240 
241 static inline struct mlx5e_sq_dma *
242 mlx5e_dma_get(struct mlx5e_txqsq *sq, u32 i)
243 {
244 	return &sq->db.dma_fifo[i & sq->dma_fifo_mask];
245 }
246 
247 static inline void
248 mlx5e_dma_push(struct mlx5e_txqsq *sq, dma_addr_t addr, u32 size,
249 	       enum mlx5e_dma_map_type map_type)
250 {
251 	struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, sq->dma_fifo_pc++);
252 
253 	dma->addr = addr;
254 	dma->size = size;
255 	dma->type = map_type;
256 }
257 
258 static inline
259 struct sk_buff **mlx5e_skb_fifo_get(struct mlx5e_skb_fifo *fifo, u16 i)
260 {
261 	return &fifo->fifo[i & fifo->mask];
262 }
263 
264 static inline
265 void mlx5e_skb_fifo_push(struct mlx5e_skb_fifo *fifo, struct sk_buff *skb)
266 {
267 	struct sk_buff **skb_item = mlx5e_skb_fifo_get(fifo, (*fifo->pc)++);
268 
269 	*skb_item = skb;
270 }
271 
272 static inline
273 struct sk_buff *mlx5e_skb_fifo_pop(struct mlx5e_skb_fifo *fifo)
274 {
275 	return *mlx5e_skb_fifo_get(fifo, (*fifo->cc)++);
276 }
277 
278 static inline void
279 mlx5e_tx_dma_unmap(struct device *pdev, struct mlx5e_sq_dma *dma)
280 {
281 	switch (dma->type) {
282 	case MLX5E_DMA_MAP_SINGLE:
283 		dma_unmap_single(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
284 		break;
285 	case MLX5E_DMA_MAP_PAGE:
286 		dma_unmap_page(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
287 		break;
288 	default:
289 		WARN_ONCE(true, "mlx5e_tx_dma_unmap unknown DMA type!\n");
290 	}
291 }
292 
293 void mlx5e_sq_xmit_simple(struct mlx5e_txqsq *sq, struct sk_buff *skb, bool xmit_more);
294 void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq);
295 
296 static inline bool mlx5e_tx_mpwqe_is_full(struct mlx5e_tx_mpwqe *session, u8 max_sq_mpw_wqebbs)
297 {
298 	return session->ds_count == max_sq_mpw_wqebbs * MLX5_SEND_WQEBB_NUM_DS;
299 }
300 
301 static inline void mlx5e_rqwq_reset(struct mlx5e_rq *rq)
302 {
303 	if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
304 		mlx5_wq_ll_reset(&rq->mpwqe.wq);
305 		rq->mpwqe.actual_wq_head = 0;
306 	} else {
307 		mlx5_wq_cyc_reset(&rq->wqe.wq);
308 	}
309 }
310 
311 static inline void mlx5e_dump_error_cqe(struct mlx5e_cq *cq, u32 qn,
312 					struct mlx5_err_cqe *err_cqe)
313 {
314 	struct mlx5_cqwq *wq = &cq->wq;
315 	u32 ci;
316 
317 	ci = mlx5_cqwq_ctr2ix(wq, wq->cc - 1);
318 
319 	netdev_err(cq->netdev,
320 		   "Error cqe on cqn 0x%x, ci 0x%x, qn 0x%x, opcode 0x%x, syndrome 0x%x, vendor syndrome 0x%x\n",
321 		   cq->mcq.cqn, ci, qn,
322 		   get_cqe_opcode((struct mlx5_cqe64 *)err_cqe),
323 		   err_cqe->syndrome, err_cqe->vendor_err_synd);
324 	mlx5_dump_err_cqe(cq->mdev, err_cqe);
325 }
326 
327 static inline u32 mlx5e_rqwq_get_size(struct mlx5e_rq *rq)
328 {
329 	switch (rq->wq_type) {
330 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
331 		return mlx5_wq_ll_get_size(&rq->mpwqe.wq);
332 	default:
333 		return mlx5_wq_cyc_get_size(&rq->wqe.wq);
334 	}
335 }
336 
337 static inline u32 mlx5e_rqwq_get_cur_sz(struct mlx5e_rq *rq)
338 {
339 	switch (rq->wq_type) {
340 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
341 		return rq->mpwqe.wq.cur_sz;
342 	default:
343 		return rq->wqe.wq.cur_sz;
344 	}
345 }
346 
347 static inline u16 mlx5e_rqwq_get_head(struct mlx5e_rq *rq)
348 {
349 	switch (rq->wq_type) {
350 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
351 		return mlx5_wq_ll_get_head(&rq->mpwqe.wq);
352 	default:
353 		return mlx5_wq_cyc_get_head(&rq->wqe.wq);
354 	}
355 }
356 
357 static inline u16 mlx5e_rqwq_get_wqe_counter(struct mlx5e_rq *rq)
358 {
359 	switch (rq->wq_type) {
360 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
361 		return mlx5_wq_ll_get_counter(&rq->mpwqe.wq);
362 	default:
363 		return mlx5_wq_cyc_get_counter(&rq->wqe.wq);
364 	}
365 }
366 
367 /* SW parser related functions */
368 
369 struct mlx5e_swp_spec {
370 	__be16 l3_proto;
371 	u8 l4_proto;
372 	u8 is_tun;
373 	__be16 tun_l3_proto;
374 	u8 tun_l4_proto;
375 };
376 
377 static inline void mlx5e_eseg_swp_offsets_add_vlan(struct mlx5_wqe_eth_seg *eseg)
378 {
379 	/* SWP offsets are in 2-bytes words */
380 	eseg->swp_outer_l3_offset += VLAN_HLEN / 2;
381 	eseg->swp_outer_l4_offset += VLAN_HLEN / 2;
382 	eseg->swp_inner_l3_offset += VLAN_HLEN / 2;
383 	eseg->swp_inner_l4_offset += VLAN_HLEN / 2;
384 }
385 
386 static inline void
387 mlx5e_set_eseg_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg,
388 		   struct mlx5e_swp_spec *swp_spec)
389 {
390 	/* SWP offsets are in 2-bytes words */
391 	eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2;
392 	if (swp_spec->l3_proto == htons(ETH_P_IPV6))
393 		eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6;
394 	if (swp_spec->l4_proto) {
395 		eseg->swp_outer_l4_offset = skb_transport_offset(skb) / 2;
396 		if (swp_spec->l4_proto == IPPROTO_UDP)
397 			eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_UDP;
398 	}
399 
400 	if (swp_spec->is_tun) {
401 		eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2;
402 		if (swp_spec->tun_l3_proto == htons(ETH_P_IPV6))
403 			eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
404 	} else { /* typically for ipsec when xfrm mode != XFRM_MODE_TUNNEL */
405 		eseg->swp_inner_l3_offset = skb_network_offset(skb) / 2;
406 		if (swp_spec->l3_proto == htons(ETH_P_IPV6))
407 			eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
408 	}
409 	switch (swp_spec->tun_l4_proto) {
410 	case IPPROTO_UDP:
411 		eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP;
412 		fallthrough;
413 	case IPPROTO_TCP:
414 		eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2;
415 		break;
416 	}
417 }
418 
419 #define MLX5E_STOP_ROOM(wqebbs) ((wqebbs) * 2 - 1)
420 
421 static inline u16 mlx5e_stop_room_for_wqe(struct mlx5_core_dev *mdev, u16 wqe_size)
422 {
423 	/* A WQE must not cross the page boundary, hence two conditions:
424 	 * 1. Its size must not exceed the page size.
425 	 * 2. If the WQE size is X, and the space remaining in a page is less
426 	 *    than X, this space needs to be padded with NOPs. So, one WQE of
427 	 *    size X may require up to X-1 WQEBBs of padding, which makes the
428 	 *    stop room of X-1 + X.
429 	 * WQE size is also limited by the hardware limit.
430 	 */
431 	WARN_ONCE(wqe_size > mlx5e_get_max_sq_wqebbs(mdev),
432 		  "wqe_size %u is greater than max SQ WQEBBs %u",
433 		  wqe_size, mlx5e_get_max_sq_wqebbs(mdev));
434 
435 
436 	return MLX5E_STOP_ROOM(wqe_size);
437 }
438 
439 static inline u16 mlx5e_stop_room_for_max_wqe(struct mlx5_core_dev *mdev)
440 {
441 	return MLX5E_STOP_ROOM(mlx5e_get_max_sq_wqebbs(mdev));
442 }
443 
444 static inline bool mlx5e_icosq_can_post_wqe(struct mlx5e_icosq *sq, u16 wqe_size)
445 {
446 	u16 room = sq->reserved_room;
447 
448 	WARN_ONCE(wqe_size > sq->max_sq_wqebbs,
449 		  "wqe_size %u is greater than max SQ WQEBBs %u",
450 		  wqe_size, sq->max_sq_wqebbs);
451 
452 	room += MLX5E_STOP_ROOM(wqe_size);
453 
454 	return mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room);
455 }
456 #endif
457