1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #ifndef __MLX5_EN_TXRX_H___ 5 #define __MLX5_EN_TXRX_H___ 6 7 #include "en.h" 8 #include <linux/indirect_call_wrapper.h> 9 10 #define MLX5E_TX_WQE_EMPTY_DS_COUNT (sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS) 11 12 /* The mult of MLX5_SEND_WQE_MAX_WQEBBS * MLX5_SEND_WQEBB_NUM_DS 13 * (16 * 4 == 64) does not fit in the 6-bit DS field of Ctrl Segment. 14 * We use a bound lower that MLX5_SEND_WQE_MAX_WQEBBS to let a 15 * full-session WQE be cache-aligned. 16 */ 17 #if L1_CACHE_BYTES < 128 18 #define MLX5E_TX_MPW_MAX_WQEBBS (MLX5_SEND_WQE_MAX_WQEBBS - 1) 19 #else 20 #define MLX5E_TX_MPW_MAX_WQEBBS (MLX5_SEND_WQE_MAX_WQEBBS - 2) 21 #endif 22 23 #define MLX5E_TX_MPW_MAX_NUM_DS (MLX5E_TX_MPW_MAX_WQEBBS * MLX5_SEND_WQEBB_NUM_DS) 24 25 #define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start)) 26 27 #define MLX5E_RX_ERR_CQE(cqe) (get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND) 28 29 static inline 30 ktime_t mlx5e_cqe_ts_to_ns(cqe_ts_to_ns func, struct mlx5_clock *clock, u64 cqe_ts) 31 { 32 return INDIRECT_CALL_2(func, mlx5_real_time_cyc2time, mlx5_timecounter_cyc2time, 33 clock, cqe_ts); 34 } 35 36 enum mlx5e_icosq_wqe_type { 37 MLX5E_ICOSQ_WQE_NOP, 38 MLX5E_ICOSQ_WQE_UMR_RX, 39 MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR, 40 #ifdef CONFIG_MLX5_EN_TLS 41 MLX5E_ICOSQ_WQE_UMR_TLS, 42 MLX5E_ICOSQ_WQE_SET_PSV_TLS, 43 MLX5E_ICOSQ_WQE_GET_PSV_TLS, 44 #endif 45 }; 46 47 /* General */ 48 static inline bool mlx5e_skb_is_multicast(struct sk_buff *skb) 49 { 50 return skb->pkt_type == PACKET_MULTICAST || skb->pkt_type == PACKET_BROADCAST; 51 } 52 53 void mlx5e_trigger_irq(struct mlx5e_icosq *sq); 54 void mlx5e_completion_event(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe); 55 void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event); 56 int mlx5e_napi_poll(struct napi_struct *napi, int budget); 57 int mlx5e_poll_ico_cq(struct mlx5e_cq *cq); 58 59 /* RX */ 60 void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info); 61 void mlx5e_page_release_dynamic(struct mlx5e_rq *rq, 62 struct mlx5e_dma_info *dma_info, 63 bool recycle); 64 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)); 65 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)); 66 int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget); 67 void mlx5e_free_rx_descs(struct mlx5e_rq *rq); 68 void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq); 69 70 /* TX */ 71 u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, 72 struct net_device *sb_dev); 73 netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev); 74 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget); 75 void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq); 76 77 static inline bool 78 mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n) 79 { 80 return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc); 81 } 82 83 static inline void *mlx5e_fetch_wqe(struct mlx5_wq_cyc *wq, u16 pi, size_t wqe_size) 84 { 85 void *wqe; 86 87 wqe = mlx5_wq_cyc_get_wqe(wq, pi); 88 memset(wqe, 0, wqe_size); 89 90 return wqe; 91 } 92 93 #define MLX5E_TX_FETCH_WQE(sq, pi) \ 94 ((struct mlx5e_tx_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_tx_wqe))) 95 96 static inline struct mlx5e_tx_wqe * 97 mlx5e_post_nop(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc) 98 { 99 u16 pi = mlx5_wq_cyc_ctr2ix(wq, *pc); 100 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); 101 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 102 103 memset(cseg, 0, sizeof(*cseg)); 104 105 cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP); 106 cseg->qpn_ds = cpu_to_be32((sqn << 8) | 0x01); 107 108 (*pc)++; 109 110 return wqe; 111 } 112 113 static inline struct mlx5e_tx_wqe * 114 mlx5e_post_nop_fence(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc) 115 { 116 u16 pi = mlx5_wq_cyc_ctr2ix(wq, *pc); 117 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); 118 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 119 120 memset(cseg, 0, sizeof(*cseg)); 121 122 cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP); 123 cseg->qpn_ds = cpu_to_be32((sqn << 8) | 0x01); 124 cseg->fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL; 125 126 (*pc)++; 127 128 return wqe; 129 } 130 131 struct mlx5e_tx_wqe_info { 132 struct sk_buff *skb; 133 u32 num_bytes; 134 u8 num_wqebbs; 135 u8 num_dma; 136 u8 num_fifo_pkts; 137 #ifdef CONFIG_MLX5_EN_TLS 138 struct page *resync_dump_frag_page; 139 #endif 140 }; 141 142 static inline u16 mlx5e_txqsq_get_next_pi(struct mlx5e_txqsq *sq, u16 size) 143 { 144 struct mlx5_wq_cyc *wq = &sq->wq; 145 u16 pi, contig_wqebbs; 146 147 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 148 contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi); 149 if (unlikely(contig_wqebbs < size)) { 150 struct mlx5e_tx_wqe_info *wi, *edge_wi; 151 152 wi = &sq->db.wqe_info[pi]; 153 edge_wi = wi + contig_wqebbs; 154 155 /* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */ 156 for (; wi < edge_wi; wi++) { 157 *wi = (struct mlx5e_tx_wqe_info) { 158 .num_wqebbs = 1, 159 }; 160 mlx5e_post_nop(wq, sq->sqn, &sq->pc); 161 } 162 sq->stats->nop += contig_wqebbs; 163 164 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 165 } 166 167 return pi; 168 } 169 170 struct mlx5e_shampo_umr { 171 u16 len; 172 }; 173 174 struct mlx5e_icosq_wqe_info { 175 u8 wqe_type; 176 u8 num_wqebbs; 177 178 /* Auxiliary data for different wqe types. */ 179 union { 180 struct { 181 struct mlx5e_rq *rq; 182 } umr; 183 struct mlx5e_shampo_umr shampo; 184 #ifdef CONFIG_MLX5_EN_TLS 185 struct { 186 struct mlx5e_ktls_offload_context_rx *priv_rx; 187 } tls_set_params; 188 struct { 189 struct mlx5e_ktls_rx_resync_buf *buf; 190 } tls_get_params; 191 #endif 192 }; 193 }; 194 195 void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq); 196 197 static inline u16 mlx5e_icosq_get_next_pi(struct mlx5e_icosq *sq, u16 size) 198 { 199 struct mlx5_wq_cyc *wq = &sq->wq; 200 u16 pi, contig_wqebbs; 201 202 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 203 contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi); 204 if (unlikely(contig_wqebbs < size)) { 205 struct mlx5e_icosq_wqe_info *wi, *edge_wi; 206 207 wi = &sq->db.wqe_info[pi]; 208 edge_wi = wi + contig_wqebbs; 209 210 /* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */ 211 for (; wi < edge_wi; wi++) { 212 *wi = (struct mlx5e_icosq_wqe_info) { 213 .wqe_type = MLX5E_ICOSQ_WQE_NOP, 214 .num_wqebbs = 1, 215 }; 216 mlx5e_post_nop(wq, sq->sqn, &sq->pc); 217 } 218 219 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 220 } 221 222 return pi; 223 } 224 225 static inline void 226 mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, void __iomem *uar_map, 227 struct mlx5_wqe_ctrl_seg *ctrl) 228 { 229 ctrl->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE; 230 /* ensure wqe is visible to device before updating doorbell record */ 231 dma_wmb(); 232 233 *wq->db = cpu_to_be32(pc); 234 235 /* ensure doorbell record is visible to device before ringing the 236 * doorbell 237 */ 238 wmb(); 239 240 mlx5_write64((__be32 *)ctrl, uar_map); 241 } 242 243 static inline void mlx5e_cq_arm(struct mlx5e_cq *cq) 244 { 245 struct mlx5_core_cq *mcq; 246 247 mcq = &cq->mcq; 248 mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc); 249 } 250 251 static inline struct mlx5e_sq_dma * 252 mlx5e_dma_get(struct mlx5e_txqsq *sq, u32 i) 253 { 254 return &sq->db.dma_fifo[i & sq->dma_fifo_mask]; 255 } 256 257 static inline void 258 mlx5e_dma_push(struct mlx5e_txqsq *sq, dma_addr_t addr, u32 size, 259 enum mlx5e_dma_map_type map_type) 260 { 261 struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, sq->dma_fifo_pc++); 262 263 dma->addr = addr; 264 dma->size = size; 265 dma->type = map_type; 266 } 267 268 static inline 269 struct sk_buff **mlx5e_skb_fifo_get(struct mlx5e_skb_fifo *fifo, u16 i) 270 { 271 return &fifo->fifo[i & fifo->mask]; 272 } 273 274 static inline 275 void mlx5e_skb_fifo_push(struct mlx5e_skb_fifo *fifo, struct sk_buff *skb) 276 { 277 struct sk_buff **skb_item = mlx5e_skb_fifo_get(fifo, (*fifo->pc)++); 278 279 *skb_item = skb; 280 } 281 282 static inline 283 struct sk_buff *mlx5e_skb_fifo_pop(struct mlx5e_skb_fifo *fifo) 284 { 285 return *mlx5e_skb_fifo_get(fifo, (*fifo->cc)++); 286 } 287 288 static inline void 289 mlx5e_tx_dma_unmap(struct device *pdev, struct mlx5e_sq_dma *dma) 290 { 291 switch (dma->type) { 292 case MLX5E_DMA_MAP_SINGLE: 293 dma_unmap_single(pdev, dma->addr, dma->size, DMA_TO_DEVICE); 294 break; 295 case MLX5E_DMA_MAP_PAGE: 296 dma_unmap_page(pdev, dma->addr, dma->size, DMA_TO_DEVICE); 297 break; 298 default: 299 WARN_ONCE(true, "mlx5e_tx_dma_unmap unknown DMA type!\n"); 300 } 301 } 302 303 void mlx5e_sq_xmit_simple(struct mlx5e_txqsq *sq, struct sk_buff *skb, bool xmit_more); 304 void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq); 305 306 static inline bool mlx5e_tx_mpwqe_is_full(struct mlx5e_tx_mpwqe *session) 307 { 308 return session->ds_count == MLX5E_TX_MPW_MAX_NUM_DS; 309 } 310 311 static inline void mlx5e_rqwq_reset(struct mlx5e_rq *rq) 312 { 313 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 314 mlx5_wq_ll_reset(&rq->mpwqe.wq); 315 rq->mpwqe.actual_wq_head = 0; 316 } else { 317 mlx5_wq_cyc_reset(&rq->wqe.wq); 318 } 319 } 320 321 static inline void mlx5e_dump_error_cqe(struct mlx5e_cq *cq, u32 qn, 322 struct mlx5_err_cqe *err_cqe) 323 { 324 struct mlx5_cqwq *wq = &cq->wq; 325 u32 ci; 326 327 ci = mlx5_cqwq_ctr2ix(wq, wq->cc - 1); 328 329 netdev_err(cq->netdev, 330 "Error cqe on cqn 0x%x, ci 0x%x, qn 0x%x, opcode 0x%x, syndrome 0x%x, vendor syndrome 0x%x\n", 331 cq->mcq.cqn, ci, qn, 332 get_cqe_opcode((struct mlx5_cqe64 *)err_cqe), 333 err_cqe->syndrome, err_cqe->vendor_err_synd); 334 mlx5_dump_err_cqe(cq->mdev, err_cqe); 335 } 336 337 static inline u32 mlx5e_rqwq_get_size(struct mlx5e_rq *rq) 338 { 339 switch (rq->wq_type) { 340 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 341 return mlx5_wq_ll_get_size(&rq->mpwqe.wq); 342 default: 343 return mlx5_wq_cyc_get_size(&rq->wqe.wq); 344 } 345 } 346 347 static inline u32 mlx5e_rqwq_get_cur_sz(struct mlx5e_rq *rq) 348 { 349 switch (rq->wq_type) { 350 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 351 return rq->mpwqe.wq.cur_sz; 352 default: 353 return rq->wqe.wq.cur_sz; 354 } 355 } 356 357 static inline u16 mlx5e_rqwq_get_head(struct mlx5e_rq *rq) 358 { 359 switch (rq->wq_type) { 360 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 361 return mlx5_wq_ll_get_head(&rq->mpwqe.wq); 362 default: 363 return mlx5_wq_cyc_get_head(&rq->wqe.wq); 364 } 365 } 366 367 static inline u16 mlx5e_rqwq_get_wqe_counter(struct mlx5e_rq *rq) 368 { 369 switch (rq->wq_type) { 370 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 371 return mlx5_wq_ll_get_counter(&rq->mpwqe.wq); 372 default: 373 return mlx5_wq_cyc_get_counter(&rq->wqe.wq); 374 } 375 } 376 377 /* SW parser related functions */ 378 379 struct mlx5e_swp_spec { 380 __be16 l3_proto; 381 u8 l4_proto; 382 u8 is_tun; 383 __be16 tun_l3_proto; 384 u8 tun_l4_proto; 385 }; 386 387 static inline void mlx5e_eseg_swp_offsets_add_vlan(struct mlx5_wqe_eth_seg *eseg) 388 { 389 /* SWP offsets are in 2-bytes words */ 390 eseg->swp_outer_l3_offset += VLAN_HLEN / 2; 391 eseg->swp_outer_l4_offset += VLAN_HLEN / 2; 392 eseg->swp_inner_l3_offset += VLAN_HLEN / 2; 393 eseg->swp_inner_l4_offset += VLAN_HLEN / 2; 394 } 395 396 static inline void 397 mlx5e_set_eseg_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg, 398 struct mlx5e_swp_spec *swp_spec) 399 { 400 /* SWP offsets are in 2-bytes words */ 401 eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2; 402 if (swp_spec->l3_proto == htons(ETH_P_IPV6)) 403 eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6; 404 if (swp_spec->l4_proto) { 405 eseg->swp_outer_l4_offset = skb_transport_offset(skb) / 2; 406 if (swp_spec->l4_proto == IPPROTO_UDP) 407 eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_UDP; 408 } 409 410 if (swp_spec->is_tun) { 411 eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2; 412 if (swp_spec->tun_l3_proto == htons(ETH_P_IPV6)) 413 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; 414 } else { /* typically for ipsec when xfrm mode != XFRM_MODE_TUNNEL */ 415 eseg->swp_inner_l3_offset = skb_network_offset(skb) / 2; 416 if (swp_spec->l3_proto == htons(ETH_P_IPV6)) 417 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; 418 } 419 switch (swp_spec->tun_l4_proto) { 420 case IPPROTO_UDP: 421 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP; 422 fallthrough; 423 case IPPROTO_TCP: 424 eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2; 425 break; 426 } 427 } 428 429 static inline u16 mlx5e_stop_room_for_wqe(u16 wqe_size) 430 { 431 BUILD_BUG_ON(PAGE_SIZE / MLX5_SEND_WQE_BB < MLX5_SEND_WQE_MAX_WQEBBS); 432 433 /* A WQE must not cross the page boundary, hence two conditions: 434 * 1. Its size must not exceed the page size. 435 * 2. If the WQE size is X, and the space remaining in a page is less 436 * than X, this space needs to be padded with NOPs. So, one WQE of 437 * size X may require up to X-1 WQEBBs of padding, which makes the 438 * stop room of X-1 + X. 439 * WQE size is also limited by the hardware limit. 440 */ 441 442 if (__builtin_constant_p(wqe_size)) 443 BUILD_BUG_ON(wqe_size > MLX5_SEND_WQE_MAX_WQEBBS); 444 else 445 WARN_ON_ONCE(wqe_size > MLX5_SEND_WQE_MAX_WQEBBS); 446 447 return wqe_size * 2 - 1; 448 } 449 450 static inline bool mlx5e_icosq_can_post_wqe(struct mlx5e_icosq *sq, u16 wqe_size) 451 { 452 u16 room = sq->reserved_room + mlx5e_stop_room_for_wqe(wqe_size); 453 454 return mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room); 455 } 456 #endif 457