1542578c6STariq Toukan /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2542578c6STariq Toukan /* Copyright (c) 2019 Mellanox Technologies. */ 3542578c6STariq Toukan 4542578c6STariq Toukan #ifndef __MLX5_EN_TXRX_H___ 5542578c6STariq Toukan #define __MLX5_EN_TXRX_H___ 6542578c6STariq Toukan 7542578c6STariq Toukan #include "en.h" 8542578c6STariq Toukan 968865419STariq Toukan #define MLX5E_SQ_NOPS_ROOM (MLX5_SEND_WQE_MAX_WQEBBS - 1) 1001614d4fSTariq Toukan #define MLX5E_SQ_STOP_ROOM (MLX5_SEND_WQE_MAX_WQEBBS +\ 1101614d4fSTariq Toukan MLX5E_SQ_NOPS_ROOM) 1201614d4fSTariq Toukan 1301614d4fSTariq Toukan #ifndef CONFIG_MLX5_EN_TLS 1401614d4fSTariq Toukan #define MLX5E_SQ_TLS_ROOM (0) 1501614d4fSTariq Toukan #else 1601614d4fSTariq Toukan /* TLS offload requires additional stop_room for: 1701614d4fSTariq Toukan * - a resync SKB. 18d2ead1f3STariq Toukan * kTLS offload requires additional stop_room for: 19d2ead1f3STariq Toukan * - static params WQE, 20d2ead1f3STariq Toukan * - progress params WQE, and 21d2ead1f3STariq Toukan * - resync DUMP per frag. 2201614d4fSTariq Toukan */ 2301614d4fSTariq Toukan #define MLX5E_SQ_TLS_ROOM \ 24d2ead1f3STariq Toukan (MLX5_SEND_WQE_MAX_WQEBBS + \ 25d2ead1f3STariq Toukan MLX5E_KTLS_STATIC_WQEBBS + MLX5E_KTLS_PROGRESS_WQEBBS + \ 26d2ead1f3STariq Toukan MAX_SKB_FRAGS * MLX5E_KTLS_MAX_DUMP_WQEBBS) 2701614d4fSTariq Toukan #endif 2801614d4fSTariq Toukan 29542578c6STariq Toukan #define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start)) 30542578c6STariq Toukan 31542578c6STariq Toukan static inline bool 32542578c6STariq Toukan mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n) 33542578c6STariq Toukan { 34542578c6STariq Toukan return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc); 35542578c6STariq Toukan } 36542578c6STariq Toukan 37fd1b2259STariq Toukan static inline void * 38fd1b2259STariq Toukan mlx5e_sq_fetch_wqe(struct mlx5e_txqsq *sq, size_t size, u16 *pi) 39542578c6STariq Toukan { 40542578c6STariq Toukan struct mlx5_wq_cyc *wq = &sq->wq; 41fd1b2259STariq Toukan void *wqe; 42542578c6STariq Toukan 43542578c6STariq Toukan *pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 44fd1b2259STariq Toukan wqe = mlx5_wq_cyc_get_wqe(wq, *pi); 45fd1b2259STariq Toukan memset(wqe, 0, size); 46fd1b2259STariq Toukan 47fd1b2259STariq Toukan return wqe; 48542578c6STariq Toukan } 49542578c6STariq Toukan 50542578c6STariq Toukan static inline struct mlx5e_tx_wqe * 51542578c6STariq Toukan mlx5e_post_nop(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc) 52542578c6STariq Toukan { 53542578c6STariq Toukan u16 pi = mlx5_wq_cyc_ctr2ix(wq, *pc); 54542578c6STariq Toukan struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); 55542578c6STariq Toukan struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 56542578c6STariq Toukan 57542578c6STariq Toukan memset(cseg, 0, sizeof(*cseg)); 58542578c6STariq Toukan 59542578c6STariq Toukan cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP); 60542578c6STariq Toukan cseg->qpn_ds = cpu_to_be32((sqn << 8) | 0x01); 61542578c6STariq Toukan 62542578c6STariq Toukan (*pc)++; 63542578c6STariq Toukan 64542578c6STariq Toukan return wqe; 65542578c6STariq Toukan } 66542578c6STariq Toukan 6737badd15STariq Toukan static inline struct mlx5e_tx_wqe * 6837badd15STariq Toukan mlx5e_post_nop_fence(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc) 6937badd15STariq Toukan { 7037badd15STariq Toukan u16 pi = mlx5_wq_cyc_ctr2ix(wq, *pc); 7137badd15STariq Toukan struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); 7237badd15STariq Toukan struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 7337badd15STariq Toukan 7437badd15STariq Toukan memset(cseg, 0, sizeof(*cseg)); 7537badd15STariq Toukan 7637badd15STariq Toukan cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP); 7737badd15STariq Toukan cseg->qpn_ds = cpu_to_be32((sqn << 8) | 0x01); 7837badd15STariq Toukan cseg->fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL; 7937badd15STariq Toukan 8037badd15STariq Toukan (*pc)++; 8137badd15STariq Toukan 8237badd15STariq Toukan return wqe; 8337badd15STariq Toukan } 8437badd15STariq Toukan 85542578c6STariq Toukan static inline void 86542578c6STariq Toukan mlx5e_fill_sq_frag_edge(struct mlx5e_txqsq *sq, struct mlx5_wq_cyc *wq, 87542578c6STariq Toukan u16 pi, u16 nnops) 88542578c6STariq Toukan { 89542578c6STariq Toukan struct mlx5e_tx_wqe_info *edge_wi, *wi = &sq->db.wqe_info[pi]; 90542578c6STariq Toukan 91542578c6STariq Toukan edge_wi = wi + nnops; 92542578c6STariq Toukan 93542578c6STariq Toukan /* fill sq frag edge with nops to avoid wqe wrapping two pages */ 94542578c6STariq Toukan for (; wi < edge_wi; wi++) { 95542578c6STariq Toukan wi->skb = NULL; 96542578c6STariq Toukan wi->num_wqebbs = 1; 97542578c6STariq Toukan mlx5e_post_nop(wq, sq->sqn, &sq->pc); 98542578c6STariq Toukan } 99542578c6STariq Toukan sq->stats->nop += nnops; 100542578c6STariq Toukan } 101542578c6STariq Toukan 102542578c6STariq Toukan static inline void 103542578c6STariq Toukan mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, void __iomem *uar_map, 104542578c6STariq Toukan struct mlx5_wqe_ctrl_seg *ctrl) 105542578c6STariq Toukan { 106542578c6STariq Toukan ctrl->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 107542578c6STariq Toukan /* ensure wqe is visible to device before updating doorbell record */ 108542578c6STariq Toukan dma_wmb(); 109542578c6STariq Toukan 110542578c6STariq Toukan *wq->db = cpu_to_be32(pc); 111542578c6STariq Toukan 112542578c6STariq Toukan /* ensure doorbell record is visible to device before ringing the 113542578c6STariq Toukan * doorbell 114542578c6STariq Toukan */ 115542578c6STariq Toukan wmb(); 116542578c6STariq Toukan 117542578c6STariq Toukan mlx5_write64((__be32 *)ctrl, uar_map); 118542578c6STariq Toukan } 119542578c6STariq Toukan 120740114a8STariq Toukan static inline bool mlx5e_transport_inline_tx_wqe(struct mlx5e_tx_wqe *wqe) 121740114a8STariq Toukan { 122740114a8STariq Toukan return !!wqe->ctrl.tisn; 123740114a8STariq Toukan } 124740114a8STariq Toukan 125542578c6STariq Toukan static inline void mlx5e_cq_arm(struct mlx5e_cq *cq) 126542578c6STariq Toukan { 127542578c6STariq Toukan struct mlx5_core_cq *mcq; 128542578c6STariq Toukan 129542578c6STariq Toukan mcq = &cq->mcq; 130542578c6STariq Toukan mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc); 131542578c6STariq Toukan } 132542578c6STariq Toukan 133542578c6STariq Toukan static inline struct mlx5e_sq_dma * 134542578c6STariq Toukan mlx5e_dma_get(struct mlx5e_txqsq *sq, u32 i) 135542578c6STariq Toukan { 136542578c6STariq Toukan return &sq->db.dma_fifo[i & sq->dma_fifo_mask]; 137542578c6STariq Toukan } 138542578c6STariq Toukan 139542578c6STariq Toukan static inline void 140542578c6STariq Toukan mlx5e_dma_push(struct mlx5e_txqsq *sq, dma_addr_t addr, u32 size, 141542578c6STariq Toukan enum mlx5e_dma_map_type map_type) 142542578c6STariq Toukan { 143542578c6STariq Toukan struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, sq->dma_fifo_pc++); 144542578c6STariq Toukan 145542578c6STariq Toukan dma->addr = addr; 146542578c6STariq Toukan dma->size = size; 147542578c6STariq Toukan dma->type = map_type; 148542578c6STariq Toukan } 149542578c6STariq Toukan 150542578c6STariq Toukan static inline void 151542578c6STariq Toukan mlx5e_tx_dma_unmap(struct device *pdev, struct mlx5e_sq_dma *dma) 152542578c6STariq Toukan { 153542578c6STariq Toukan switch (dma->type) { 154542578c6STariq Toukan case MLX5E_DMA_MAP_SINGLE: 155542578c6STariq Toukan dma_unmap_single(pdev, dma->addr, dma->size, DMA_TO_DEVICE); 156542578c6STariq Toukan break; 157542578c6STariq Toukan case MLX5E_DMA_MAP_PAGE: 158542578c6STariq Toukan dma_unmap_page(pdev, dma->addr, dma->size, DMA_TO_DEVICE); 159542578c6STariq Toukan break; 160542578c6STariq Toukan default: 161542578c6STariq Toukan WARN_ONCE(true, "mlx5e_tx_dma_unmap unknown DMA type!\n"); 162542578c6STariq Toukan } 163542578c6STariq Toukan } 164542578c6STariq Toukan 165542578c6STariq Toukan /* SW parser related functions */ 166542578c6STariq Toukan 167542578c6STariq Toukan struct mlx5e_swp_spec { 168542578c6STariq Toukan __be16 l3_proto; 169542578c6STariq Toukan u8 l4_proto; 170542578c6STariq Toukan u8 is_tun; 171542578c6STariq Toukan __be16 tun_l3_proto; 172542578c6STariq Toukan u8 tun_l4_proto; 173542578c6STariq Toukan }; 174542578c6STariq Toukan 175542578c6STariq Toukan static inline void 176542578c6STariq Toukan mlx5e_set_eseg_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg, 177542578c6STariq Toukan struct mlx5e_swp_spec *swp_spec) 178542578c6STariq Toukan { 179542578c6STariq Toukan /* SWP offsets are in 2-bytes words */ 180542578c6STariq Toukan eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2; 181542578c6STariq Toukan if (swp_spec->l3_proto == htons(ETH_P_IPV6)) 182542578c6STariq Toukan eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6; 183542578c6STariq Toukan if (swp_spec->l4_proto) { 184542578c6STariq Toukan eseg->swp_outer_l4_offset = skb_transport_offset(skb) / 2; 185542578c6STariq Toukan if (swp_spec->l4_proto == IPPROTO_UDP) 186542578c6STariq Toukan eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_UDP; 187542578c6STariq Toukan } 188542578c6STariq Toukan 189542578c6STariq Toukan if (swp_spec->is_tun) { 190542578c6STariq Toukan eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2; 191542578c6STariq Toukan if (swp_spec->tun_l3_proto == htons(ETH_P_IPV6)) 192542578c6STariq Toukan eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; 193542578c6STariq Toukan } else { /* typically for ipsec when xfrm mode != XFRM_MODE_TUNNEL */ 194542578c6STariq Toukan eseg->swp_inner_l3_offset = skb_network_offset(skb) / 2; 195542578c6STariq Toukan if (swp_spec->l3_proto == htons(ETH_P_IPV6)) 196542578c6STariq Toukan eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; 197542578c6STariq Toukan } 198542578c6STariq Toukan switch (swp_spec->tun_l4_proto) { 199542578c6STariq Toukan case IPPROTO_UDP: 200542578c6STariq Toukan eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP; 201542578c6STariq Toukan /* fall through */ 202542578c6STariq Toukan case IPPROTO_TCP: 203542578c6STariq Toukan eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2; 204542578c6STariq Toukan break; 205542578c6STariq Toukan } 206542578c6STariq Toukan } 207542578c6STariq Toukan 208542578c6STariq Toukan #endif 209