1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #ifndef __MLX5_EN_TXRX_H___ 5 #define __MLX5_EN_TXRX_H___ 6 7 #include "en.h" 8 #include <linux/indirect_call_wrapper.h> 9 10 #define MLX5E_TX_WQE_EMPTY_DS_COUNT (sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS) 11 12 #define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start)) 13 14 #define MLX5E_RX_ERR_CQE(cqe) (get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND) 15 16 static inline 17 ktime_t mlx5e_cqe_ts_to_ns(cqe_ts_to_ns func, struct mlx5_clock *clock, u64 cqe_ts) 18 { 19 return INDIRECT_CALL_2(func, mlx5_real_time_cyc2time, mlx5_timecounter_cyc2time, 20 clock, cqe_ts); 21 } 22 23 enum mlx5e_icosq_wqe_type { 24 MLX5E_ICOSQ_WQE_NOP, 25 MLX5E_ICOSQ_WQE_UMR_RX, 26 MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR, 27 #ifdef CONFIG_MLX5_EN_TLS 28 MLX5E_ICOSQ_WQE_UMR_TLS, 29 MLX5E_ICOSQ_WQE_SET_PSV_TLS, 30 MLX5E_ICOSQ_WQE_GET_PSV_TLS, 31 #endif 32 }; 33 34 /* General */ 35 static inline bool mlx5e_skb_is_multicast(struct sk_buff *skb) 36 { 37 return skb->pkt_type == PACKET_MULTICAST || skb->pkt_type == PACKET_BROADCAST; 38 } 39 40 void mlx5e_trigger_irq(struct mlx5e_icosq *sq); 41 void mlx5e_completion_event(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe); 42 void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event); 43 int mlx5e_napi_poll(struct napi_struct *napi, int budget); 44 int mlx5e_poll_ico_cq(struct mlx5e_cq *cq); 45 46 /* RX */ 47 void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info); 48 void mlx5e_page_release_dynamic(struct mlx5e_rq *rq, 49 struct mlx5e_dma_info *dma_info, 50 bool recycle); 51 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)); 52 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)); 53 int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget); 54 void mlx5e_free_rx_descs(struct mlx5e_rq *rq); 55 void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq); 56 57 /* TX */ 58 netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev); 59 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget); 60 void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq); 61 62 static inline bool 63 mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n) 64 { 65 return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc); 66 } 67 68 static inline void *mlx5e_fetch_wqe(struct mlx5_wq_cyc *wq, u16 pi, size_t wqe_size) 69 { 70 void *wqe; 71 72 wqe = mlx5_wq_cyc_get_wqe(wq, pi); 73 memset(wqe, 0, wqe_size); 74 75 return wqe; 76 } 77 78 #define MLX5E_TX_FETCH_WQE(sq, pi) \ 79 ((struct mlx5e_tx_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_tx_wqe))) 80 81 static inline struct mlx5e_tx_wqe * 82 mlx5e_post_nop(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc) 83 { 84 u16 pi = mlx5_wq_cyc_ctr2ix(wq, *pc); 85 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); 86 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 87 88 memset(cseg, 0, sizeof(*cseg)); 89 90 cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP); 91 cseg->qpn_ds = cpu_to_be32((sqn << 8) | 0x01); 92 93 (*pc)++; 94 95 return wqe; 96 } 97 98 static inline struct mlx5e_tx_wqe * 99 mlx5e_post_nop_fence(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc) 100 { 101 u16 pi = mlx5_wq_cyc_ctr2ix(wq, *pc); 102 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); 103 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 104 105 memset(cseg, 0, sizeof(*cseg)); 106 107 cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP); 108 cseg->qpn_ds = cpu_to_be32((sqn << 8) | 0x01); 109 cseg->fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL; 110 111 (*pc)++; 112 113 return wqe; 114 } 115 116 struct mlx5e_tx_wqe_info { 117 struct sk_buff *skb; 118 u32 num_bytes; 119 u8 num_wqebbs; 120 u8 num_dma; 121 u8 num_fifo_pkts; 122 #ifdef CONFIG_MLX5_EN_TLS 123 struct page *resync_dump_frag_page; 124 #endif 125 }; 126 127 static inline u16 mlx5e_txqsq_get_next_pi(struct mlx5e_txqsq *sq, u16 size) 128 { 129 struct mlx5_wq_cyc *wq = &sq->wq; 130 u16 pi, contig_wqebbs; 131 132 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 133 contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi); 134 if (unlikely(contig_wqebbs < size)) { 135 struct mlx5e_tx_wqe_info *wi, *edge_wi; 136 137 wi = &sq->db.wqe_info[pi]; 138 edge_wi = wi + contig_wqebbs; 139 140 /* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */ 141 for (; wi < edge_wi; wi++) { 142 *wi = (struct mlx5e_tx_wqe_info) { 143 .num_wqebbs = 1, 144 }; 145 mlx5e_post_nop(wq, sq->sqn, &sq->pc); 146 } 147 sq->stats->nop += contig_wqebbs; 148 149 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 150 } 151 152 return pi; 153 } 154 155 static inline u16 mlx5e_shampo_get_cqe_header_index(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) 156 { 157 return be16_to_cpu(cqe->shampo.header_entry_index) & (rq->mpwqe.shampo->hd_per_wq - 1); 158 } 159 160 struct mlx5e_shampo_umr { 161 u16 len; 162 }; 163 164 struct mlx5e_icosq_wqe_info { 165 u8 wqe_type; 166 u8 num_wqebbs; 167 168 /* Auxiliary data for different wqe types. */ 169 union { 170 struct { 171 struct mlx5e_rq *rq; 172 } umr; 173 struct mlx5e_shampo_umr shampo; 174 #ifdef CONFIG_MLX5_EN_TLS 175 struct { 176 struct mlx5e_ktls_offload_context_rx *priv_rx; 177 } tls_set_params; 178 struct { 179 struct mlx5e_ktls_rx_resync_buf *buf; 180 } tls_get_params; 181 #endif 182 }; 183 }; 184 185 void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq); 186 187 static inline u16 mlx5e_icosq_get_next_pi(struct mlx5e_icosq *sq, u16 size) 188 { 189 struct mlx5_wq_cyc *wq = &sq->wq; 190 u16 pi, contig_wqebbs; 191 192 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 193 contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi); 194 if (unlikely(contig_wqebbs < size)) { 195 struct mlx5e_icosq_wqe_info *wi, *edge_wi; 196 197 wi = &sq->db.wqe_info[pi]; 198 edge_wi = wi + contig_wqebbs; 199 200 /* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */ 201 for (; wi < edge_wi; wi++) { 202 *wi = (struct mlx5e_icosq_wqe_info) { 203 .wqe_type = MLX5E_ICOSQ_WQE_NOP, 204 .num_wqebbs = 1, 205 }; 206 mlx5e_post_nop(wq, sq->sqn, &sq->pc); 207 } 208 209 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 210 } 211 212 return pi; 213 } 214 215 static inline void 216 mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, void __iomem *uar_map, 217 struct mlx5_wqe_ctrl_seg *ctrl) 218 { 219 ctrl->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE; 220 /* ensure wqe is visible to device before updating doorbell record */ 221 dma_wmb(); 222 223 *wq->db = cpu_to_be32(pc); 224 225 /* ensure doorbell record is visible to device before ringing the 226 * doorbell 227 */ 228 wmb(); 229 230 mlx5_write64((__be32 *)ctrl, uar_map); 231 } 232 233 static inline void mlx5e_cq_arm(struct mlx5e_cq *cq) 234 { 235 struct mlx5_core_cq *mcq; 236 237 mcq = &cq->mcq; 238 mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc); 239 } 240 241 static inline struct mlx5e_sq_dma * 242 mlx5e_dma_get(struct mlx5e_txqsq *sq, u32 i) 243 { 244 return &sq->db.dma_fifo[i & sq->dma_fifo_mask]; 245 } 246 247 static inline void 248 mlx5e_dma_push(struct mlx5e_txqsq *sq, dma_addr_t addr, u32 size, 249 enum mlx5e_dma_map_type map_type) 250 { 251 struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, sq->dma_fifo_pc++); 252 253 dma->addr = addr; 254 dma->size = size; 255 dma->type = map_type; 256 } 257 258 static inline 259 struct sk_buff **mlx5e_skb_fifo_get(struct mlx5e_skb_fifo *fifo, u16 i) 260 { 261 return &fifo->fifo[i & fifo->mask]; 262 } 263 264 static inline 265 void mlx5e_skb_fifo_push(struct mlx5e_skb_fifo *fifo, struct sk_buff *skb) 266 { 267 struct sk_buff **skb_item = mlx5e_skb_fifo_get(fifo, (*fifo->pc)++); 268 269 *skb_item = skb; 270 } 271 272 static inline 273 struct sk_buff *mlx5e_skb_fifo_pop(struct mlx5e_skb_fifo *fifo) 274 { 275 return *mlx5e_skb_fifo_get(fifo, (*fifo->cc)++); 276 } 277 278 static inline void 279 mlx5e_tx_dma_unmap(struct device *pdev, struct mlx5e_sq_dma *dma) 280 { 281 switch (dma->type) { 282 case MLX5E_DMA_MAP_SINGLE: 283 dma_unmap_single(pdev, dma->addr, dma->size, DMA_TO_DEVICE); 284 break; 285 case MLX5E_DMA_MAP_PAGE: 286 dma_unmap_page(pdev, dma->addr, dma->size, DMA_TO_DEVICE); 287 break; 288 default: 289 WARN_ONCE(true, "mlx5e_tx_dma_unmap unknown DMA type!\n"); 290 } 291 } 292 293 void mlx5e_sq_xmit_simple(struct mlx5e_txqsq *sq, struct sk_buff *skb, bool xmit_more); 294 void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq); 295 296 static inline bool mlx5e_tx_mpwqe_is_full(struct mlx5e_tx_mpwqe *session, u8 max_sq_mpw_wqebbs) 297 { 298 return session->ds_count == max_sq_mpw_wqebbs * MLX5_SEND_WQEBB_NUM_DS; 299 } 300 301 static inline void mlx5e_rqwq_reset(struct mlx5e_rq *rq) 302 { 303 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 304 mlx5_wq_ll_reset(&rq->mpwqe.wq); 305 rq->mpwqe.actual_wq_head = 0; 306 } else { 307 mlx5_wq_cyc_reset(&rq->wqe.wq); 308 } 309 } 310 311 static inline void mlx5e_dump_error_cqe(struct mlx5e_cq *cq, u32 qn, 312 struct mlx5_err_cqe *err_cqe) 313 { 314 struct mlx5_cqwq *wq = &cq->wq; 315 u32 ci; 316 317 ci = mlx5_cqwq_ctr2ix(wq, wq->cc - 1); 318 319 netdev_err(cq->netdev, 320 "Error cqe on cqn 0x%x, ci 0x%x, qn 0x%x, opcode 0x%x, syndrome 0x%x, vendor syndrome 0x%x\n", 321 cq->mcq.cqn, ci, qn, 322 get_cqe_opcode((struct mlx5_cqe64 *)err_cqe), 323 err_cqe->syndrome, err_cqe->vendor_err_synd); 324 mlx5_dump_err_cqe(cq->mdev, err_cqe); 325 } 326 327 static inline u32 mlx5e_rqwq_get_size(struct mlx5e_rq *rq) 328 { 329 switch (rq->wq_type) { 330 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 331 return mlx5_wq_ll_get_size(&rq->mpwqe.wq); 332 default: 333 return mlx5_wq_cyc_get_size(&rq->wqe.wq); 334 } 335 } 336 337 static inline u32 mlx5e_rqwq_get_cur_sz(struct mlx5e_rq *rq) 338 { 339 switch (rq->wq_type) { 340 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 341 return rq->mpwqe.wq.cur_sz; 342 default: 343 return rq->wqe.wq.cur_sz; 344 } 345 } 346 347 static inline u16 mlx5e_rqwq_get_head(struct mlx5e_rq *rq) 348 { 349 switch (rq->wq_type) { 350 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 351 return mlx5_wq_ll_get_head(&rq->mpwqe.wq); 352 default: 353 return mlx5_wq_cyc_get_head(&rq->wqe.wq); 354 } 355 } 356 357 static inline u16 mlx5e_rqwq_get_wqe_counter(struct mlx5e_rq *rq) 358 { 359 switch (rq->wq_type) { 360 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 361 return mlx5_wq_ll_get_counter(&rq->mpwqe.wq); 362 default: 363 return mlx5_wq_cyc_get_counter(&rq->wqe.wq); 364 } 365 } 366 367 /* SW parser related functions */ 368 369 struct mlx5e_swp_spec { 370 __be16 l3_proto; 371 u8 l4_proto; 372 u8 is_tun; 373 __be16 tun_l3_proto; 374 u8 tun_l4_proto; 375 }; 376 377 static inline void mlx5e_eseg_swp_offsets_add_vlan(struct mlx5_wqe_eth_seg *eseg) 378 { 379 /* SWP offsets are in 2-bytes words */ 380 eseg->swp_outer_l3_offset += VLAN_HLEN / 2; 381 eseg->swp_outer_l4_offset += VLAN_HLEN / 2; 382 eseg->swp_inner_l3_offset += VLAN_HLEN / 2; 383 eseg->swp_inner_l4_offset += VLAN_HLEN / 2; 384 } 385 386 static inline void 387 mlx5e_set_eseg_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg, 388 struct mlx5e_swp_spec *swp_spec) 389 { 390 /* SWP offsets are in 2-bytes words */ 391 eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2; 392 if (swp_spec->l3_proto == htons(ETH_P_IPV6)) 393 eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6; 394 if (swp_spec->l4_proto) { 395 eseg->swp_outer_l4_offset = skb_transport_offset(skb) / 2; 396 if (swp_spec->l4_proto == IPPROTO_UDP) 397 eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_UDP; 398 } 399 400 if (swp_spec->is_tun) { 401 eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2; 402 if (swp_spec->tun_l3_proto == htons(ETH_P_IPV6)) 403 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; 404 } else { /* typically for ipsec when xfrm mode != XFRM_MODE_TUNNEL */ 405 eseg->swp_inner_l3_offset = skb_network_offset(skb) / 2; 406 if (swp_spec->l3_proto == htons(ETH_P_IPV6)) 407 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; 408 } 409 switch (swp_spec->tun_l4_proto) { 410 case IPPROTO_UDP: 411 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP; 412 fallthrough; 413 case IPPROTO_TCP: 414 eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2; 415 break; 416 } 417 } 418 419 #define MLX5E_STOP_ROOM(wqebbs) ((wqebbs) * 2 - 1) 420 421 static inline u16 mlx5e_stop_room_for_wqe(struct mlx5_core_dev *mdev, u16 wqe_size) 422 { 423 /* A WQE must not cross the page boundary, hence two conditions: 424 * 1. Its size must not exceed the page size. 425 * 2. If the WQE size is X, and the space remaining in a page is less 426 * than X, this space needs to be padded with NOPs. So, one WQE of 427 * size X may require up to X-1 WQEBBs of padding, which makes the 428 * stop room of X-1 + X. 429 * WQE size is also limited by the hardware limit. 430 */ 431 WARN_ONCE(wqe_size > mlx5e_get_max_sq_wqebbs(mdev), 432 "wqe_size %u is greater than max SQ WQEBBs %u", 433 wqe_size, mlx5e_get_max_sq_wqebbs(mdev)); 434 435 436 return MLX5E_STOP_ROOM(wqe_size); 437 } 438 439 static inline u16 mlx5e_stop_room_for_max_wqe(struct mlx5_core_dev *mdev) 440 { 441 return MLX5E_STOP_ROOM(mlx5e_get_max_sq_wqebbs(mdev)); 442 } 443 444 static inline bool mlx5e_icosq_can_post_wqe(struct mlx5e_icosq *sq, u16 wqe_size) 445 { 446 u16 room = sq->reserved_room; 447 448 WARN_ONCE(wqe_size > sq->max_sq_wqebbs, 449 "wqe_size %u is greater than max SQ WQEBBs %u", 450 wqe_size, sq->max_sq_wqebbs); 451 452 room += MLX5E_STOP_ROOM(wqe_size); 453 454 return mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room); 455 } 456 #endif 457