1 /* 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/tcp.h> 34 #include <linux/if_vlan.h> 35 #include <linux/ptp_classify.h> 36 #include <net/geneve.h> 37 #include <net/dsfield.h> 38 #include "en.h" 39 #include "en/txrx.h" 40 #include "ipoib/ipoib.h" 41 #include "en_accel/en_accel.h" 42 #include "en/ptp.h" 43 44 static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma) 45 { 46 int i; 47 48 for (i = 0; i < num_dma; i++) { 49 struct mlx5e_sq_dma *last_pushed_dma = 50 mlx5e_dma_get(sq, --sq->dma_fifo_pc); 51 52 mlx5e_tx_dma_unmap(sq->pdev, last_pushed_dma); 53 } 54 } 55 56 #ifdef CONFIG_MLX5_CORE_EN_DCB 57 static inline int mlx5e_get_dscp_up(struct mlx5e_priv *priv, struct sk_buff *skb) 58 { 59 int dscp_cp = 0; 60 61 if (skb->protocol == htons(ETH_P_IP)) 62 dscp_cp = ipv4_get_dsfield(ip_hdr(skb)) >> 2; 63 else if (skb->protocol == htons(ETH_P_IPV6)) 64 dscp_cp = ipv6_get_dsfield(ipv6_hdr(skb)) >> 2; 65 66 return priv->dcbx_dp.dscp2prio[dscp_cp]; 67 } 68 #endif 69 70 static bool mlx5e_use_ptpsq(struct sk_buff *skb) 71 { 72 struct flow_keys fk; 73 74 if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) 75 return false; 76 77 if (fk.basic.n_proto == htons(ETH_P_1588)) 78 return true; 79 80 if (fk.basic.n_proto != htons(ETH_P_IP) && 81 fk.basic.n_proto != htons(ETH_P_IPV6)) 82 return false; 83 84 return (fk.basic.ip_proto == IPPROTO_UDP && 85 fk.ports.dst == htons(PTP_EV_PORT)); 86 } 87 88 static u16 mlx5e_select_ptpsq(struct net_device *dev, struct sk_buff *skb) 89 { 90 struct mlx5e_priv *priv = netdev_priv(dev); 91 int up = 0; 92 93 if (!netdev_get_num_tc(dev)) 94 goto return_txq; 95 96 #ifdef CONFIG_MLX5_CORE_EN_DCB 97 if (priv->dcbx_dp.trust_state == MLX5_QPTS_TRUST_DSCP) 98 up = mlx5e_get_dscp_up(priv, skb); 99 else 100 #endif 101 if (skb_vlan_tag_present(skb)) 102 up = skb_vlan_tag_get_prio(skb); 103 104 return_txq: 105 return priv->port_ptp_tc2realtxq[up]; 106 } 107 108 static int mlx5e_select_htb_queue(struct mlx5e_priv *priv, struct sk_buff *skb, 109 u16 htb_maj_id) 110 { 111 u16 classid; 112 113 if ((TC_H_MAJ(skb->priority) >> 16) == htb_maj_id) 114 classid = TC_H_MIN(skb->priority); 115 else 116 classid = READ_ONCE(priv->htb.defcls); 117 118 if (!classid) 119 return 0; 120 121 return mlx5e_get_txq_by_classid(priv, classid); 122 } 123 124 u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, 125 struct net_device *sb_dev) 126 { 127 struct mlx5e_priv *priv = netdev_priv(dev); 128 int num_tc_x_num_ch; 129 int txq_ix; 130 int up = 0; 131 int ch_ix; 132 133 /* Sync with mlx5e_update_num_tc_x_num_ch - avoid refetching. */ 134 num_tc_x_num_ch = READ_ONCE(priv->num_tc_x_num_ch); 135 if (unlikely(dev->real_num_tx_queues > num_tc_x_num_ch)) { 136 /* Order maj_id before defcls - pairs with mlx5e_htb_root_add. */ 137 u16 htb_maj_id = smp_load_acquire(&priv->htb.maj_id); 138 139 if (unlikely(htb_maj_id)) { 140 txq_ix = mlx5e_select_htb_queue(priv, skb, htb_maj_id); 141 if (txq_ix > 0) 142 return txq_ix; 143 } 144 145 if (unlikely(priv->channels.port_ptp)) 146 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 147 mlx5e_use_ptpsq(skb)) 148 return mlx5e_select_ptpsq(dev, skb); 149 150 txq_ix = netdev_pick_tx(dev, skb, NULL); 151 /* Fix netdev_pick_tx() not to choose ptp_channel and HTB txqs. 152 * If they are selected, switch to regular queues. 153 * Driver to select these queues only at mlx5e_select_ptpsq() 154 * and mlx5e_select_htb_queue(). 155 */ 156 if (unlikely(txq_ix >= num_tc_x_num_ch)) 157 txq_ix %= num_tc_x_num_ch; 158 } else { 159 txq_ix = netdev_pick_tx(dev, skb, NULL); 160 } 161 162 if (!netdev_get_num_tc(dev)) 163 return txq_ix; 164 165 #ifdef CONFIG_MLX5_CORE_EN_DCB 166 if (priv->dcbx_dp.trust_state == MLX5_QPTS_TRUST_DSCP) 167 up = mlx5e_get_dscp_up(priv, skb); 168 else 169 #endif 170 if (skb_vlan_tag_present(skb)) 171 up = skb_vlan_tag_get_prio(skb); 172 173 /* Normalize any picked txq_ix to [0, num_channels), 174 * So we can return a txq_ix that matches the channel and 175 * packet UP. 176 */ 177 ch_ix = priv->txq2sq[txq_ix]->ch_ix; 178 179 return priv->channel_tc2realtxq[ch_ix][up]; 180 } 181 182 static inline int mlx5e_skb_l2_header_offset(struct sk_buff *skb) 183 { 184 #define MLX5E_MIN_INLINE (ETH_HLEN + VLAN_HLEN) 185 186 return max(skb_network_offset(skb), MLX5E_MIN_INLINE); 187 } 188 189 static inline int mlx5e_skb_l3_header_offset(struct sk_buff *skb) 190 { 191 if (skb_transport_header_was_set(skb)) 192 return skb_transport_offset(skb); 193 else 194 return mlx5e_skb_l2_header_offset(skb); 195 } 196 197 static inline u16 mlx5e_calc_min_inline(enum mlx5_inline_modes mode, 198 struct sk_buff *skb) 199 { 200 u16 hlen; 201 202 switch (mode) { 203 case MLX5_INLINE_MODE_NONE: 204 return 0; 205 case MLX5_INLINE_MODE_TCP_UDP: 206 hlen = eth_get_headlen(skb->dev, skb->data, skb_headlen(skb)); 207 if (hlen == ETH_HLEN && !skb_vlan_tag_present(skb)) 208 hlen += VLAN_HLEN; 209 break; 210 case MLX5_INLINE_MODE_IP: 211 hlen = mlx5e_skb_l3_header_offset(skb); 212 break; 213 case MLX5_INLINE_MODE_L2: 214 default: 215 hlen = mlx5e_skb_l2_header_offset(skb); 216 } 217 return min_t(u16, hlen, skb_headlen(skb)); 218 } 219 220 static inline void mlx5e_insert_vlan(void *start, struct sk_buff *skb, u16 ihs) 221 { 222 struct vlan_ethhdr *vhdr = (struct vlan_ethhdr *)start; 223 int cpy1_sz = 2 * ETH_ALEN; 224 int cpy2_sz = ihs - cpy1_sz; 225 226 memcpy(vhdr, skb->data, cpy1_sz); 227 vhdr->h_vlan_proto = skb->vlan_proto; 228 vhdr->h_vlan_TCI = cpu_to_be16(skb_vlan_tag_get(skb)); 229 memcpy(&vhdr->h_vlan_encapsulated_proto, skb->data + cpy1_sz, cpy2_sz); 230 } 231 232 /* If packet is not IP's CHECKSUM_PARTIAL (e.g. icmd packet), 233 * need to set L3 checksum flag for IPsec 234 */ 235 static void 236 ipsec_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb, 237 struct mlx5_wqe_eth_seg *eseg) 238 { 239 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM; 240 if (skb->encapsulation) { 241 eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM; 242 sq->stats->csum_partial_inner++; 243 } else { 244 sq->stats->csum_partial++; 245 } 246 } 247 248 static inline void 249 mlx5e_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb, 250 struct mlx5e_accel_tx_state *accel, 251 struct mlx5_wqe_eth_seg *eseg) 252 { 253 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 254 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM; 255 if (skb->encapsulation) { 256 eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM | 257 MLX5_ETH_WQE_L4_INNER_CSUM; 258 sq->stats->csum_partial_inner++; 259 } else { 260 eseg->cs_flags |= MLX5_ETH_WQE_L4_CSUM; 261 sq->stats->csum_partial++; 262 } 263 #ifdef CONFIG_MLX5_EN_TLS 264 } else if (unlikely(accel && accel->tls.tls_tisn)) { 265 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; 266 sq->stats->csum_partial++; 267 #endif 268 } else if (unlikely(mlx5e_ipsec_eseg_meta(eseg))) { 269 ipsec_txwqe_build_eseg_csum(sq, skb, eseg); 270 } else 271 sq->stats->csum_none++; 272 } 273 274 static inline u16 275 mlx5e_tx_get_gso_ihs(struct mlx5e_txqsq *sq, struct sk_buff *skb) 276 { 277 struct mlx5e_sq_stats *stats = sq->stats; 278 u16 ihs; 279 280 if (skb->encapsulation) { 281 ihs = skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb); 282 stats->tso_inner_packets++; 283 stats->tso_inner_bytes += skb->len - ihs; 284 } else { 285 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) 286 ihs = skb_transport_offset(skb) + sizeof(struct udphdr); 287 else 288 ihs = skb_transport_offset(skb) + tcp_hdrlen(skb); 289 stats->tso_packets++; 290 stats->tso_bytes += skb->len - ihs; 291 } 292 293 return ihs; 294 } 295 296 static inline int 297 mlx5e_txwqe_build_dsegs(struct mlx5e_txqsq *sq, struct sk_buff *skb, 298 unsigned char *skb_data, u16 headlen, 299 struct mlx5_wqe_data_seg *dseg) 300 { 301 dma_addr_t dma_addr = 0; 302 u8 num_dma = 0; 303 int i; 304 305 if (headlen) { 306 dma_addr = dma_map_single(sq->pdev, skb_data, headlen, 307 DMA_TO_DEVICE); 308 if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) 309 goto dma_unmap_wqe_err; 310 311 dseg->addr = cpu_to_be64(dma_addr); 312 dseg->lkey = sq->mkey_be; 313 dseg->byte_count = cpu_to_be32(headlen); 314 315 mlx5e_dma_push(sq, dma_addr, headlen, MLX5E_DMA_MAP_SINGLE); 316 num_dma++; 317 dseg++; 318 } 319 320 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 321 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 322 int fsz = skb_frag_size(frag); 323 324 dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz, 325 DMA_TO_DEVICE); 326 if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) 327 goto dma_unmap_wqe_err; 328 329 dseg->addr = cpu_to_be64(dma_addr); 330 dseg->lkey = sq->mkey_be; 331 dseg->byte_count = cpu_to_be32(fsz); 332 333 mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE); 334 num_dma++; 335 dseg++; 336 } 337 338 return num_dma; 339 340 dma_unmap_wqe_err: 341 mlx5e_dma_unmap_wqe_err(sq, num_dma); 342 return -ENOMEM; 343 } 344 345 struct mlx5e_tx_attr { 346 u32 num_bytes; 347 u16 headlen; 348 u16 ihs; 349 __be16 mss; 350 u16 insz; 351 u8 opcode; 352 }; 353 354 struct mlx5e_tx_wqe_attr { 355 u16 ds_cnt; 356 u16 ds_cnt_inl; 357 u16 ds_cnt_ids; 358 u8 num_wqebbs; 359 }; 360 361 static u8 362 mlx5e_tx_wqe_inline_mode(struct mlx5e_txqsq *sq, struct sk_buff *skb, 363 struct mlx5e_accel_tx_state *accel) 364 { 365 u8 mode; 366 367 #ifdef CONFIG_MLX5_EN_TLS 368 if (accel && accel->tls.tls_tisn) 369 return MLX5_INLINE_MODE_TCP_UDP; 370 #endif 371 372 mode = sq->min_inline_mode; 373 374 if (skb_vlan_tag_present(skb) && 375 test_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state)) 376 mode = max_t(u8, MLX5_INLINE_MODE_L2, mode); 377 378 return mode; 379 } 380 381 static void mlx5e_sq_xmit_prepare(struct mlx5e_txqsq *sq, struct sk_buff *skb, 382 struct mlx5e_accel_tx_state *accel, 383 struct mlx5e_tx_attr *attr) 384 { 385 struct mlx5e_sq_stats *stats = sq->stats; 386 387 if (skb_is_gso(skb)) { 388 u16 ihs = mlx5e_tx_get_gso_ihs(sq, skb); 389 390 *attr = (struct mlx5e_tx_attr) { 391 .opcode = MLX5_OPCODE_LSO, 392 .mss = cpu_to_be16(skb_shinfo(skb)->gso_size), 393 .ihs = ihs, 394 .num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs, 395 .headlen = skb_headlen(skb) - ihs, 396 }; 397 398 stats->packets += skb_shinfo(skb)->gso_segs; 399 } else { 400 u8 mode = mlx5e_tx_wqe_inline_mode(sq, skb, accel); 401 u16 ihs = mlx5e_calc_min_inline(mode, skb); 402 403 *attr = (struct mlx5e_tx_attr) { 404 .opcode = MLX5_OPCODE_SEND, 405 .mss = cpu_to_be16(0), 406 .ihs = ihs, 407 .num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN), 408 .headlen = skb_headlen(skb) - ihs, 409 }; 410 411 stats->packets++; 412 } 413 414 attr->insz = mlx5e_accel_tx_ids_len(sq, accel); 415 stats->bytes += attr->num_bytes; 416 } 417 418 static void mlx5e_sq_calc_wqe_attr(struct sk_buff *skb, const struct mlx5e_tx_attr *attr, 419 struct mlx5e_tx_wqe_attr *wqe_attr) 420 { 421 u16 ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT; 422 u16 ds_cnt_inl = 0; 423 u16 ds_cnt_ids = 0; 424 425 if (attr->insz) 426 ds_cnt_ids = DIV_ROUND_UP(sizeof(struct mlx5_wqe_inline_seg) + attr->insz, 427 MLX5_SEND_WQE_DS); 428 429 ds_cnt += !!attr->headlen + skb_shinfo(skb)->nr_frags + ds_cnt_ids; 430 if (attr->ihs) { 431 u16 inl = attr->ihs - INL_HDR_START_SZ; 432 433 if (skb_vlan_tag_present(skb)) 434 inl += VLAN_HLEN; 435 436 ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS); 437 ds_cnt += ds_cnt_inl; 438 } 439 440 *wqe_attr = (struct mlx5e_tx_wqe_attr) { 441 .ds_cnt = ds_cnt, 442 .ds_cnt_inl = ds_cnt_inl, 443 .ds_cnt_ids = ds_cnt_ids, 444 .num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS), 445 }; 446 } 447 448 static void mlx5e_tx_skb_update_hwts_flags(struct sk_buff *skb) 449 { 450 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 451 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 452 } 453 454 static void mlx5e_tx_check_stop(struct mlx5e_txqsq *sq) 455 { 456 if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, sq->stop_room))) { 457 netif_tx_stop_queue(sq->txq); 458 sq->stats->stopped++; 459 } 460 } 461 462 static inline void 463 mlx5e_txwqe_complete(struct mlx5e_txqsq *sq, struct sk_buff *skb, 464 const struct mlx5e_tx_attr *attr, 465 const struct mlx5e_tx_wqe_attr *wqe_attr, u8 num_dma, 466 struct mlx5e_tx_wqe_info *wi, struct mlx5_wqe_ctrl_seg *cseg, 467 bool xmit_more) 468 { 469 struct mlx5_wq_cyc *wq = &sq->wq; 470 bool send_doorbell; 471 472 *wi = (struct mlx5e_tx_wqe_info) { 473 .skb = skb, 474 .num_bytes = attr->num_bytes, 475 .num_dma = num_dma, 476 .num_wqebbs = wqe_attr->num_wqebbs, 477 .num_fifo_pkts = 0, 478 }; 479 480 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | attr->opcode); 481 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | wqe_attr->ds_cnt); 482 483 mlx5e_tx_skb_update_hwts_flags(skb); 484 485 sq->pc += wi->num_wqebbs; 486 487 mlx5e_tx_check_stop(sq); 488 489 if (unlikely(sq->ptpsq)) { 490 mlx5e_skb_cb_hwtstamp_init(skb); 491 mlx5e_skb_fifo_push(&sq->ptpsq->skb_fifo, skb); 492 skb_get(skb); 493 } 494 495 send_doorbell = __netdev_tx_sent_queue(sq->txq, attr->num_bytes, xmit_more); 496 if (send_doorbell) 497 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, cseg); 498 } 499 500 static void 501 mlx5e_sq_xmit_wqe(struct mlx5e_txqsq *sq, struct sk_buff *skb, 502 const struct mlx5e_tx_attr *attr, const struct mlx5e_tx_wqe_attr *wqe_attr, 503 struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more) 504 { 505 struct mlx5_wqe_ctrl_seg *cseg; 506 struct mlx5_wqe_eth_seg *eseg; 507 struct mlx5_wqe_data_seg *dseg; 508 struct mlx5e_tx_wqe_info *wi; 509 510 struct mlx5e_sq_stats *stats = sq->stats; 511 int num_dma; 512 513 stats->xmit_more += xmit_more; 514 515 /* fill wqe */ 516 wi = &sq->db.wqe_info[pi]; 517 cseg = &wqe->ctrl; 518 eseg = &wqe->eth; 519 dseg = wqe->data; 520 521 eseg->mss = attr->mss; 522 523 if (attr->ihs) { 524 if (skb_vlan_tag_present(skb)) { 525 eseg->inline_hdr.sz |= cpu_to_be16(attr->ihs + VLAN_HLEN); 526 mlx5e_insert_vlan(eseg->inline_hdr.start, skb, attr->ihs); 527 stats->added_vlan_packets++; 528 } else { 529 eseg->inline_hdr.sz |= cpu_to_be16(attr->ihs); 530 memcpy(eseg->inline_hdr.start, skb->data, attr->ihs); 531 } 532 dseg += wqe_attr->ds_cnt_inl; 533 } else if (skb_vlan_tag_present(skb)) { 534 eseg->insert.type = cpu_to_be16(MLX5_ETH_WQE_INSERT_VLAN); 535 if (skb->vlan_proto == cpu_to_be16(ETH_P_8021AD)) 536 eseg->insert.type |= cpu_to_be16(MLX5_ETH_WQE_SVLAN); 537 eseg->insert.vlan_tci = cpu_to_be16(skb_vlan_tag_get(skb)); 538 stats->added_vlan_packets++; 539 } 540 541 dseg += wqe_attr->ds_cnt_ids; 542 num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb->data + attr->ihs, 543 attr->headlen, dseg); 544 if (unlikely(num_dma < 0)) 545 goto err_drop; 546 547 mlx5e_txwqe_complete(sq, skb, attr, wqe_attr, num_dma, wi, cseg, xmit_more); 548 549 return; 550 551 err_drop: 552 stats->dropped++; 553 dev_kfree_skb_any(skb); 554 } 555 556 static bool mlx5e_tx_skb_supports_mpwqe(struct sk_buff *skb, struct mlx5e_tx_attr *attr) 557 { 558 return !skb_is_nonlinear(skb) && !skb_vlan_tag_present(skb) && !attr->ihs && 559 !attr->insz; 560 } 561 562 static bool mlx5e_tx_mpwqe_same_eseg(struct mlx5e_txqsq *sq, struct mlx5_wqe_eth_seg *eseg) 563 { 564 struct mlx5e_tx_mpwqe *session = &sq->mpwqe; 565 566 /* Assumes the session is already running and has at least one packet. */ 567 return !memcmp(&session->wqe->eth, eseg, MLX5E_ACCEL_ESEG_LEN); 568 } 569 570 static void mlx5e_tx_mpwqe_session_start(struct mlx5e_txqsq *sq, 571 struct mlx5_wqe_eth_seg *eseg) 572 { 573 struct mlx5e_tx_mpwqe *session = &sq->mpwqe; 574 struct mlx5e_tx_wqe *wqe; 575 u16 pi; 576 577 pi = mlx5e_txqsq_get_next_pi(sq, MLX5E_TX_MPW_MAX_WQEBBS); 578 wqe = MLX5E_TX_FETCH_WQE(sq, pi); 579 prefetchw(wqe->data); 580 581 *session = (struct mlx5e_tx_mpwqe) { 582 .wqe = wqe, 583 .bytes_count = 0, 584 .ds_count = MLX5E_TX_WQE_EMPTY_DS_COUNT, 585 .pkt_count = 0, 586 .inline_on = 0, 587 }; 588 589 memcpy(&session->wqe->eth, eseg, MLX5E_ACCEL_ESEG_LEN); 590 591 sq->stats->mpwqe_blks++; 592 } 593 594 static bool mlx5e_tx_mpwqe_session_is_active(struct mlx5e_txqsq *sq) 595 { 596 return sq->mpwqe.wqe; 597 } 598 599 static void mlx5e_tx_mpwqe_add_dseg(struct mlx5e_txqsq *sq, struct mlx5e_xmit_data *txd) 600 { 601 struct mlx5e_tx_mpwqe *session = &sq->mpwqe; 602 struct mlx5_wqe_data_seg *dseg; 603 604 dseg = (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count; 605 606 session->pkt_count++; 607 session->bytes_count += txd->len; 608 609 dseg->addr = cpu_to_be64(txd->dma_addr); 610 dseg->byte_count = cpu_to_be32(txd->len); 611 dseg->lkey = sq->mkey_be; 612 session->ds_count++; 613 614 sq->stats->mpwqe_pkts++; 615 } 616 617 static struct mlx5_wqe_ctrl_seg *mlx5e_tx_mpwqe_session_complete(struct mlx5e_txqsq *sq) 618 { 619 struct mlx5e_tx_mpwqe *session = &sq->mpwqe; 620 u8 ds_count = session->ds_count; 621 struct mlx5_wqe_ctrl_seg *cseg; 622 struct mlx5e_tx_wqe_info *wi; 623 u16 pi; 624 625 cseg = &session->wqe->ctrl; 626 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_ENHANCED_MPSW); 627 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_count); 628 629 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc); 630 wi = &sq->db.wqe_info[pi]; 631 *wi = (struct mlx5e_tx_wqe_info) { 632 .skb = NULL, 633 .num_bytes = session->bytes_count, 634 .num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS), 635 .num_dma = session->pkt_count, 636 .num_fifo_pkts = session->pkt_count, 637 }; 638 639 sq->pc += wi->num_wqebbs; 640 641 session->wqe = NULL; 642 643 mlx5e_tx_check_stop(sq); 644 645 return cseg; 646 } 647 648 static void 649 mlx5e_sq_xmit_mpwqe(struct mlx5e_txqsq *sq, struct sk_buff *skb, 650 struct mlx5_wqe_eth_seg *eseg, bool xmit_more) 651 { 652 struct mlx5_wqe_ctrl_seg *cseg; 653 struct mlx5e_xmit_data txd; 654 655 if (!mlx5e_tx_mpwqe_session_is_active(sq)) { 656 mlx5e_tx_mpwqe_session_start(sq, eseg); 657 } else if (!mlx5e_tx_mpwqe_same_eseg(sq, eseg)) { 658 mlx5e_tx_mpwqe_session_complete(sq); 659 mlx5e_tx_mpwqe_session_start(sq, eseg); 660 } 661 662 sq->stats->xmit_more += xmit_more; 663 664 txd.data = skb->data; 665 txd.len = skb->len; 666 667 txd.dma_addr = dma_map_single(sq->pdev, txd.data, txd.len, DMA_TO_DEVICE); 668 if (unlikely(dma_mapping_error(sq->pdev, txd.dma_addr))) 669 goto err_unmap; 670 mlx5e_dma_push(sq, txd.dma_addr, txd.len, MLX5E_DMA_MAP_SINGLE); 671 672 mlx5e_skb_fifo_push(&sq->db.skb_fifo, skb); 673 674 mlx5e_tx_mpwqe_add_dseg(sq, &txd); 675 676 mlx5e_tx_skb_update_hwts_flags(skb); 677 678 if (unlikely(mlx5e_tx_mpwqe_is_full(&sq->mpwqe))) { 679 /* Might stop the queue and affect the retval of __netdev_tx_sent_queue. */ 680 cseg = mlx5e_tx_mpwqe_session_complete(sq); 681 682 if (__netdev_tx_sent_queue(sq->txq, txd.len, xmit_more)) 683 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); 684 } else if (__netdev_tx_sent_queue(sq->txq, txd.len, xmit_more)) { 685 /* Might stop the queue, but we were asked to ring the doorbell anyway. */ 686 cseg = mlx5e_tx_mpwqe_session_complete(sq); 687 688 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); 689 } 690 691 return; 692 693 err_unmap: 694 mlx5e_dma_unmap_wqe_err(sq, 1); 695 sq->stats->dropped++; 696 dev_kfree_skb_any(skb); 697 } 698 699 void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq) 700 { 701 /* Unlikely in non-MPWQE workloads; not important in MPWQE workloads. */ 702 if (unlikely(mlx5e_tx_mpwqe_session_is_active(sq))) 703 mlx5e_tx_mpwqe_session_complete(sq); 704 } 705 706 static bool mlx5e_txwqe_build_eseg(struct mlx5e_priv *priv, struct mlx5e_txqsq *sq, 707 struct sk_buff *skb, struct mlx5e_accel_tx_state *accel, 708 struct mlx5_wqe_eth_seg *eseg, u16 ihs) 709 { 710 if (unlikely(!mlx5e_accel_tx_eseg(priv, skb, eseg, ihs))) 711 return false; 712 713 mlx5e_txwqe_build_eseg_csum(sq, skb, accel, eseg); 714 715 return true; 716 } 717 718 netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) 719 { 720 struct mlx5e_priv *priv = netdev_priv(dev); 721 struct mlx5e_accel_tx_state accel = {}; 722 struct mlx5e_tx_wqe_attr wqe_attr; 723 struct mlx5e_tx_attr attr; 724 struct mlx5e_tx_wqe *wqe; 725 struct mlx5e_txqsq *sq; 726 u16 pi; 727 728 sq = priv->txq2sq[skb_get_queue_mapping(skb)]; 729 if (unlikely(!sq)) { 730 dev_kfree_skb_any(skb); 731 return NETDEV_TX_OK; 732 } 733 734 /* May send SKBs and WQEs. */ 735 if (unlikely(!mlx5e_accel_tx_begin(dev, sq, skb, &accel))) 736 return NETDEV_TX_OK; 737 738 mlx5e_sq_xmit_prepare(sq, skb, &accel, &attr); 739 740 if (test_bit(MLX5E_SQ_STATE_MPWQE, &sq->state)) { 741 if (mlx5e_tx_skb_supports_mpwqe(skb, &attr)) { 742 struct mlx5_wqe_eth_seg eseg = {}; 743 744 if (unlikely(!mlx5e_txwqe_build_eseg(priv, sq, skb, &accel, &eseg, 745 attr.ihs))) 746 return NETDEV_TX_OK; 747 748 mlx5e_sq_xmit_mpwqe(sq, skb, &eseg, netdev_xmit_more()); 749 return NETDEV_TX_OK; 750 } 751 752 mlx5e_tx_mpwqe_ensure_complete(sq); 753 } 754 755 mlx5e_sq_calc_wqe_attr(skb, &attr, &wqe_attr); 756 pi = mlx5e_txqsq_get_next_pi(sq, wqe_attr.num_wqebbs); 757 wqe = MLX5E_TX_FETCH_WQE(sq, pi); 758 759 /* May update the WQE, but may not post other WQEs. */ 760 mlx5e_accel_tx_finish(sq, wqe, &accel, 761 (struct mlx5_wqe_inline_seg *)(wqe->data + wqe_attr.ds_cnt_inl)); 762 if (unlikely(!mlx5e_txwqe_build_eseg(priv, sq, skb, &accel, &wqe->eth, attr.ihs))) 763 return NETDEV_TX_OK; 764 765 mlx5e_sq_xmit_wqe(sq, skb, &attr, &wqe_attr, wqe, pi, netdev_xmit_more()); 766 767 return NETDEV_TX_OK; 768 } 769 770 void mlx5e_sq_xmit_simple(struct mlx5e_txqsq *sq, struct sk_buff *skb, bool xmit_more) 771 { 772 struct mlx5e_tx_wqe_attr wqe_attr; 773 struct mlx5e_tx_attr attr; 774 struct mlx5e_tx_wqe *wqe; 775 u16 pi; 776 777 mlx5e_sq_xmit_prepare(sq, skb, NULL, &attr); 778 mlx5e_sq_calc_wqe_attr(skb, &attr, &wqe_attr); 779 pi = mlx5e_txqsq_get_next_pi(sq, wqe_attr.num_wqebbs); 780 wqe = MLX5E_TX_FETCH_WQE(sq, pi); 781 mlx5e_txwqe_build_eseg_csum(sq, skb, NULL, &wqe->eth); 782 mlx5e_sq_xmit_wqe(sq, skb, &attr, &wqe_attr, wqe, pi, xmit_more); 783 } 784 785 static void mlx5e_tx_wi_dma_unmap(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, 786 u32 *dma_fifo_cc) 787 { 788 int i; 789 790 for (i = 0; i < wi->num_dma; i++) { 791 struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, (*dma_fifo_cc)++); 792 793 mlx5e_tx_dma_unmap(sq->pdev, dma); 794 } 795 } 796 797 static void mlx5e_consume_skb(struct mlx5e_txqsq *sq, struct sk_buff *skb, 798 struct mlx5_cqe64 *cqe, int napi_budget) 799 { 800 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 801 struct skb_shared_hwtstamps hwts = {}; 802 u64 ts = get_cqe_ts(cqe); 803 804 hwts.hwtstamp = mlx5e_cqe_ts_to_ns(sq->ptp_cyc2time, sq->clock, ts); 805 if (sq->ptpsq) 806 mlx5e_skb_cb_hwtstamp_handler(skb, MLX5E_SKB_CB_CQE_HWTSTAMP, 807 hwts.hwtstamp, sq->ptpsq->cq_stats); 808 else 809 skb_tstamp_tx(skb, &hwts); 810 } 811 812 napi_consume_skb(skb, napi_budget); 813 } 814 815 static void mlx5e_tx_wi_consume_fifo_skbs(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, 816 struct mlx5_cqe64 *cqe, int napi_budget) 817 { 818 int i; 819 820 for (i = 0; i < wi->num_fifo_pkts; i++) { 821 struct sk_buff *skb = mlx5e_skb_fifo_pop(&sq->db.skb_fifo); 822 823 mlx5e_consume_skb(sq, skb, cqe, napi_budget); 824 } 825 } 826 827 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget) 828 { 829 struct mlx5e_sq_stats *stats; 830 struct mlx5e_txqsq *sq; 831 struct mlx5_cqe64 *cqe; 832 u32 dma_fifo_cc; 833 u32 nbytes; 834 u16 npkts; 835 u16 sqcc; 836 int i; 837 838 sq = container_of(cq, struct mlx5e_txqsq, cq); 839 840 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) 841 return false; 842 843 cqe = mlx5_cqwq_get_cqe(&cq->wq); 844 if (!cqe) 845 return false; 846 847 stats = sq->stats; 848 849 npkts = 0; 850 nbytes = 0; 851 852 /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), 853 * otherwise a cq overrun may occur 854 */ 855 sqcc = sq->cc; 856 857 /* avoid dirtying sq cache line every cqe */ 858 dma_fifo_cc = sq->dma_fifo_cc; 859 860 i = 0; 861 do { 862 struct mlx5e_tx_wqe_info *wi; 863 u16 wqe_counter; 864 bool last_wqe; 865 u16 ci; 866 867 mlx5_cqwq_pop(&cq->wq); 868 869 wqe_counter = be16_to_cpu(cqe->wqe_counter); 870 871 do { 872 last_wqe = (sqcc == wqe_counter); 873 874 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); 875 wi = &sq->db.wqe_info[ci]; 876 877 sqcc += wi->num_wqebbs; 878 879 if (likely(wi->skb)) { 880 mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc); 881 mlx5e_consume_skb(sq, wi->skb, cqe, napi_budget); 882 883 npkts++; 884 nbytes += wi->num_bytes; 885 continue; 886 } 887 888 if (unlikely(mlx5e_ktls_tx_try_handle_resync_dump_comp(sq, wi, 889 &dma_fifo_cc))) 890 continue; 891 892 if (wi->num_fifo_pkts) { 893 mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc); 894 mlx5e_tx_wi_consume_fifo_skbs(sq, wi, cqe, napi_budget); 895 896 npkts += wi->num_fifo_pkts; 897 nbytes += wi->num_bytes; 898 } 899 } while (!last_wqe); 900 901 if (unlikely(get_cqe_opcode(cqe) == MLX5_CQE_REQ_ERR)) { 902 if (!test_and_set_bit(MLX5E_SQ_STATE_RECOVERING, 903 &sq->state)) { 904 mlx5e_dump_error_cqe(&sq->cq, sq->sqn, 905 (struct mlx5_err_cqe *)cqe); 906 mlx5_wq_cyc_wqe_dump(&sq->wq, ci, wi->num_wqebbs); 907 queue_work(cq->priv->wq, &sq->recover_work); 908 } 909 stats->cqe_err++; 910 } 911 912 } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq))); 913 914 stats->cqes += i; 915 916 mlx5_cqwq_update_db_record(&cq->wq); 917 918 /* ensure cq space is freed before enabling more cqes */ 919 wmb(); 920 921 sq->dma_fifo_cc = dma_fifo_cc; 922 sq->cc = sqcc; 923 924 netdev_tx_completed_queue(sq->txq, npkts, nbytes); 925 926 if (netif_tx_queue_stopped(sq->txq) && 927 mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, sq->stop_room) && 928 !test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) { 929 netif_tx_wake_queue(sq->txq); 930 stats->wake++; 931 } 932 933 return (i == MLX5E_TX_CQ_POLL_BUDGET); 934 } 935 936 static void mlx5e_tx_wi_kfree_fifo_skbs(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi) 937 { 938 int i; 939 940 for (i = 0; i < wi->num_fifo_pkts; i++) 941 dev_kfree_skb_any(mlx5e_skb_fifo_pop(&sq->db.skb_fifo)); 942 } 943 944 void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq) 945 { 946 struct mlx5e_tx_wqe_info *wi; 947 u32 dma_fifo_cc, nbytes = 0; 948 u16 ci, sqcc, npkts = 0; 949 950 sqcc = sq->cc; 951 dma_fifo_cc = sq->dma_fifo_cc; 952 953 while (sqcc != sq->pc) { 954 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); 955 wi = &sq->db.wqe_info[ci]; 956 957 sqcc += wi->num_wqebbs; 958 959 if (likely(wi->skb)) { 960 mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc); 961 dev_kfree_skb_any(wi->skb); 962 963 npkts++; 964 nbytes += wi->num_bytes; 965 continue; 966 } 967 968 if (unlikely(mlx5e_ktls_tx_try_handle_resync_dump_comp(sq, wi, &dma_fifo_cc))) 969 continue; 970 971 if (wi->num_fifo_pkts) { 972 mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc); 973 mlx5e_tx_wi_kfree_fifo_skbs(sq, wi); 974 975 npkts += wi->num_fifo_pkts; 976 nbytes += wi->num_bytes; 977 } 978 } 979 980 sq->dma_fifo_cc = dma_fifo_cc; 981 sq->cc = sqcc; 982 983 netdev_tx_completed_queue(sq->txq, npkts, nbytes); 984 } 985 986 #ifdef CONFIG_MLX5_CORE_IPOIB 987 static inline void 988 mlx5i_txwqe_build_datagram(struct mlx5_av *av, u32 dqpn, u32 dqkey, 989 struct mlx5_wqe_datagram_seg *dseg) 990 { 991 memcpy(&dseg->av, av, sizeof(struct mlx5_av)); 992 dseg->av.dqp_dct = cpu_to_be32(dqpn | MLX5_EXTENDED_UD_AV); 993 dseg->av.key.qkey.qkey = cpu_to_be32(dqkey); 994 } 995 996 static void mlx5i_sq_calc_wqe_attr(struct sk_buff *skb, 997 const struct mlx5e_tx_attr *attr, 998 struct mlx5e_tx_wqe_attr *wqe_attr) 999 { 1000 u16 ds_cnt = sizeof(struct mlx5i_tx_wqe) / MLX5_SEND_WQE_DS; 1001 u16 ds_cnt_inl = 0; 1002 1003 ds_cnt += !!attr->headlen + skb_shinfo(skb)->nr_frags; 1004 1005 if (attr->ihs) { 1006 u16 inl = attr->ihs - INL_HDR_START_SZ; 1007 1008 ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS); 1009 ds_cnt += ds_cnt_inl; 1010 } 1011 1012 *wqe_attr = (struct mlx5e_tx_wqe_attr) { 1013 .ds_cnt = ds_cnt, 1014 .ds_cnt_inl = ds_cnt_inl, 1015 .num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS), 1016 }; 1017 } 1018 1019 void mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, 1020 struct mlx5_av *av, u32 dqpn, u32 dqkey, bool xmit_more) 1021 { 1022 struct mlx5e_tx_wqe_attr wqe_attr; 1023 struct mlx5e_tx_attr attr; 1024 struct mlx5i_tx_wqe *wqe; 1025 1026 struct mlx5_wqe_datagram_seg *datagram; 1027 struct mlx5_wqe_ctrl_seg *cseg; 1028 struct mlx5_wqe_eth_seg *eseg; 1029 struct mlx5_wqe_data_seg *dseg; 1030 struct mlx5e_tx_wqe_info *wi; 1031 1032 struct mlx5e_sq_stats *stats = sq->stats; 1033 int num_dma; 1034 u16 pi; 1035 1036 mlx5e_sq_xmit_prepare(sq, skb, NULL, &attr); 1037 mlx5i_sq_calc_wqe_attr(skb, &attr, &wqe_attr); 1038 1039 pi = mlx5e_txqsq_get_next_pi(sq, wqe_attr.num_wqebbs); 1040 wqe = MLX5I_SQ_FETCH_WQE(sq, pi); 1041 1042 stats->xmit_more += xmit_more; 1043 1044 /* fill wqe */ 1045 wi = &sq->db.wqe_info[pi]; 1046 cseg = &wqe->ctrl; 1047 datagram = &wqe->datagram; 1048 eseg = &wqe->eth; 1049 dseg = wqe->data; 1050 1051 mlx5i_txwqe_build_datagram(av, dqpn, dqkey, datagram); 1052 1053 mlx5e_txwqe_build_eseg_csum(sq, skb, NULL, eseg); 1054 1055 eseg->mss = attr.mss; 1056 1057 if (attr.ihs) { 1058 memcpy(eseg->inline_hdr.start, skb->data, attr.ihs); 1059 eseg->inline_hdr.sz = cpu_to_be16(attr.ihs); 1060 dseg += wqe_attr.ds_cnt_inl; 1061 } 1062 1063 num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb->data + attr.ihs, 1064 attr.headlen, dseg); 1065 if (unlikely(num_dma < 0)) 1066 goto err_drop; 1067 1068 mlx5e_txwqe_complete(sq, skb, &attr, &wqe_attr, num_dma, wi, cseg, xmit_more); 1069 1070 return; 1071 1072 err_drop: 1073 stats->dropped++; 1074 dev_kfree_skb_any(skb); 1075 } 1076 #endif 1077