1 /* 2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34 #include <asm/page.h> 35 #include <linux/mlx4/cq.h> 36 #include <linux/slab.h> 37 #include <linux/mlx4/qp.h> 38 #include <linux/skbuff.h> 39 #include <linux/if_vlan.h> 40 #include <linux/vmalloc.h> 41 #include <linux/tcp.h> 42 #include <linux/moduleparam.h> 43 44 #include "mlx4_en.h" 45 46 enum { 47 MAX_INLINE = 104, /* 128 - 16 - 4 - 4 */ 48 MAX_BF = 256, 49 }; 50 51 static int inline_thold __read_mostly = MAX_INLINE; 52 53 module_param_named(inline_thold, inline_thold, int, 0444); 54 MODULE_PARM_DESC(inline_thold, "threshold for using inline data"); 55 56 int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, 57 struct mlx4_en_tx_ring *ring, int qpn, u32 size, 58 u16 stride) 59 { 60 struct mlx4_en_dev *mdev = priv->mdev; 61 int tmp; 62 int err; 63 64 ring->size = size; 65 ring->size_mask = size - 1; 66 ring->stride = stride; 67 68 inline_thold = min(inline_thold, MAX_INLINE); 69 70 tmp = size * sizeof(struct mlx4_en_tx_info); 71 ring->tx_info = vmalloc(tmp); 72 if (!ring->tx_info) 73 return -ENOMEM; 74 75 en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n", 76 ring->tx_info, tmp); 77 78 ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL); 79 if (!ring->bounce_buf) { 80 err = -ENOMEM; 81 goto err_tx; 82 } 83 ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE); 84 85 err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size, 86 2 * PAGE_SIZE); 87 if (err) { 88 en_err(priv, "Failed allocating hwq resources\n"); 89 goto err_bounce; 90 } 91 92 err = mlx4_en_map_buffer(&ring->wqres.buf); 93 if (err) { 94 en_err(priv, "Failed to map TX buffer\n"); 95 goto err_hwq_res; 96 } 97 98 ring->buf = ring->wqres.buf.direct.buf; 99 100 en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d " 101 "buf_size:%d dma:%llx\n", ring, ring->buf, ring->size, 102 ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map); 103 104 ring->qpn = qpn; 105 err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp); 106 if (err) { 107 en_err(priv, "Failed allocating qp %d\n", ring->qpn); 108 goto err_map; 109 } 110 ring->qp.event = mlx4_en_sqp_event; 111 112 err = mlx4_bf_alloc(mdev->dev, &ring->bf); 113 if (err) { 114 en_dbg(DRV, priv, "working without blueflame (%d)", err); 115 ring->bf.uar = &mdev->priv_uar; 116 ring->bf.uar->map = mdev->uar_map; 117 ring->bf_enabled = false; 118 } else 119 ring->bf_enabled = true; 120 121 ring->hwtstamp_tx_type = priv->hwtstamp_config.tx_type; 122 123 return 0; 124 125 err_map: 126 mlx4_en_unmap_buffer(&ring->wqres.buf); 127 err_hwq_res: 128 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); 129 err_bounce: 130 kfree(ring->bounce_buf); 131 ring->bounce_buf = NULL; 132 err_tx: 133 vfree(ring->tx_info); 134 ring->tx_info = NULL; 135 return err; 136 } 137 138 void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, 139 struct mlx4_en_tx_ring *ring) 140 { 141 struct mlx4_en_dev *mdev = priv->mdev; 142 en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn); 143 144 if (ring->bf_enabled) 145 mlx4_bf_free(mdev->dev, &ring->bf); 146 mlx4_qp_remove(mdev->dev, &ring->qp); 147 mlx4_qp_free(mdev->dev, &ring->qp); 148 mlx4_en_unmap_buffer(&ring->wqres.buf); 149 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); 150 kfree(ring->bounce_buf); 151 ring->bounce_buf = NULL; 152 vfree(ring->tx_info); 153 ring->tx_info = NULL; 154 } 155 156 int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, 157 struct mlx4_en_tx_ring *ring, 158 int cq, int user_prio) 159 { 160 struct mlx4_en_dev *mdev = priv->mdev; 161 int err; 162 163 ring->cqn = cq; 164 ring->prod = 0; 165 ring->cons = 0xffffffff; 166 ring->last_nr_txbb = 1; 167 ring->poll_cnt = 0; 168 memset(ring->tx_info, 0, ring->size * sizeof(struct mlx4_en_tx_info)); 169 memset(ring->buf, 0, ring->buf_size); 170 171 ring->qp_state = MLX4_QP_STATE_RST; 172 ring->doorbell_qpn = ring->qp.qpn << 8; 173 174 mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 1, 0, ring->qpn, 175 ring->cqn, user_prio, &ring->context); 176 if (ring->bf_enabled) 177 ring->context.usr_page = cpu_to_be32(ring->bf.uar->index); 178 179 err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, &ring->context, 180 &ring->qp, &ring->qp_state); 181 182 return err; 183 } 184 185 void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, 186 struct mlx4_en_tx_ring *ring) 187 { 188 struct mlx4_en_dev *mdev = priv->mdev; 189 190 mlx4_qp_modify(mdev->dev, NULL, ring->qp_state, 191 MLX4_QP_STATE_RST, NULL, 0, 0, &ring->qp); 192 } 193 194 195 static u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, 196 struct mlx4_en_tx_ring *ring, 197 int index, u8 owner, u64 timestamp) 198 { 199 struct mlx4_en_dev *mdev = priv->mdev; 200 struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; 201 struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE; 202 struct mlx4_wqe_data_seg *data = (void *) tx_desc + tx_info->data_offset; 203 struct sk_buff *skb = tx_info->skb; 204 struct skb_frag_struct *frag; 205 void *end = ring->buf + ring->buf_size; 206 int frags = skb_shinfo(skb)->nr_frags; 207 int i; 208 __be32 *ptr = (__be32 *)tx_desc; 209 __be32 stamp = cpu_to_be32(STAMP_VAL | (!!owner << STAMP_SHIFT)); 210 struct skb_shared_hwtstamps hwts; 211 212 if (timestamp) { 213 mlx4_en_fill_hwtstamps(mdev, &hwts, timestamp); 214 skb_tstamp_tx(skb, &hwts); 215 } 216 217 /* Optimize the common case when there are no wraparounds */ 218 if (likely((void *) tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) { 219 if (!tx_info->inl) { 220 if (tx_info->linear) { 221 dma_unmap_single(priv->ddev, 222 (dma_addr_t) be64_to_cpu(data->addr), 223 be32_to_cpu(data->byte_count), 224 PCI_DMA_TODEVICE); 225 ++data; 226 } 227 228 for (i = 0; i < frags; i++) { 229 frag = &skb_shinfo(skb)->frags[i]; 230 dma_unmap_page(priv->ddev, 231 (dma_addr_t) be64_to_cpu(data[i].addr), 232 skb_frag_size(frag), PCI_DMA_TODEVICE); 233 } 234 } 235 /* Stamp the freed descriptor */ 236 for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { 237 *ptr = stamp; 238 ptr += STAMP_DWORDS; 239 } 240 241 } else { 242 if (!tx_info->inl) { 243 if ((void *) data >= end) { 244 data = ring->buf + ((void *)data - end); 245 } 246 247 if (tx_info->linear) { 248 dma_unmap_single(priv->ddev, 249 (dma_addr_t) be64_to_cpu(data->addr), 250 be32_to_cpu(data->byte_count), 251 PCI_DMA_TODEVICE); 252 ++data; 253 } 254 255 for (i = 0; i < frags; i++) { 256 /* Check for wraparound before unmapping */ 257 if ((void *) data >= end) 258 data = ring->buf; 259 frag = &skb_shinfo(skb)->frags[i]; 260 dma_unmap_page(priv->ddev, 261 (dma_addr_t) be64_to_cpu(data->addr), 262 skb_frag_size(frag), PCI_DMA_TODEVICE); 263 ++data; 264 } 265 } 266 /* Stamp the freed descriptor */ 267 for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { 268 *ptr = stamp; 269 ptr += STAMP_DWORDS; 270 if ((void *) ptr >= end) { 271 ptr = ring->buf; 272 stamp ^= cpu_to_be32(0x80000000); 273 } 274 } 275 276 } 277 dev_kfree_skb_any(skb); 278 return tx_info->nr_txbb; 279 } 280 281 282 int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) 283 { 284 struct mlx4_en_priv *priv = netdev_priv(dev); 285 int cnt = 0; 286 287 /* Skip last polled descriptor */ 288 ring->cons += ring->last_nr_txbb; 289 en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n", 290 ring->cons, ring->prod); 291 292 if ((u32) (ring->prod - ring->cons) > ring->size) { 293 if (netif_msg_tx_err(priv)) 294 en_warn(priv, "Tx consumer passed producer!\n"); 295 return 0; 296 } 297 298 while (ring->cons != ring->prod) { 299 ring->last_nr_txbb = mlx4_en_free_tx_desc(priv, ring, 300 ring->cons & ring->size_mask, 301 !!(ring->cons & ring->size), 0); 302 ring->cons += ring->last_nr_txbb; 303 cnt++; 304 } 305 306 netdev_tx_reset_queue(ring->tx_queue); 307 308 if (cnt) 309 en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt); 310 311 return cnt; 312 } 313 314 static void mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq) 315 { 316 struct mlx4_en_priv *priv = netdev_priv(dev); 317 struct mlx4_cq *mcq = &cq->mcq; 318 struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring]; 319 struct mlx4_cqe *cqe; 320 u16 index; 321 u16 new_index, ring_index; 322 u32 txbbs_skipped = 0; 323 u32 cons_index = mcq->cons_index; 324 int size = cq->size; 325 u32 size_mask = ring->size_mask; 326 struct mlx4_cqe *buf = cq->buf; 327 u32 packets = 0; 328 u32 bytes = 0; 329 int factor = priv->cqe_factor; 330 u64 timestamp = 0; 331 332 if (!priv->port_up) 333 return; 334 335 index = cons_index & size_mask; 336 cqe = &buf[(index << factor) + factor]; 337 ring_index = ring->cons & size_mask; 338 339 /* Process all completed CQEs */ 340 while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, 341 cons_index & size)) { 342 /* 343 * make sure we read the CQE after we read the 344 * ownership bit 345 */ 346 rmb(); 347 348 /* Skip over last polled CQE */ 349 new_index = be16_to_cpu(cqe->wqe_index) & size_mask; 350 351 do { 352 txbbs_skipped += ring->last_nr_txbb; 353 ring_index = (ring_index + ring->last_nr_txbb) & size_mask; 354 if (ring->tx_info[ring_index].ts_requested) 355 timestamp = mlx4_en_get_cqe_ts(cqe); 356 357 /* free next descriptor */ 358 ring->last_nr_txbb = mlx4_en_free_tx_desc( 359 priv, ring, ring_index, 360 !!((ring->cons + txbbs_skipped) & 361 ring->size), timestamp); 362 packets++; 363 bytes += ring->tx_info[ring_index].nr_bytes; 364 } while (ring_index != new_index); 365 366 ++cons_index; 367 index = cons_index & size_mask; 368 cqe = &buf[(index << factor) + factor]; 369 } 370 371 372 /* 373 * To prevent CQ overflow we first update CQ consumer and only then 374 * the ring consumer. 375 */ 376 mcq->cons_index = cons_index; 377 mlx4_cq_set_ci(mcq); 378 wmb(); 379 ring->cons += txbbs_skipped; 380 netdev_tx_completed_queue(ring->tx_queue, packets, bytes); 381 382 /* 383 * Wakeup Tx queue if this stopped, and at least 1 packet 384 * was completed 385 */ 386 if (netif_tx_queue_stopped(ring->tx_queue) && txbbs_skipped > 0) { 387 netif_tx_wake_queue(ring->tx_queue); 388 priv->port_stats.wake_queue++; 389 } 390 } 391 392 void mlx4_en_tx_irq(struct mlx4_cq *mcq) 393 { 394 struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); 395 struct mlx4_en_priv *priv = netdev_priv(cq->dev); 396 397 mlx4_en_process_tx_cq(cq->dev, cq); 398 mlx4_en_arm_cq(priv, cq); 399 } 400 401 402 static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv, 403 struct mlx4_en_tx_ring *ring, 404 u32 index, 405 unsigned int desc_size) 406 { 407 u32 copy = (ring->size - index) * TXBB_SIZE; 408 int i; 409 410 for (i = desc_size - copy - 4; i >= 0; i -= 4) { 411 if ((i & (TXBB_SIZE - 1)) == 0) 412 wmb(); 413 414 *((u32 *) (ring->buf + i)) = 415 *((u32 *) (ring->bounce_buf + copy + i)); 416 } 417 418 for (i = copy - 4; i >= 4 ; i -= 4) { 419 if ((i & (TXBB_SIZE - 1)) == 0) 420 wmb(); 421 422 *((u32 *) (ring->buf + index * TXBB_SIZE + i)) = 423 *((u32 *) (ring->bounce_buf + i)); 424 } 425 426 /* Return real descriptor location */ 427 return ring->buf + index * TXBB_SIZE; 428 } 429 430 static int is_inline(struct sk_buff *skb, void **pfrag) 431 { 432 void *ptr; 433 434 if (inline_thold && !skb_is_gso(skb) && skb->len <= inline_thold) { 435 if (skb_shinfo(skb)->nr_frags == 1) { 436 ptr = skb_frag_address_safe(&skb_shinfo(skb)->frags[0]); 437 if (unlikely(!ptr)) 438 return 0; 439 440 if (pfrag) 441 *pfrag = ptr; 442 443 return 1; 444 } else if (unlikely(skb_shinfo(skb)->nr_frags)) 445 return 0; 446 else 447 return 1; 448 } 449 450 return 0; 451 } 452 453 static int inline_size(struct sk_buff *skb) 454 { 455 if (skb->len + CTRL_SIZE + sizeof(struct mlx4_wqe_inline_seg) 456 <= MLX4_INLINE_ALIGN) 457 return ALIGN(skb->len + CTRL_SIZE + 458 sizeof(struct mlx4_wqe_inline_seg), 16); 459 else 460 return ALIGN(skb->len + CTRL_SIZE + 2 * 461 sizeof(struct mlx4_wqe_inline_seg), 16); 462 } 463 464 static int get_real_size(struct sk_buff *skb, struct net_device *dev, 465 int *lso_header_size) 466 { 467 struct mlx4_en_priv *priv = netdev_priv(dev); 468 int real_size; 469 470 if (skb_is_gso(skb)) { 471 *lso_header_size = skb_transport_offset(skb) + tcp_hdrlen(skb); 472 real_size = CTRL_SIZE + skb_shinfo(skb)->nr_frags * DS_SIZE + 473 ALIGN(*lso_header_size + 4, DS_SIZE); 474 if (unlikely(*lso_header_size != skb_headlen(skb))) { 475 /* We add a segment for the skb linear buffer only if 476 * it contains data */ 477 if (*lso_header_size < skb_headlen(skb)) 478 real_size += DS_SIZE; 479 else { 480 if (netif_msg_tx_err(priv)) 481 en_warn(priv, "Non-linear headers\n"); 482 return 0; 483 } 484 } 485 } else { 486 *lso_header_size = 0; 487 if (!is_inline(skb, NULL)) 488 real_size = CTRL_SIZE + (skb_shinfo(skb)->nr_frags + 1) * DS_SIZE; 489 else 490 real_size = inline_size(skb); 491 } 492 493 return real_size; 494 } 495 496 static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc, struct sk_buff *skb, 497 int real_size, u16 *vlan_tag, int tx_ind, void *fragptr) 498 { 499 struct mlx4_wqe_inline_seg *inl = &tx_desc->inl; 500 int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof *inl; 501 502 if (skb->len <= spc) { 503 inl->byte_count = cpu_to_be32(1 << 31 | skb->len); 504 skb_copy_from_linear_data(skb, inl + 1, skb_headlen(skb)); 505 if (skb_shinfo(skb)->nr_frags) 506 memcpy(((void *)(inl + 1)) + skb_headlen(skb), fragptr, 507 skb_frag_size(&skb_shinfo(skb)->frags[0])); 508 509 } else { 510 inl->byte_count = cpu_to_be32(1 << 31 | spc); 511 if (skb_headlen(skb) <= spc) { 512 skb_copy_from_linear_data(skb, inl + 1, skb_headlen(skb)); 513 if (skb_headlen(skb) < spc) { 514 memcpy(((void *)(inl + 1)) + skb_headlen(skb), 515 fragptr, spc - skb_headlen(skb)); 516 fragptr += spc - skb_headlen(skb); 517 } 518 inl = (void *) (inl + 1) + spc; 519 memcpy(((void *)(inl + 1)), fragptr, skb->len - spc); 520 } else { 521 skb_copy_from_linear_data(skb, inl + 1, spc); 522 inl = (void *) (inl + 1) + spc; 523 skb_copy_from_linear_data_offset(skb, spc, inl + 1, 524 skb_headlen(skb) - spc); 525 if (skb_shinfo(skb)->nr_frags) 526 memcpy(((void *)(inl + 1)) + skb_headlen(skb) - spc, 527 fragptr, skb_frag_size(&skb_shinfo(skb)->frags[0])); 528 } 529 530 wmb(); 531 inl->byte_count = cpu_to_be32(1 << 31 | (skb->len - spc)); 532 } 533 } 534 535 u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb) 536 { 537 struct mlx4_en_priv *priv = netdev_priv(dev); 538 u16 rings_p_up = priv->num_tx_rings_p_up; 539 u8 up = 0; 540 541 if (dev->num_tc) 542 return skb_tx_hash(dev, skb); 543 544 if (vlan_tx_tag_present(skb)) 545 up = vlan_tx_tag_get(skb) >> VLAN_PRIO_SHIFT; 546 547 return __skb_tx_hash(dev, skb, rings_p_up) + up * rings_p_up; 548 } 549 550 static void mlx4_bf_copy(void __iomem *dst, unsigned long *src, unsigned bytecnt) 551 { 552 __iowrite64_copy(dst, src, bytecnt / 8); 553 } 554 555 netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) 556 { 557 struct mlx4_en_priv *priv = netdev_priv(dev); 558 struct mlx4_en_dev *mdev = priv->mdev; 559 struct mlx4_en_tx_ring *ring; 560 struct mlx4_en_tx_desc *tx_desc; 561 struct mlx4_wqe_data_seg *data; 562 struct skb_frag_struct *frag; 563 struct mlx4_en_tx_info *tx_info; 564 struct ethhdr *ethh; 565 int tx_ind = 0; 566 int nr_txbb; 567 int desc_size; 568 int real_size; 569 dma_addr_t dma; 570 u32 index, bf_index; 571 __be32 op_own; 572 u16 vlan_tag = 0; 573 int i; 574 int lso_header_size; 575 void *fragptr; 576 bool bounce = false; 577 578 if (!priv->port_up) 579 goto tx_drop; 580 581 real_size = get_real_size(skb, dev, &lso_header_size); 582 if (unlikely(!real_size)) 583 goto tx_drop; 584 585 /* Align descriptor to TXBB size */ 586 desc_size = ALIGN(real_size, TXBB_SIZE); 587 nr_txbb = desc_size / TXBB_SIZE; 588 if (unlikely(nr_txbb > MAX_DESC_TXBBS)) { 589 if (netif_msg_tx_err(priv)) 590 en_warn(priv, "Oversized header or SG list\n"); 591 goto tx_drop; 592 } 593 594 tx_ind = skb->queue_mapping; 595 ring = &priv->tx_ring[tx_ind]; 596 if (vlan_tx_tag_present(skb)) 597 vlan_tag = vlan_tx_tag_get(skb); 598 599 /* Check available TXBBs And 2K spare for prefetch */ 600 if (unlikely(((int)(ring->prod - ring->cons)) > 601 ring->size - HEADROOM - MAX_DESC_TXBBS)) { 602 /* every full Tx ring stops queue */ 603 netif_tx_stop_queue(ring->tx_queue); 604 priv->port_stats.queue_stopped++; 605 606 /* If queue was emptied after the if, and before the 607 * stop_queue - need to wake the queue, or else it will remain 608 * stopped forever. 609 * Need a memory barrier to make sure ring->cons was not 610 * updated before queue was stopped. 611 */ 612 wmb(); 613 614 if (unlikely(((int)(ring->prod - ring->cons)) <= 615 ring->size - HEADROOM - MAX_DESC_TXBBS)) { 616 netif_tx_wake_queue(ring->tx_queue); 617 priv->port_stats.wake_queue++; 618 } else { 619 return NETDEV_TX_BUSY; 620 } 621 } 622 623 /* Track current inflight packets for performance analysis */ 624 AVG_PERF_COUNTER(priv->pstats.inflight_avg, 625 (u32) (ring->prod - ring->cons - 1)); 626 627 /* Packet is good - grab an index and transmit it */ 628 index = ring->prod & ring->size_mask; 629 bf_index = ring->prod; 630 631 /* See if we have enough space for whole descriptor TXBB for setting 632 * SW ownership on next descriptor; if not, use a bounce buffer. */ 633 if (likely(index + nr_txbb <= ring->size)) 634 tx_desc = ring->buf + index * TXBB_SIZE; 635 else { 636 tx_desc = (struct mlx4_en_tx_desc *) ring->bounce_buf; 637 bounce = true; 638 } 639 640 /* Save skb in tx_info ring */ 641 tx_info = &ring->tx_info[index]; 642 tx_info->skb = skb; 643 tx_info->nr_txbb = nr_txbb; 644 645 /* 646 * For timestamping add flag to skb_shinfo and 647 * set flag for further reference 648 */ 649 if (ring->hwtstamp_tx_type == HWTSTAMP_TX_ON && 650 skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) { 651 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 652 tx_info->ts_requested = 1; 653 } 654 655 /* Prepare ctrl segement apart opcode+ownership, which depends on 656 * whether LSO is used */ 657 tx_desc->ctrl.vlan_tag = cpu_to_be16(vlan_tag); 658 tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN * 659 !!vlan_tx_tag_present(skb); 660 tx_desc->ctrl.fence_size = (real_size / 16) & 0x3f; 661 tx_desc->ctrl.srcrb_flags = priv->ctrl_flags; 662 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 663 tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM | 664 MLX4_WQE_CTRL_TCP_UDP_CSUM); 665 ring->tx_csum++; 666 } 667 668 if (priv->flags & MLX4_EN_FLAG_ENABLE_HW_LOOPBACK) { 669 /* Copy dst mac address to wqe. This allows loopback in eSwitch, 670 * so that VFs and PF can communicate with each other 671 */ 672 ethh = (struct ethhdr *)skb->data; 673 tx_desc->ctrl.srcrb_flags16[0] = get_unaligned((__be16 *)ethh->h_dest); 674 tx_desc->ctrl.imm = get_unaligned((__be32 *)(ethh->h_dest + 2)); 675 } 676 677 /* Handle LSO (TSO) packets */ 678 if (lso_header_size) { 679 /* Mark opcode as LSO */ 680 op_own = cpu_to_be32(MLX4_OPCODE_LSO | (1 << 6)) | 681 ((ring->prod & ring->size) ? 682 cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0); 683 684 /* Fill in the LSO prefix */ 685 tx_desc->lso.mss_hdr_size = cpu_to_be32( 686 skb_shinfo(skb)->gso_size << 16 | lso_header_size); 687 688 /* Copy headers; 689 * note that we already verified that it is linear */ 690 memcpy(tx_desc->lso.header, skb->data, lso_header_size); 691 data = ((void *) &tx_desc->lso + 692 ALIGN(lso_header_size + 4, DS_SIZE)); 693 694 priv->port_stats.tso_packets++; 695 i = ((skb->len - lso_header_size) / skb_shinfo(skb)->gso_size) + 696 !!((skb->len - lso_header_size) % skb_shinfo(skb)->gso_size); 697 tx_info->nr_bytes = skb->len + (i - 1) * lso_header_size; 698 ring->packets += i; 699 } else { 700 /* Normal (Non LSO) packet */ 701 op_own = cpu_to_be32(MLX4_OPCODE_SEND) | 702 ((ring->prod & ring->size) ? 703 cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0); 704 data = &tx_desc->data; 705 tx_info->nr_bytes = max_t(unsigned int, skb->len, ETH_ZLEN); 706 ring->packets++; 707 708 } 709 ring->bytes += tx_info->nr_bytes; 710 netdev_tx_sent_queue(ring->tx_queue, tx_info->nr_bytes); 711 AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, skb->len); 712 713 714 /* valid only for none inline segments */ 715 tx_info->data_offset = (void *) data - (void *) tx_desc; 716 717 tx_info->linear = (lso_header_size < skb_headlen(skb) && !is_inline(skb, NULL)) ? 1 : 0; 718 data += skb_shinfo(skb)->nr_frags + tx_info->linear - 1; 719 720 if (!is_inline(skb, &fragptr)) { 721 /* Map fragments */ 722 for (i = skb_shinfo(skb)->nr_frags - 1; i >= 0; i--) { 723 frag = &skb_shinfo(skb)->frags[i]; 724 dma = skb_frag_dma_map(priv->ddev, frag, 725 0, skb_frag_size(frag), 726 DMA_TO_DEVICE); 727 data->addr = cpu_to_be64(dma); 728 data->lkey = cpu_to_be32(mdev->mr.key); 729 wmb(); 730 data->byte_count = cpu_to_be32(skb_frag_size(frag)); 731 --data; 732 } 733 734 /* Map linear part */ 735 if (tx_info->linear) { 736 dma = dma_map_single(priv->ddev, skb->data + lso_header_size, 737 skb_headlen(skb) - lso_header_size, PCI_DMA_TODEVICE); 738 data->addr = cpu_to_be64(dma); 739 data->lkey = cpu_to_be32(mdev->mr.key); 740 wmb(); 741 data->byte_count = cpu_to_be32(skb_headlen(skb) - lso_header_size); 742 } 743 tx_info->inl = 0; 744 } else { 745 build_inline_wqe(tx_desc, skb, real_size, &vlan_tag, tx_ind, fragptr); 746 tx_info->inl = 1; 747 } 748 749 ring->prod += nr_txbb; 750 751 /* If we used a bounce buffer then copy descriptor back into place */ 752 if (bounce) 753 tx_desc = mlx4_en_bounce_to_desc(priv, ring, index, desc_size); 754 755 skb_tx_timestamp(skb); 756 757 if (ring->bf_enabled && desc_size <= MAX_BF && !bounce && !vlan_tx_tag_present(skb)) { 758 *(__be32 *) (&tx_desc->ctrl.vlan_tag) |= cpu_to_be32(ring->doorbell_qpn); 759 op_own |= htonl((bf_index & 0xffff) << 8); 760 /* Ensure new descirptor hits memory 761 * before setting ownership of this descriptor to HW */ 762 wmb(); 763 tx_desc->ctrl.owner_opcode = op_own; 764 765 wmb(); 766 767 mlx4_bf_copy(ring->bf.reg + ring->bf.offset, (unsigned long *) &tx_desc->ctrl, 768 desc_size); 769 770 wmb(); 771 772 ring->bf.offset ^= ring->bf.buf_size; 773 } else { 774 /* Ensure new descirptor hits memory 775 * before setting ownership of this descriptor to HW */ 776 wmb(); 777 tx_desc->ctrl.owner_opcode = op_own; 778 wmb(); 779 iowrite32be(ring->doorbell_qpn, ring->bf.uar->map + MLX4_SEND_DOORBELL); 780 } 781 782 return NETDEV_TX_OK; 783 784 tx_drop: 785 dev_kfree_skb_any(skb); 786 priv->stats.tx_dropped++; 787 return NETDEV_TX_OK; 788 } 789 790