1 /* 2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34 #include <asm/page.h> 35 #include <linux/mlx4/cq.h> 36 #include <linux/slab.h> 37 #include <linux/mlx4/qp.h> 38 #include <linux/skbuff.h> 39 #include <linux/if_vlan.h> 40 #include <linux/prefetch.h> 41 #include <linux/vmalloc.h> 42 #include <linux/tcp.h> 43 #include <linux/ip.h> 44 #include <linux/moduleparam.h> 45 46 #include "mlx4_en.h" 47 48 int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, 49 struct mlx4_en_tx_ring **pring, u32 size, 50 u16 stride, int node, int queue_index) 51 { 52 struct mlx4_en_dev *mdev = priv->mdev; 53 struct mlx4_en_tx_ring *ring; 54 int tmp; 55 int err; 56 57 ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node); 58 if (!ring) { 59 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 60 if (!ring) { 61 en_err(priv, "Failed allocating TX ring\n"); 62 return -ENOMEM; 63 } 64 } 65 66 ring->size = size; 67 ring->size_mask = size - 1; 68 ring->stride = stride; 69 70 tmp = size * sizeof(struct mlx4_en_tx_info); 71 ring->tx_info = kmalloc_node(tmp, GFP_KERNEL | __GFP_NOWARN, node); 72 if (!ring->tx_info) { 73 ring->tx_info = vmalloc(tmp); 74 if (!ring->tx_info) { 75 err = -ENOMEM; 76 goto err_ring; 77 } 78 } 79 80 en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n", 81 ring->tx_info, tmp); 82 83 ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, GFP_KERNEL, node); 84 if (!ring->bounce_buf) { 85 ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL); 86 if (!ring->bounce_buf) { 87 err = -ENOMEM; 88 goto err_info; 89 } 90 } 91 ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE); 92 93 /* Allocate HW buffers on provided NUMA node */ 94 set_dev_node(&mdev->dev->persist->pdev->dev, node); 95 err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size, 96 2 * PAGE_SIZE); 97 set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node); 98 if (err) { 99 en_err(priv, "Failed allocating hwq resources\n"); 100 goto err_bounce; 101 } 102 103 err = mlx4_en_map_buffer(&ring->wqres.buf); 104 if (err) { 105 en_err(priv, "Failed to map TX buffer\n"); 106 goto err_hwq_res; 107 } 108 109 ring->buf = ring->wqres.buf.direct.buf; 110 111 en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d buf_size:%d dma:%llx\n", 112 ring, ring->buf, ring->size, ring->buf_size, 113 (unsigned long long) ring->wqres.buf.direct.map); 114 115 err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn, 116 MLX4_RESERVE_ETH_BF_QP); 117 if (err) { 118 en_err(priv, "failed reserving qp for TX ring\n"); 119 goto err_map; 120 } 121 122 err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp, GFP_KERNEL); 123 if (err) { 124 en_err(priv, "Failed allocating qp %d\n", ring->qpn); 125 goto err_reserve; 126 } 127 ring->qp.event = mlx4_en_sqp_event; 128 129 err = mlx4_bf_alloc(mdev->dev, &ring->bf, node); 130 if (err) { 131 en_dbg(DRV, priv, "working without blueflame (%d)\n", err); 132 ring->bf.uar = &mdev->priv_uar; 133 ring->bf.uar->map = mdev->uar_map; 134 ring->bf_enabled = false; 135 ring->bf_alloced = false; 136 priv->pflags &= ~MLX4_EN_PRIV_FLAGS_BLUEFLAME; 137 } else { 138 ring->bf_alloced = true; 139 ring->bf_enabled = !!(priv->pflags & 140 MLX4_EN_PRIV_FLAGS_BLUEFLAME); 141 } 142 143 ring->hwtstamp_tx_type = priv->hwtstamp_config.tx_type; 144 ring->queue_index = queue_index; 145 146 if (queue_index < priv->num_tx_rings_p_up) 147 cpumask_set_cpu_local_first(queue_index, 148 priv->mdev->dev->numa_node, 149 &ring->affinity_mask); 150 151 *pring = ring; 152 return 0; 153 154 err_reserve: 155 mlx4_qp_release_range(mdev->dev, ring->qpn, 1); 156 err_map: 157 mlx4_en_unmap_buffer(&ring->wqres.buf); 158 err_hwq_res: 159 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); 160 err_bounce: 161 kfree(ring->bounce_buf); 162 ring->bounce_buf = NULL; 163 err_info: 164 kvfree(ring->tx_info); 165 ring->tx_info = NULL; 166 err_ring: 167 kfree(ring); 168 *pring = NULL; 169 return err; 170 } 171 172 void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, 173 struct mlx4_en_tx_ring **pring) 174 { 175 struct mlx4_en_dev *mdev = priv->mdev; 176 struct mlx4_en_tx_ring *ring = *pring; 177 en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn); 178 179 if (ring->bf_alloced) 180 mlx4_bf_free(mdev->dev, &ring->bf); 181 mlx4_qp_remove(mdev->dev, &ring->qp); 182 mlx4_qp_free(mdev->dev, &ring->qp); 183 mlx4_en_unmap_buffer(&ring->wqres.buf); 184 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); 185 kfree(ring->bounce_buf); 186 ring->bounce_buf = NULL; 187 kvfree(ring->tx_info); 188 ring->tx_info = NULL; 189 kfree(ring); 190 *pring = NULL; 191 } 192 193 int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, 194 struct mlx4_en_tx_ring *ring, 195 int cq, int user_prio) 196 { 197 struct mlx4_en_dev *mdev = priv->mdev; 198 int err; 199 200 ring->cqn = cq; 201 ring->prod = 0; 202 ring->cons = 0xffffffff; 203 ring->last_nr_txbb = 1; 204 memset(ring->tx_info, 0, ring->size * sizeof(struct mlx4_en_tx_info)); 205 memset(ring->buf, 0, ring->buf_size); 206 207 ring->qp_state = MLX4_QP_STATE_RST; 208 ring->doorbell_qpn = cpu_to_be32(ring->qp.qpn << 8); 209 ring->mr_key = cpu_to_be32(mdev->mr.key); 210 211 mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 1, 0, ring->qpn, 212 ring->cqn, user_prio, &ring->context); 213 if (ring->bf_alloced) 214 ring->context.usr_page = cpu_to_be32(ring->bf.uar->index); 215 216 err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, &ring->context, 217 &ring->qp, &ring->qp_state); 218 if (!cpumask_empty(&ring->affinity_mask)) 219 netif_set_xps_queue(priv->dev, &ring->affinity_mask, 220 ring->queue_index); 221 222 return err; 223 } 224 225 void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, 226 struct mlx4_en_tx_ring *ring) 227 { 228 struct mlx4_en_dev *mdev = priv->mdev; 229 230 mlx4_qp_modify(mdev->dev, NULL, ring->qp_state, 231 MLX4_QP_STATE_RST, NULL, 0, 0, &ring->qp); 232 } 233 234 static void mlx4_en_stamp_wqe(struct mlx4_en_priv *priv, 235 struct mlx4_en_tx_ring *ring, int index, 236 u8 owner) 237 { 238 __be32 stamp = cpu_to_be32(STAMP_VAL | (!!owner << STAMP_SHIFT)); 239 struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE; 240 struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; 241 void *end = ring->buf + ring->buf_size; 242 __be32 *ptr = (__be32 *)tx_desc; 243 int i; 244 245 /* Optimize the common case when there are no wraparounds */ 246 if (likely((void *)tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) { 247 /* Stamp the freed descriptor */ 248 for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; 249 i += STAMP_STRIDE) { 250 *ptr = stamp; 251 ptr += STAMP_DWORDS; 252 } 253 } else { 254 /* Stamp the freed descriptor */ 255 for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; 256 i += STAMP_STRIDE) { 257 *ptr = stamp; 258 ptr += STAMP_DWORDS; 259 if ((void *)ptr >= end) { 260 ptr = ring->buf; 261 stamp ^= cpu_to_be32(0x80000000); 262 } 263 } 264 } 265 } 266 267 268 static u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, 269 struct mlx4_en_tx_ring *ring, 270 int index, u8 owner, u64 timestamp) 271 { 272 struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; 273 struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE; 274 struct mlx4_wqe_data_seg *data = (void *) tx_desc + tx_info->data_offset; 275 void *end = ring->buf + ring->buf_size; 276 struct sk_buff *skb = tx_info->skb; 277 int nr_maps = tx_info->nr_maps; 278 int i; 279 280 /* We do not touch skb here, so prefetch skb->users location 281 * to speedup consume_skb() 282 */ 283 prefetchw(&skb->users); 284 285 if (unlikely(timestamp)) { 286 struct skb_shared_hwtstamps hwts; 287 288 mlx4_en_fill_hwtstamps(priv->mdev, &hwts, timestamp); 289 skb_tstamp_tx(skb, &hwts); 290 } 291 292 /* Optimize the common case when there are no wraparounds */ 293 if (likely((void *) tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) { 294 if (!tx_info->inl) { 295 if (tx_info->linear) 296 dma_unmap_single(priv->ddev, 297 tx_info->map0_dma, 298 tx_info->map0_byte_count, 299 PCI_DMA_TODEVICE); 300 else 301 dma_unmap_page(priv->ddev, 302 tx_info->map0_dma, 303 tx_info->map0_byte_count, 304 PCI_DMA_TODEVICE); 305 for (i = 1; i < nr_maps; i++) { 306 data++; 307 dma_unmap_page(priv->ddev, 308 (dma_addr_t)be64_to_cpu(data->addr), 309 be32_to_cpu(data->byte_count), 310 PCI_DMA_TODEVICE); 311 } 312 } 313 } else { 314 if (!tx_info->inl) { 315 if ((void *) data >= end) { 316 data = ring->buf + ((void *)data - end); 317 } 318 319 if (tx_info->linear) 320 dma_unmap_single(priv->ddev, 321 tx_info->map0_dma, 322 tx_info->map0_byte_count, 323 PCI_DMA_TODEVICE); 324 else 325 dma_unmap_page(priv->ddev, 326 tx_info->map0_dma, 327 tx_info->map0_byte_count, 328 PCI_DMA_TODEVICE); 329 for (i = 1; i < nr_maps; i++) { 330 data++; 331 /* Check for wraparound before unmapping */ 332 if ((void *) data >= end) 333 data = ring->buf; 334 dma_unmap_page(priv->ddev, 335 (dma_addr_t)be64_to_cpu(data->addr), 336 be32_to_cpu(data->byte_count), 337 PCI_DMA_TODEVICE); 338 } 339 } 340 } 341 dev_consume_skb_any(skb); 342 return tx_info->nr_txbb; 343 } 344 345 346 int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) 347 { 348 struct mlx4_en_priv *priv = netdev_priv(dev); 349 int cnt = 0; 350 351 /* Skip last polled descriptor */ 352 ring->cons += ring->last_nr_txbb; 353 en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n", 354 ring->cons, ring->prod); 355 356 if ((u32) (ring->prod - ring->cons) > ring->size) { 357 if (netif_msg_tx_err(priv)) 358 en_warn(priv, "Tx consumer passed producer!\n"); 359 return 0; 360 } 361 362 while (ring->cons != ring->prod) { 363 ring->last_nr_txbb = mlx4_en_free_tx_desc(priv, ring, 364 ring->cons & ring->size_mask, 365 !!(ring->cons & ring->size), 0); 366 ring->cons += ring->last_nr_txbb; 367 cnt++; 368 } 369 370 netdev_tx_reset_queue(ring->tx_queue); 371 372 if (cnt) 373 en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt); 374 375 return cnt; 376 } 377 378 static bool mlx4_en_process_tx_cq(struct net_device *dev, 379 struct mlx4_en_cq *cq) 380 { 381 struct mlx4_en_priv *priv = netdev_priv(dev); 382 struct mlx4_cq *mcq = &cq->mcq; 383 struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; 384 struct mlx4_cqe *cqe; 385 u16 index; 386 u16 new_index, ring_index, stamp_index; 387 u32 txbbs_skipped = 0; 388 u32 txbbs_stamp = 0; 389 u32 cons_index = mcq->cons_index; 390 int size = cq->size; 391 u32 size_mask = ring->size_mask; 392 struct mlx4_cqe *buf = cq->buf; 393 u32 packets = 0; 394 u32 bytes = 0; 395 int factor = priv->cqe_factor; 396 u64 timestamp = 0; 397 int done = 0; 398 int budget = priv->tx_work_limit; 399 u32 last_nr_txbb; 400 u32 ring_cons; 401 402 if (!priv->port_up) 403 return true; 404 405 netdev_txq_bql_complete_prefetchw(ring->tx_queue); 406 407 index = cons_index & size_mask; 408 cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor; 409 last_nr_txbb = ACCESS_ONCE(ring->last_nr_txbb); 410 ring_cons = ACCESS_ONCE(ring->cons); 411 ring_index = ring_cons & size_mask; 412 stamp_index = ring_index; 413 414 /* Process all completed CQEs */ 415 while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, 416 cons_index & size) && (done < budget)) { 417 /* 418 * make sure we read the CQE after we read the 419 * ownership bit 420 */ 421 dma_rmb(); 422 423 if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == 424 MLX4_CQE_OPCODE_ERROR)) { 425 struct mlx4_err_cqe *cqe_err = (struct mlx4_err_cqe *)cqe; 426 427 en_err(priv, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n", 428 cqe_err->vendor_err_syndrome, 429 cqe_err->syndrome); 430 } 431 432 /* Skip over last polled CQE */ 433 new_index = be16_to_cpu(cqe->wqe_index) & size_mask; 434 435 do { 436 txbbs_skipped += last_nr_txbb; 437 ring_index = (ring_index + last_nr_txbb) & size_mask; 438 if (ring->tx_info[ring_index].ts_requested) 439 timestamp = mlx4_en_get_cqe_ts(cqe); 440 441 /* free next descriptor */ 442 last_nr_txbb = mlx4_en_free_tx_desc( 443 priv, ring, ring_index, 444 !!((ring_cons + txbbs_skipped) & 445 ring->size), timestamp); 446 447 mlx4_en_stamp_wqe(priv, ring, stamp_index, 448 !!((ring_cons + txbbs_stamp) & 449 ring->size)); 450 stamp_index = ring_index; 451 txbbs_stamp = txbbs_skipped; 452 packets++; 453 bytes += ring->tx_info[ring_index].nr_bytes; 454 } while ((++done < budget) && (ring_index != new_index)); 455 456 ++cons_index; 457 index = cons_index & size_mask; 458 cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor; 459 } 460 461 462 /* 463 * To prevent CQ overflow we first update CQ consumer and only then 464 * the ring consumer. 465 */ 466 mcq->cons_index = cons_index; 467 mlx4_cq_set_ci(mcq); 468 wmb(); 469 470 /* we want to dirty this cache line once */ 471 ACCESS_ONCE(ring->last_nr_txbb) = last_nr_txbb; 472 ACCESS_ONCE(ring->cons) = ring_cons + txbbs_skipped; 473 474 netdev_tx_completed_queue(ring->tx_queue, packets, bytes); 475 476 /* 477 * Wakeup Tx queue if this stopped, and at least 1 packet 478 * was completed 479 */ 480 if (netif_tx_queue_stopped(ring->tx_queue) && txbbs_skipped > 0) { 481 netif_tx_wake_queue(ring->tx_queue); 482 ring->wake_queue++; 483 } 484 return done < budget; 485 } 486 487 void mlx4_en_tx_irq(struct mlx4_cq *mcq) 488 { 489 struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); 490 struct mlx4_en_priv *priv = netdev_priv(cq->dev); 491 492 if (likely(priv->port_up)) 493 napi_schedule_irqoff(&cq->napi); 494 else 495 mlx4_en_arm_cq(priv, cq); 496 } 497 498 /* TX CQ polling - called by NAPI */ 499 int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget) 500 { 501 struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi); 502 struct net_device *dev = cq->dev; 503 struct mlx4_en_priv *priv = netdev_priv(dev); 504 int clean_complete; 505 506 clean_complete = mlx4_en_process_tx_cq(dev, cq); 507 if (!clean_complete) 508 return budget; 509 510 napi_complete(napi); 511 mlx4_en_arm_cq(priv, cq); 512 513 return 0; 514 } 515 516 static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv, 517 struct mlx4_en_tx_ring *ring, 518 u32 index, 519 unsigned int desc_size) 520 { 521 u32 copy = (ring->size - index) * TXBB_SIZE; 522 int i; 523 524 for (i = desc_size - copy - 4; i >= 0; i -= 4) { 525 if ((i & (TXBB_SIZE - 1)) == 0) 526 wmb(); 527 528 *((u32 *) (ring->buf + i)) = 529 *((u32 *) (ring->bounce_buf + copy + i)); 530 } 531 532 for (i = copy - 4; i >= 4 ; i -= 4) { 533 if ((i & (TXBB_SIZE - 1)) == 0) 534 wmb(); 535 536 *((u32 *) (ring->buf + index * TXBB_SIZE + i)) = 537 *((u32 *) (ring->bounce_buf + i)); 538 } 539 540 /* Return real descriptor location */ 541 return ring->buf + index * TXBB_SIZE; 542 } 543 544 /* Decide if skb can be inlined in tx descriptor to avoid dma mapping 545 * 546 * It seems strange we do not simply use skb_copy_bits(). 547 * This would allow to inline all skbs iff skb->len <= inline_thold 548 * 549 * Note that caller already checked skb was not a gso packet 550 */ 551 static bool is_inline(int inline_thold, const struct sk_buff *skb, 552 const struct skb_shared_info *shinfo, 553 void **pfrag) 554 { 555 void *ptr; 556 557 if (skb->len > inline_thold || !inline_thold) 558 return false; 559 560 if (shinfo->nr_frags == 1) { 561 ptr = skb_frag_address_safe(&shinfo->frags[0]); 562 if (unlikely(!ptr)) 563 return false; 564 *pfrag = ptr; 565 return true; 566 } 567 if (shinfo->nr_frags) 568 return false; 569 return true; 570 } 571 572 static int inline_size(const struct sk_buff *skb) 573 { 574 if (skb->len + CTRL_SIZE + sizeof(struct mlx4_wqe_inline_seg) 575 <= MLX4_INLINE_ALIGN) 576 return ALIGN(skb->len + CTRL_SIZE + 577 sizeof(struct mlx4_wqe_inline_seg), 16); 578 else 579 return ALIGN(skb->len + CTRL_SIZE + 2 * 580 sizeof(struct mlx4_wqe_inline_seg), 16); 581 } 582 583 static int get_real_size(const struct sk_buff *skb, 584 const struct skb_shared_info *shinfo, 585 struct net_device *dev, 586 int *lso_header_size, 587 bool *inline_ok, 588 void **pfrag) 589 { 590 struct mlx4_en_priv *priv = netdev_priv(dev); 591 int real_size; 592 593 if (shinfo->gso_size) { 594 *inline_ok = false; 595 if (skb->encapsulation) 596 *lso_header_size = (skb_inner_transport_header(skb) - skb->data) + inner_tcp_hdrlen(skb); 597 else 598 *lso_header_size = skb_transport_offset(skb) + tcp_hdrlen(skb); 599 real_size = CTRL_SIZE + shinfo->nr_frags * DS_SIZE + 600 ALIGN(*lso_header_size + 4, DS_SIZE); 601 if (unlikely(*lso_header_size != skb_headlen(skb))) { 602 /* We add a segment for the skb linear buffer only if 603 * it contains data */ 604 if (*lso_header_size < skb_headlen(skb)) 605 real_size += DS_SIZE; 606 else { 607 if (netif_msg_tx_err(priv)) 608 en_warn(priv, "Non-linear headers\n"); 609 return 0; 610 } 611 } 612 } else { 613 *lso_header_size = 0; 614 *inline_ok = is_inline(priv->prof->inline_thold, skb, 615 shinfo, pfrag); 616 617 if (*inline_ok) 618 real_size = inline_size(skb); 619 else 620 real_size = CTRL_SIZE + 621 (shinfo->nr_frags + 1) * DS_SIZE; 622 } 623 624 return real_size; 625 } 626 627 static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc, 628 const struct sk_buff *skb, 629 const struct skb_shared_info *shinfo, 630 int real_size, u16 *vlan_tag, 631 int tx_ind, void *fragptr) 632 { 633 struct mlx4_wqe_inline_seg *inl = &tx_desc->inl; 634 int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof *inl; 635 unsigned int hlen = skb_headlen(skb); 636 637 if (skb->len <= spc) { 638 if (likely(skb->len >= MIN_PKT_LEN)) { 639 inl->byte_count = cpu_to_be32(1 << 31 | skb->len); 640 } else { 641 inl->byte_count = cpu_to_be32(1 << 31 | MIN_PKT_LEN); 642 memset(((void *)(inl + 1)) + skb->len, 0, 643 MIN_PKT_LEN - skb->len); 644 } 645 skb_copy_from_linear_data(skb, inl + 1, hlen); 646 if (shinfo->nr_frags) 647 memcpy(((void *)(inl + 1)) + hlen, fragptr, 648 skb_frag_size(&shinfo->frags[0])); 649 650 } else { 651 inl->byte_count = cpu_to_be32(1 << 31 | spc); 652 if (hlen <= spc) { 653 skb_copy_from_linear_data(skb, inl + 1, hlen); 654 if (hlen < spc) { 655 memcpy(((void *)(inl + 1)) + hlen, 656 fragptr, spc - hlen); 657 fragptr += spc - hlen; 658 } 659 inl = (void *) (inl + 1) + spc; 660 memcpy(((void *)(inl + 1)), fragptr, skb->len - spc); 661 } else { 662 skb_copy_from_linear_data(skb, inl + 1, spc); 663 inl = (void *) (inl + 1) + spc; 664 skb_copy_from_linear_data_offset(skb, spc, inl + 1, 665 hlen - spc); 666 if (shinfo->nr_frags) 667 memcpy(((void *)(inl + 1)) + hlen - spc, 668 fragptr, 669 skb_frag_size(&shinfo->frags[0])); 670 } 671 672 dma_wmb(); 673 inl->byte_count = cpu_to_be32(1 << 31 | (skb->len - spc)); 674 } 675 } 676 677 u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb, 678 void *accel_priv, select_queue_fallback_t fallback) 679 { 680 struct mlx4_en_priv *priv = netdev_priv(dev); 681 u16 rings_p_up = priv->num_tx_rings_p_up; 682 u8 up = 0; 683 684 if (dev->num_tc) 685 return skb_tx_hash(dev, skb); 686 687 if (skb_vlan_tag_present(skb)) 688 up = skb_vlan_tag_get(skb) >> VLAN_PRIO_SHIFT; 689 690 return fallback(dev, skb) % rings_p_up + up * rings_p_up; 691 } 692 693 static void mlx4_bf_copy(void __iomem *dst, const void *src, 694 unsigned int bytecnt) 695 { 696 __iowrite64_copy(dst, src, bytecnt / 8); 697 } 698 699 netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) 700 { 701 struct skb_shared_info *shinfo = skb_shinfo(skb); 702 struct mlx4_en_priv *priv = netdev_priv(dev); 703 struct device *ddev = priv->ddev; 704 struct mlx4_en_tx_ring *ring; 705 struct mlx4_en_tx_desc *tx_desc; 706 struct mlx4_wqe_data_seg *data; 707 struct mlx4_en_tx_info *tx_info; 708 int tx_ind = 0; 709 int nr_txbb; 710 int desc_size; 711 int real_size; 712 u32 index, bf_index; 713 __be32 op_own; 714 u16 vlan_tag = 0; 715 int i_frag; 716 int lso_header_size; 717 void *fragptr = NULL; 718 bool bounce = false; 719 bool send_doorbell; 720 bool stop_queue; 721 bool inline_ok; 722 u32 ring_cons; 723 724 if (!priv->port_up) 725 goto tx_drop; 726 727 tx_ind = skb_get_queue_mapping(skb); 728 ring = priv->tx_ring[tx_ind]; 729 730 /* fetch ring->cons far ahead before needing it to avoid stall */ 731 ring_cons = ACCESS_ONCE(ring->cons); 732 733 real_size = get_real_size(skb, shinfo, dev, &lso_header_size, 734 &inline_ok, &fragptr); 735 if (unlikely(!real_size)) 736 goto tx_drop; 737 738 /* Align descriptor to TXBB size */ 739 desc_size = ALIGN(real_size, TXBB_SIZE); 740 nr_txbb = desc_size / TXBB_SIZE; 741 if (unlikely(nr_txbb > MAX_DESC_TXBBS)) { 742 if (netif_msg_tx_err(priv)) 743 en_warn(priv, "Oversized header or SG list\n"); 744 goto tx_drop; 745 } 746 747 if (skb_vlan_tag_present(skb)) 748 vlan_tag = skb_vlan_tag_get(skb); 749 750 751 netdev_txq_bql_enqueue_prefetchw(ring->tx_queue); 752 753 /* Track current inflight packets for performance analysis */ 754 AVG_PERF_COUNTER(priv->pstats.inflight_avg, 755 (u32)(ring->prod - ring_cons - 1)); 756 757 /* Packet is good - grab an index and transmit it */ 758 index = ring->prod & ring->size_mask; 759 bf_index = ring->prod; 760 761 /* See if we have enough space for whole descriptor TXBB for setting 762 * SW ownership on next descriptor; if not, use a bounce buffer. */ 763 if (likely(index + nr_txbb <= ring->size)) 764 tx_desc = ring->buf + index * TXBB_SIZE; 765 else { 766 tx_desc = (struct mlx4_en_tx_desc *) ring->bounce_buf; 767 bounce = true; 768 } 769 770 /* Save skb in tx_info ring */ 771 tx_info = &ring->tx_info[index]; 772 tx_info->skb = skb; 773 tx_info->nr_txbb = nr_txbb; 774 775 data = &tx_desc->data; 776 if (lso_header_size) 777 data = ((void *)&tx_desc->lso + ALIGN(lso_header_size + 4, 778 DS_SIZE)); 779 780 /* valid only for none inline segments */ 781 tx_info->data_offset = (void *)data - (void *)tx_desc; 782 783 tx_info->inl = inline_ok; 784 785 tx_info->linear = (lso_header_size < skb_headlen(skb) && 786 !inline_ok) ? 1 : 0; 787 788 tx_info->nr_maps = shinfo->nr_frags + tx_info->linear; 789 data += tx_info->nr_maps - 1; 790 791 if (!tx_info->inl) { 792 dma_addr_t dma = 0; 793 u32 byte_count = 0; 794 795 /* Map fragments if any */ 796 for (i_frag = shinfo->nr_frags - 1; i_frag >= 0; i_frag--) { 797 const struct skb_frag_struct *frag; 798 799 frag = &shinfo->frags[i_frag]; 800 byte_count = skb_frag_size(frag); 801 dma = skb_frag_dma_map(ddev, frag, 802 0, byte_count, 803 DMA_TO_DEVICE); 804 if (dma_mapping_error(ddev, dma)) 805 goto tx_drop_unmap; 806 807 data->addr = cpu_to_be64(dma); 808 data->lkey = ring->mr_key; 809 dma_wmb(); 810 data->byte_count = cpu_to_be32(byte_count); 811 --data; 812 } 813 814 /* Map linear part if needed */ 815 if (tx_info->linear) { 816 byte_count = skb_headlen(skb) - lso_header_size; 817 818 dma = dma_map_single(ddev, skb->data + 819 lso_header_size, byte_count, 820 PCI_DMA_TODEVICE); 821 if (dma_mapping_error(ddev, dma)) 822 goto tx_drop_unmap; 823 824 data->addr = cpu_to_be64(dma); 825 data->lkey = ring->mr_key; 826 dma_wmb(); 827 data->byte_count = cpu_to_be32(byte_count); 828 } 829 /* tx completion can avoid cache line miss for common cases */ 830 tx_info->map0_dma = dma; 831 tx_info->map0_byte_count = byte_count; 832 } 833 834 /* 835 * For timestamping add flag to skb_shinfo and 836 * set flag for further reference 837 */ 838 tx_info->ts_requested = 0; 839 if (unlikely(ring->hwtstamp_tx_type == HWTSTAMP_TX_ON && 840 shinfo->tx_flags & SKBTX_HW_TSTAMP)) { 841 shinfo->tx_flags |= SKBTX_IN_PROGRESS; 842 tx_info->ts_requested = 1; 843 } 844 845 /* Prepare ctrl segement apart opcode+ownership, which depends on 846 * whether LSO is used */ 847 tx_desc->ctrl.srcrb_flags = priv->ctrl_flags; 848 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 849 if (!skb->encapsulation) 850 tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM | 851 MLX4_WQE_CTRL_TCP_UDP_CSUM); 852 else 853 tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM); 854 ring->tx_csum++; 855 } 856 857 if (priv->flags & MLX4_EN_FLAG_ENABLE_HW_LOOPBACK) { 858 struct ethhdr *ethh; 859 860 /* Copy dst mac address to wqe. This allows loopback in eSwitch, 861 * so that VFs and PF can communicate with each other 862 */ 863 ethh = (struct ethhdr *)skb->data; 864 tx_desc->ctrl.srcrb_flags16[0] = get_unaligned((__be16 *)ethh->h_dest); 865 tx_desc->ctrl.imm = get_unaligned((__be32 *)(ethh->h_dest + 2)); 866 } 867 868 /* Handle LSO (TSO) packets */ 869 if (lso_header_size) { 870 int i; 871 872 /* Mark opcode as LSO */ 873 op_own = cpu_to_be32(MLX4_OPCODE_LSO | (1 << 6)) | 874 ((ring->prod & ring->size) ? 875 cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0); 876 877 /* Fill in the LSO prefix */ 878 tx_desc->lso.mss_hdr_size = cpu_to_be32( 879 shinfo->gso_size << 16 | lso_header_size); 880 881 /* Copy headers; 882 * note that we already verified that it is linear */ 883 memcpy(tx_desc->lso.header, skb->data, lso_header_size); 884 885 ring->tso_packets++; 886 887 i = ((skb->len - lso_header_size) / shinfo->gso_size) + 888 !!((skb->len - lso_header_size) % shinfo->gso_size); 889 tx_info->nr_bytes = skb->len + (i - 1) * lso_header_size; 890 ring->packets += i; 891 } else { 892 /* Normal (Non LSO) packet */ 893 op_own = cpu_to_be32(MLX4_OPCODE_SEND) | 894 ((ring->prod & ring->size) ? 895 cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0); 896 tx_info->nr_bytes = max_t(unsigned int, skb->len, ETH_ZLEN); 897 ring->packets++; 898 } 899 ring->bytes += tx_info->nr_bytes; 900 netdev_tx_sent_queue(ring->tx_queue, tx_info->nr_bytes); 901 AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, skb->len); 902 903 if (tx_info->inl) 904 build_inline_wqe(tx_desc, skb, shinfo, real_size, &vlan_tag, 905 tx_ind, fragptr); 906 907 if (skb->encapsulation) { 908 struct iphdr *ipv4 = (struct iphdr *)skb_inner_network_header(skb); 909 if (ipv4->protocol == IPPROTO_TCP || ipv4->protocol == IPPROTO_UDP) 910 op_own |= cpu_to_be32(MLX4_WQE_CTRL_IIP | MLX4_WQE_CTRL_ILP); 911 else 912 op_own |= cpu_to_be32(MLX4_WQE_CTRL_IIP); 913 } 914 915 ring->prod += nr_txbb; 916 917 /* If we used a bounce buffer then copy descriptor back into place */ 918 if (unlikely(bounce)) 919 tx_desc = mlx4_en_bounce_to_desc(priv, ring, index, desc_size); 920 921 skb_tx_timestamp(skb); 922 923 /* Check available TXBBs And 2K spare for prefetch */ 924 stop_queue = (int)(ring->prod - ring_cons) > 925 ring->size - HEADROOM - MAX_DESC_TXBBS; 926 if (unlikely(stop_queue)) { 927 netif_tx_stop_queue(ring->tx_queue); 928 ring->queue_stopped++; 929 } 930 send_doorbell = !skb->xmit_more || netif_xmit_stopped(ring->tx_queue); 931 932 real_size = (real_size / 16) & 0x3f; 933 934 if (ring->bf_enabled && desc_size <= MAX_BF && !bounce && 935 !skb_vlan_tag_present(skb) && send_doorbell) { 936 tx_desc->ctrl.bf_qpn = ring->doorbell_qpn | 937 cpu_to_be32(real_size); 938 939 op_own |= htonl((bf_index & 0xffff) << 8); 940 /* Ensure new descriptor hits memory 941 * before setting ownership of this descriptor to HW 942 */ 943 dma_wmb(); 944 tx_desc->ctrl.owner_opcode = op_own; 945 946 wmb(); 947 948 mlx4_bf_copy(ring->bf.reg + ring->bf.offset, &tx_desc->ctrl, 949 desc_size); 950 951 wmb(); 952 953 ring->bf.offset ^= ring->bf.buf_size; 954 } else { 955 tx_desc->ctrl.vlan_tag = cpu_to_be16(vlan_tag); 956 tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN * 957 !!skb_vlan_tag_present(skb); 958 tx_desc->ctrl.fence_size = real_size; 959 960 /* Ensure new descriptor hits memory 961 * before setting ownership of this descriptor to HW 962 */ 963 dma_wmb(); 964 tx_desc->ctrl.owner_opcode = op_own; 965 if (send_doorbell) { 966 wmb(); 967 /* Since there is no iowrite*_native() that writes the 968 * value as is, without byteswapping - using the one 969 * the doesn't do byteswapping in the relevant arch 970 * endianness. 971 */ 972 #if defined(__LITTLE_ENDIAN) 973 iowrite32( 974 #else 975 iowrite32be( 976 #endif 977 ring->doorbell_qpn, 978 ring->bf.uar->map + MLX4_SEND_DOORBELL); 979 } else { 980 ring->xmit_more++; 981 } 982 } 983 984 if (unlikely(stop_queue)) { 985 /* If queue was emptied after the if (stop_queue) , and before 986 * the netif_tx_stop_queue() - need to wake the queue, 987 * or else it will remain stopped forever. 988 * Need a memory barrier to make sure ring->cons was not 989 * updated before queue was stopped. 990 */ 991 smp_rmb(); 992 993 ring_cons = ACCESS_ONCE(ring->cons); 994 if (unlikely(((int)(ring->prod - ring_cons)) <= 995 ring->size - HEADROOM - MAX_DESC_TXBBS)) { 996 netif_tx_wake_queue(ring->tx_queue); 997 ring->wake_queue++; 998 } 999 } 1000 return NETDEV_TX_OK; 1001 1002 tx_drop_unmap: 1003 en_err(priv, "DMA mapping error\n"); 1004 1005 while (++i_frag < shinfo->nr_frags) { 1006 ++data; 1007 dma_unmap_page(ddev, (dma_addr_t) be64_to_cpu(data->addr), 1008 be32_to_cpu(data->byte_count), 1009 PCI_DMA_TODEVICE); 1010 } 1011 1012 tx_drop: 1013 dev_kfree_skb_any(skb); 1014 priv->stats.tx_dropped++; 1015 return NETDEV_TX_OK; 1016 } 1017 1018