1 /* 2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34 #include <linux/mlx4/cq.h> 35 #include <linux/slab.h> 36 #include <linux/mlx4/qp.h> 37 #include <linux/skbuff.h> 38 #include <linux/rculist.h> 39 #include <linux/if_ether.h> 40 #include <linux/if_vlan.h> 41 #include <linux/vmalloc.h> 42 43 #include "mlx4_en.h" 44 45 static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv, 46 struct mlx4_en_rx_desc *rx_desc, 47 struct mlx4_en_rx_alloc *frags, 48 struct mlx4_en_rx_alloc *ring_alloc) 49 { 50 struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS]; 51 struct mlx4_en_frag_info *frag_info; 52 struct page *page; 53 dma_addr_t dma; 54 int i; 55 56 for (i = 0; i < priv->num_frags; i++) { 57 frag_info = &priv->frag_info[i]; 58 if (ring_alloc[i].offset == frag_info->last_offset) { 59 page = alloc_pages(GFP_ATOMIC | __GFP_COMP, 60 MLX4_EN_ALLOC_ORDER); 61 if (!page) 62 goto out; 63 dma = dma_map_page(priv->ddev, page, 0, 64 MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE); 65 if (dma_mapping_error(priv->ddev, dma)) { 66 put_page(page); 67 goto out; 68 } 69 page_alloc[i].page = page; 70 page_alloc[i].dma = dma; 71 page_alloc[i].offset = frag_info->frag_align; 72 } else { 73 page_alloc[i].page = ring_alloc[i].page; 74 get_page(ring_alloc[i].page); 75 page_alloc[i].dma = ring_alloc[i].dma; 76 page_alloc[i].offset = ring_alloc[i].offset + 77 frag_info->frag_stride; 78 } 79 } 80 81 for (i = 0; i < priv->num_frags; i++) { 82 frags[i] = ring_alloc[i]; 83 dma = ring_alloc[i].dma + ring_alloc[i].offset; 84 ring_alloc[i] = page_alloc[i]; 85 rx_desc->data[i].addr = cpu_to_be64(dma); 86 } 87 88 return 0; 89 90 91 out: 92 while (i--) { 93 frag_info = &priv->frag_info[i]; 94 if (ring_alloc[i].offset == frag_info->last_offset) 95 dma_unmap_page(priv->ddev, page_alloc[i].dma, 96 MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE); 97 put_page(page_alloc[i].page); 98 } 99 return -ENOMEM; 100 } 101 102 static void mlx4_en_free_frag(struct mlx4_en_priv *priv, 103 struct mlx4_en_rx_alloc *frags, 104 int i) 105 { 106 struct mlx4_en_frag_info *frag_info = &priv->frag_info[i]; 107 108 if (frags[i].offset == frag_info->last_offset) { 109 dma_unmap_page(priv->ddev, frags[i].dma, MLX4_EN_ALLOC_SIZE, 110 PCI_DMA_FROMDEVICE); 111 } 112 if (frags[i].page) 113 put_page(frags[i].page); 114 } 115 116 static int mlx4_en_init_allocator(struct mlx4_en_priv *priv, 117 struct mlx4_en_rx_ring *ring) 118 { 119 struct mlx4_en_rx_alloc *page_alloc; 120 int i; 121 122 for (i = 0; i < priv->num_frags; i++) { 123 page_alloc = &ring->page_alloc[i]; 124 page_alloc->page = alloc_pages(GFP_ATOMIC | __GFP_COMP, 125 MLX4_EN_ALLOC_ORDER); 126 if (!page_alloc->page) 127 goto out; 128 129 page_alloc->dma = dma_map_page(priv->ddev, page_alloc->page, 0, 130 MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE); 131 if (dma_mapping_error(priv->ddev, page_alloc->dma)) { 132 put_page(page_alloc->page); 133 page_alloc->page = NULL; 134 goto out; 135 } 136 page_alloc->offset = priv->frag_info[i].frag_align; 137 en_dbg(DRV, priv, "Initialized allocator:%d with page:%p\n", 138 i, page_alloc->page); 139 } 140 return 0; 141 142 out: 143 while (i--) { 144 page_alloc = &ring->page_alloc[i]; 145 dma_unmap_page(priv->ddev, page_alloc->dma, 146 MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE); 147 put_page(page_alloc->page); 148 page_alloc->page = NULL; 149 } 150 return -ENOMEM; 151 } 152 153 static void mlx4_en_destroy_allocator(struct mlx4_en_priv *priv, 154 struct mlx4_en_rx_ring *ring) 155 { 156 struct mlx4_en_rx_alloc *page_alloc; 157 int i; 158 159 for (i = 0; i < priv->num_frags; i++) { 160 page_alloc = &ring->page_alloc[i]; 161 en_dbg(DRV, priv, "Freeing allocator:%d count:%d\n", 162 i, page_count(page_alloc->page)); 163 164 dma_unmap_page(priv->ddev, page_alloc->dma, 165 MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE); 166 put_page(page_alloc->page); 167 page_alloc->page = NULL; 168 } 169 } 170 171 static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, 172 struct mlx4_en_rx_ring *ring, int index) 173 { 174 struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index; 175 int possible_frags; 176 int i; 177 178 /* Set size and memtype fields */ 179 for (i = 0; i < priv->num_frags; i++) { 180 rx_desc->data[i].byte_count = 181 cpu_to_be32(priv->frag_info[i].frag_size); 182 rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key); 183 } 184 185 /* If the number of used fragments does not fill up the ring stride, 186 * remaining (unused) fragments must be padded with null address/size 187 * and a special memory key */ 188 possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; 189 for (i = priv->num_frags; i < possible_frags; i++) { 190 rx_desc->data[i].byte_count = 0; 191 rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); 192 rx_desc->data[i].addr = 0; 193 } 194 } 195 196 static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, 197 struct mlx4_en_rx_ring *ring, int index) 198 { 199 struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride); 200 struct mlx4_en_rx_alloc *frags = ring->rx_info + 201 (index << priv->log_rx_info); 202 203 return mlx4_en_alloc_frags(priv, rx_desc, frags, ring->page_alloc); 204 } 205 206 static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) 207 { 208 *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); 209 } 210 211 static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv, 212 struct mlx4_en_rx_ring *ring, 213 int index) 214 { 215 struct mlx4_en_rx_alloc *frags; 216 int nr; 217 218 frags = ring->rx_info + (index << priv->log_rx_info); 219 for (nr = 0; nr < priv->num_frags; nr++) { 220 en_dbg(DRV, priv, "Freeing fragment:%d\n", nr); 221 mlx4_en_free_frag(priv, frags, nr); 222 } 223 } 224 225 static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv) 226 { 227 struct mlx4_en_rx_ring *ring; 228 int ring_ind; 229 int buf_ind; 230 int new_size; 231 232 for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) { 233 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 234 ring = &priv->rx_ring[ring_ind]; 235 236 if (mlx4_en_prepare_rx_desc(priv, ring, 237 ring->actual_size)) { 238 if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) { 239 en_err(priv, "Failed to allocate " 240 "enough rx buffers\n"); 241 return -ENOMEM; 242 } else { 243 new_size = rounddown_pow_of_two(ring->actual_size); 244 en_warn(priv, "Only %d buffers allocated " 245 "reducing ring size to %d", 246 ring->actual_size, new_size); 247 goto reduce_rings; 248 } 249 } 250 ring->actual_size++; 251 ring->prod++; 252 } 253 } 254 return 0; 255 256 reduce_rings: 257 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 258 ring = &priv->rx_ring[ring_ind]; 259 while (ring->actual_size > new_size) { 260 ring->actual_size--; 261 ring->prod--; 262 mlx4_en_free_rx_desc(priv, ring, ring->actual_size); 263 } 264 } 265 266 return 0; 267 } 268 269 static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv, 270 struct mlx4_en_rx_ring *ring) 271 { 272 int index; 273 274 en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n", 275 ring->cons, ring->prod); 276 277 /* Unmap and free Rx buffers */ 278 BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size); 279 while (ring->cons != ring->prod) { 280 index = ring->cons & ring->size_mask; 281 en_dbg(DRV, priv, "Processing descriptor:%d\n", index); 282 mlx4_en_free_rx_desc(priv, ring, index); 283 ++ring->cons; 284 } 285 } 286 287 int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, 288 struct mlx4_en_rx_ring *ring, u32 size, u16 stride) 289 { 290 struct mlx4_en_dev *mdev = priv->mdev; 291 int err = -ENOMEM; 292 int tmp; 293 294 ring->prod = 0; 295 ring->cons = 0; 296 ring->size = size; 297 ring->size_mask = size - 1; 298 ring->stride = stride; 299 ring->log_stride = ffs(ring->stride) - 1; 300 ring->buf_size = ring->size * ring->stride + TXBB_SIZE; 301 302 tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS * 303 sizeof(struct mlx4_en_rx_alloc)); 304 ring->rx_info = vmalloc(tmp); 305 if (!ring->rx_info) 306 return -ENOMEM; 307 308 en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n", 309 ring->rx_info, tmp); 310 311 err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, 312 ring->buf_size, 2 * PAGE_SIZE); 313 if (err) 314 goto err_ring; 315 316 err = mlx4_en_map_buffer(&ring->wqres.buf); 317 if (err) { 318 en_err(priv, "Failed to map RX buffer\n"); 319 goto err_hwq; 320 } 321 ring->buf = ring->wqres.buf.direct.buf; 322 323 ring->hwtstamp_rx_filter = priv->hwtstamp_config.rx_filter; 324 325 return 0; 326 327 err_hwq: 328 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); 329 err_ring: 330 vfree(ring->rx_info); 331 ring->rx_info = NULL; 332 return err; 333 } 334 335 int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) 336 { 337 struct mlx4_en_rx_ring *ring; 338 int i; 339 int ring_ind; 340 int err; 341 int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + 342 DS_SIZE * priv->num_frags); 343 344 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 345 ring = &priv->rx_ring[ring_ind]; 346 347 ring->prod = 0; 348 ring->cons = 0; 349 ring->actual_size = 0; 350 ring->cqn = priv->rx_cq[ring_ind].mcq.cqn; 351 352 ring->stride = stride; 353 if (ring->stride <= TXBB_SIZE) 354 ring->buf += TXBB_SIZE; 355 356 ring->log_stride = ffs(ring->stride) - 1; 357 ring->buf_size = ring->size * ring->stride; 358 359 memset(ring->buf, 0, ring->buf_size); 360 mlx4_en_update_rx_prod_db(ring); 361 362 /* Initialize all descriptors */ 363 for (i = 0; i < ring->size; i++) 364 mlx4_en_init_rx_desc(priv, ring, i); 365 366 /* Initialize page allocators */ 367 err = mlx4_en_init_allocator(priv, ring); 368 if (err) { 369 en_err(priv, "Failed initializing ring allocator\n"); 370 if (ring->stride <= TXBB_SIZE) 371 ring->buf -= TXBB_SIZE; 372 ring_ind--; 373 goto err_allocator; 374 } 375 } 376 err = mlx4_en_fill_rx_buffers(priv); 377 if (err) 378 goto err_buffers; 379 380 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 381 ring = &priv->rx_ring[ring_ind]; 382 383 ring->size_mask = ring->actual_size - 1; 384 mlx4_en_update_rx_prod_db(ring); 385 } 386 387 return 0; 388 389 err_buffers: 390 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) 391 mlx4_en_free_rx_buf(priv, &priv->rx_ring[ring_ind]); 392 393 ring_ind = priv->rx_ring_num - 1; 394 err_allocator: 395 while (ring_ind >= 0) { 396 if (priv->rx_ring[ring_ind].stride <= TXBB_SIZE) 397 priv->rx_ring[ring_ind].buf -= TXBB_SIZE; 398 mlx4_en_destroy_allocator(priv, &priv->rx_ring[ring_ind]); 399 ring_ind--; 400 } 401 return err; 402 } 403 404 void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, 405 struct mlx4_en_rx_ring *ring, u32 size, u16 stride) 406 { 407 struct mlx4_en_dev *mdev = priv->mdev; 408 409 mlx4_en_unmap_buffer(&ring->wqres.buf); 410 mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); 411 vfree(ring->rx_info); 412 ring->rx_info = NULL; 413 #ifdef CONFIG_RFS_ACCEL 414 mlx4_en_cleanup_filters(priv, ring); 415 #endif 416 } 417 418 void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, 419 struct mlx4_en_rx_ring *ring) 420 { 421 mlx4_en_free_rx_buf(priv, ring); 422 if (ring->stride <= TXBB_SIZE) 423 ring->buf -= TXBB_SIZE; 424 mlx4_en_destroy_allocator(priv, ring); 425 } 426 427 428 static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv, 429 struct mlx4_en_rx_desc *rx_desc, 430 struct mlx4_en_rx_alloc *frags, 431 struct sk_buff *skb, 432 int length) 433 { 434 struct skb_frag_struct *skb_frags_rx = skb_shinfo(skb)->frags; 435 struct mlx4_en_frag_info *frag_info; 436 int nr; 437 dma_addr_t dma; 438 439 /* Collect used fragments while replacing them in the HW descriptors */ 440 for (nr = 0; nr < priv->num_frags; nr++) { 441 frag_info = &priv->frag_info[nr]; 442 if (length <= frag_info->frag_prefix_size) 443 break; 444 if (!frags[nr].page) 445 goto fail; 446 447 dma = be64_to_cpu(rx_desc->data[nr].addr); 448 dma_sync_single_for_cpu(priv->ddev, dma, frag_info->frag_size, 449 DMA_FROM_DEVICE); 450 451 /* Save page reference in skb */ 452 get_page(frags[nr].page); 453 __skb_frag_set_page(&skb_frags_rx[nr], frags[nr].page); 454 skb_frag_size_set(&skb_frags_rx[nr], frag_info->frag_size); 455 skb_frags_rx[nr].page_offset = frags[nr].offset; 456 skb->truesize += frag_info->frag_stride; 457 } 458 /* Adjust size of last fragment to match actual length */ 459 if (nr > 0) 460 skb_frag_size_set(&skb_frags_rx[nr - 1], 461 length - priv->frag_info[nr - 1].frag_prefix_size); 462 return nr; 463 464 fail: 465 while (nr > 0) { 466 nr--; 467 __skb_frag_unref(&skb_frags_rx[nr]); 468 } 469 return 0; 470 } 471 472 473 static struct sk_buff *mlx4_en_rx_skb(struct mlx4_en_priv *priv, 474 struct mlx4_en_rx_desc *rx_desc, 475 struct mlx4_en_rx_alloc *frags, 476 unsigned int length) 477 { 478 struct sk_buff *skb; 479 void *va; 480 int used_frags; 481 dma_addr_t dma; 482 483 skb = netdev_alloc_skb(priv->dev, SMALL_PACKET_SIZE + NET_IP_ALIGN); 484 if (!skb) { 485 en_dbg(RX_ERR, priv, "Failed allocating skb\n"); 486 return NULL; 487 } 488 skb_reserve(skb, NET_IP_ALIGN); 489 skb->len = length; 490 491 /* Get pointer to first fragment so we could copy the headers into the 492 * (linear part of the) skb */ 493 va = page_address(frags[0].page) + frags[0].offset; 494 495 if (length <= SMALL_PACKET_SIZE) { 496 /* We are copying all relevant data to the skb - temporarily 497 * sync buffers for the copy */ 498 dma = be64_to_cpu(rx_desc->data[0].addr); 499 dma_sync_single_for_cpu(priv->ddev, dma, length, 500 DMA_FROM_DEVICE); 501 skb_copy_to_linear_data(skb, va, length); 502 skb->tail += length; 503 } else { 504 /* Move relevant fragments to skb */ 505 used_frags = mlx4_en_complete_rx_desc(priv, rx_desc, frags, 506 skb, length); 507 if (unlikely(!used_frags)) { 508 kfree_skb(skb); 509 return NULL; 510 } 511 skb_shinfo(skb)->nr_frags = used_frags; 512 513 /* Copy headers into the skb linear buffer */ 514 memcpy(skb->data, va, HEADER_COPY_SIZE); 515 skb->tail += HEADER_COPY_SIZE; 516 517 /* Skip headers in first fragment */ 518 skb_shinfo(skb)->frags[0].page_offset += HEADER_COPY_SIZE; 519 520 /* Adjust size of first fragment */ 521 skb_frag_size_sub(&skb_shinfo(skb)->frags[0], HEADER_COPY_SIZE); 522 skb->data_len = length - HEADER_COPY_SIZE; 523 } 524 return skb; 525 } 526 527 static void validate_loopback(struct mlx4_en_priv *priv, struct sk_buff *skb) 528 { 529 int i; 530 int offset = ETH_HLEN; 531 532 for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) { 533 if (*(skb->data + offset) != (unsigned char) (i & 0xff)) 534 goto out_loopback; 535 } 536 /* Loopback found */ 537 priv->loopback_ok = 1; 538 539 out_loopback: 540 dev_kfree_skb_any(skb); 541 } 542 543 static void mlx4_en_refill_rx_buffers(struct mlx4_en_priv *priv, 544 struct mlx4_en_rx_ring *ring) 545 { 546 int index = ring->prod & ring->size_mask; 547 548 while ((u32) (ring->prod - ring->cons) < ring->actual_size) { 549 if (mlx4_en_prepare_rx_desc(priv, ring, index)) 550 break; 551 ring->prod++; 552 index = ring->prod & ring->size_mask; 553 } 554 } 555 556 int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget) 557 { 558 struct mlx4_en_priv *priv = netdev_priv(dev); 559 struct mlx4_en_dev *mdev = priv->mdev; 560 struct mlx4_cqe *cqe; 561 struct mlx4_en_rx_ring *ring = &priv->rx_ring[cq->ring]; 562 struct mlx4_en_rx_alloc *frags; 563 struct mlx4_en_rx_desc *rx_desc; 564 struct sk_buff *skb; 565 int index; 566 int nr; 567 unsigned int length; 568 int polled = 0; 569 int ip_summed; 570 int factor = priv->cqe_factor; 571 u64 timestamp; 572 573 if (!priv->port_up) 574 return 0; 575 576 /* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx 577 * descriptor offset can be deduced from the CQE index instead of 578 * reading 'cqe->index' */ 579 index = cq->mcq.cons_index & ring->size_mask; 580 cqe = &cq->buf[(index << factor) + factor]; 581 582 /* Process all completed CQEs */ 583 while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, 584 cq->mcq.cons_index & cq->size)) { 585 586 frags = ring->rx_info + (index << priv->log_rx_info); 587 rx_desc = ring->buf + (index << ring->log_stride); 588 589 /* 590 * make sure we read the CQE after we read the ownership bit 591 */ 592 rmb(); 593 594 /* Drop packet on bad receive or bad checksum */ 595 if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == 596 MLX4_CQE_OPCODE_ERROR)) { 597 en_err(priv, "CQE completed in error - vendor " 598 "syndrom:%d syndrom:%d\n", 599 ((struct mlx4_err_cqe *) cqe)->vendor_err_syndrome, 600 ((struct mlx4_err_cqe *) cqe)->syndrome); 601 goto next; 602 } 603 if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) { 604 en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n"); 605 goto next; 606 } 607 608 /* Check if we need to drop the packet if SRIOV is not enabled 609 * and not performing the selftest or flb disabled 610 */ 611 if (priv->flags & MLX4_EN_FLAG_RX_FILTER_NEEDED) { 612 struct ethhdr *ethh; 613 dma_addr_t dma; 614 /* Get pointer to first fragment since we haven't 615 * skb yet and cast it to ethhdr struct 616 */ 617 dma = be64_to_cpu(rx_desc->data[0].addr); 618 dma_sync_single_for_cpu(priv->ddev, dma, sizeof(*ethh), 619 DMA_FROM_DEVICE); 620 ethh = (struct ethhdr *)(page_address(frags[0].page) + 621 frags[0].offset); 622 623 if (is_multicast_ether_addr(ethh->h_dest)) { 624 struct mlx4_mac_entry *entry; 625 struct hlist_head *bucket; 626 unsigned int mac_hash; 627 628 /* Drop the packet, since HW loopback-ed it */ 629 mac_hash = ethh->h_source[MLX4_EN_MAC_HASH_IDX]; 630 bucket = &priv->mac_hash[mac_hash]; 631 rcu_read_lock(); 632 hlist_for_each_entry_rcu(entry, bucket, hlist) { 633 if (ether_addr_equal_64bits(entry->mac, 634 ethh->h_source)) { 635 rcu_read_unlock(); 636 goto next; 637 } 638 } 639 rcu_read_unlock(); 640 } 641 } 642 643 /* 644 * Packet is OK - process it. 645 */ 646 length = be32_to_cpu(cqe->byte_cnt); 647 length -= ring->fcs_del; 648 ring->bytes += length; 649 ring->packets++; 650 651 if (likely(dev->features & NETIF_F_RXCSUM)) { 652 if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) && 653 (cqe->checksum == cpu_to_be16(0xffff))) { 654 ring->csum_ok++; 655 /* This packet is eligible for GRO if it is: 656 * - DIX Ethernet (type interpretation) 657 * - TCP/IP (v4) 658 * - without IP options 659 * - not an IP fragment */ 660 if (dev->features & NETIF_F_GRO) { 661 struct sk_buff *gro_skb = napi_get_frags(&cq->napi); 662 if (!gro_skb) 663 goto next; 664 665 nr = mlx4_en_complete_rx_desc(priv, 666 rx_desc, frags, gro_skb, 667 length); 668 if (!nr) 669 goto next; 670 671 skb_shinfo(gro_skb)->nr_frags = nr; 672 gro_skb->len = length; 673 gro_skb->data_len = length; 674 gro_skb->ip_summed = CHECKSUM_UNNECESSARY; 675 676 if ((cqe->vlan_my_qpn & 677 cpu_to_be32(MLX4_CQE_VLAN_PRESENT_MASK)) && 678 (dev->features & NETIF_F_HW_VLAN_CTAG_RX)) { 679 u16 vid = be16_to_cpu(cqe->sl_vid); 680 681 __vlan_hwaccel_put_tag(gro_skb, htons(ETH_P_8021Q), vid); 682 } 683 684 if (dev->features & NETIF_F_RXHASH) 685 gro_skb->rxhash = be32_to_cpu(cqe->immed_rss_invalid); 686 687 skb_record_rx_queue(gro_skb, cq->ring); 688 689 if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) { 690 timestamp = mlx4_en_get_cqe_ts(cqe); 691 mlx4_en_fill_hwtstamps(mdev, 692 skb_hwtstamps(gro_skb), 693 timestamp); 694 } 695 696 napi_gro_frags(&cq->napi); 697 goto next; 698 } 699 700 /* GRO not possible, complete processing here */ 701 ip_summed = CHECKSUM_UNNECESSARY; 702 } else { 703 ip_summed = CHECKSUM_NONE; 704 ring->csum_none++; 705 } 706 } else { 707 ip_summed = CHECKSUM_NONE; 708 ring->csum_none++; 709 } 710 711 skb = mlx4_en_rx_skb(priv, rx_desc, frags, length); 712 if (!skb) { 713 priv->stats.rx_dropped++; 714 goto next; 715 } 716 717 if (unlikely(priv->validate_loopback)) { 718 validate_loopback(priv, skb); 719 goto next; 720 } 721 722 skb->ip_summed = ip_summed; 723 skb->protocol = eth_type_trans(skb, dev); 724 skb_record_rx_queue(skb, cq->ring); 725 726 if (dev->features & NETIF_F_RXHASH) 727 skb->rxhash = be32_to_cpu(cqe->immed_rss_invalid); 728 729 if ((be32_to_cpu(cqe->vlan_my_qpn) & 730 MLX4_CQE_VLAN_PRESENT_MASK) && 731 (dev->features & NETIF_F_HW_VLAN_CTAG_RX)) 732 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), be16_to_cpu(cqe->sl_vid)); 733 734 if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) { 735 timestamp = mlx4_en_get_cqe_ts(cqe); 736 mlx4_en_fill_hwtstamps(mdev, skb_hwtstamps(skb), 737 timestamp); 738 } 739 740 /* Push it up the stack */ 741 netif_receive_skb(skb); 742 743 next: 744 for (nr = 0; nr < priv->num_frags; nr++) 745 mlx4_en_free_frag(priv, frags, nr); 746 747 ++cq->mcq.cons_index; 748 index = (cq->mcq.cons_index) & ring->size_mask; 749 cqe = &cq->buf[(index << factor) + factor]; 750 if (++polled == budget) 751 goto out; 752 } 753 754 out: 755 AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled); 756 mlx4_cq_set_ci(&cq->mcq); 757 wmb(); /* ensure HW sees CQ consumer before we post new buffers */ 758 ring->cons = cq->mcq.cons_index; 759 mlx4_en_refill_rx_buffers(priv, ring); 760 mlx4_en_update_rx_prod_db(ring); 761 return polled; 762 } 763 764 765 void mlx4_en_rx_irq(struct mlx4_cq *mcq) 766 { 767 struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); 768 struct mlx4_en_priv *priv = netdev_priv(cq->dev); 769 770 if (priv->port_up) 771 napi_schedule(&cq->napi); 772 else 773 mlx4_en_arm_cq(priv, cq); 774 } 775 776 /* Rx CQ polling - called by NAPI */ 777 int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget) 778 { 779 struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi); 780 struct net_device *dev = cq->dev; 781 struct mlx4_en_priv *priv = netdev_priv(dev); 782 int done; 783 784 done = mlx4_en_process_rx_cq(dev, cq, budget); 785 786 /* If we used up all the quota - we're probably not done yet... */ 787 if (done == budget) 788 INC_PERF_COUNTER(priv->pstats.napi_quota); 789 else { 790 /* Done for now */ 791 napi_complete(napi); 792 mlx4_en_arm_cq(priv, cq); 793 } 794 return done; 795 } 796 797 798 /* Calculate the last offset position that accommodates a full fragment 799 * (assuming fagment size = stride-align) */ 800 static int mlx4_en_last_alloc_offset(struct mlx4_en_priv *priv, u16 stride, u16 align) 801 { 802 u16 res = MLX4_EN_ALLOC_SIZE % stride; 803 u16 offset = MLX4_EN_ALLOC_SIZE - stride - res + align; 804 805 en_dbg(DRV, priv, "Calculated last offset for stride:%d align:%d " 806 "res:%d offset:%d\n", stride, align, res, offset); 807 return offset; 808 } 809 810 811 static int frag_sizes[] = { 812 FRAG_SZ0, 813 FRAG_SZ1, 814 FRAG_SZ2, 815 FRAG_SZ3 816 }; 817 818 void mlx4_en_calc_rx_buf(struct net_device *dev) 819 { 820 struct mlx4_en_priv *priv = netdev_priv(dev); 821 int eff_mtu = dev->mtu + ETH_HLEN + VLAN_HLEN + ETH_LLC_SNAP_SIZE; 822 int buf_size = 0; 823 int i = 0; 824 825 while (buf_size < eff_mtu) { 826 priv->frag_info[i].frag_size = 827 (eff_mtu > buf_size + frag_sizes[i]) ? 828 frag_sizes[i] : eff_mtu - buf_size; 829 priv->frag_info[i].frag_prefix_size = buf_size; 830 if (!i) { 831 priv->frag_info[i].frag_align = NET_IP_ALIGN; 832 priv->frag_info[i].frag_stride = 833 ALIGN(frag_sizes[i] + NET_IP_ALIGN, SMP_CACHE_BYTES); 834 } else { 835 priv->frag_info[i].frag_align = 0; 836 priv->frag_info[i].frag_stride = 837 ALIGN(frag_sizes[i], SMP_CACHE_BYTES); 838 } 839 priv->frag_info[i].last_offset = mlx4_en_last_alloc_offset( 840 priv, priv->frag_info[i].frag_stride, 841 priv->frag_info[i].frag_align); 842 buf_size += priv->frag_info[i].frag_size; 843 i++; 844 } 845 846 priv->num_frags = i; 847 priv->rx_skb_size = eff_mtu; 848 priv->log_rx_info = ROUNDUP_LOG2(i * sizeof(struct mlx4_en_rx_alloc)); 849 850 en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d " 851 "num_frags:%d):\n", eff_mtu, priv->num_frags); 852 for (i = 0; i < priv->num_frags; i++) { 853 en_dbg(DRV, priv, " frag:%d - size:%d prefix:%d align:%d " 854 "stride:%d last_offset:%d\n", i, 855 priv->frag_info[i].frag_size, 856 priv->frag_info[i].frag_prefix_size, 857 priv->frag_info[i].frag_align, 858 priv->frag_info[i].frag_stride, 859 priv->frag_info[i].last_offset); 860 } 861 } 862 863 /* RSS related functions */ 864 865 static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn, 866 struct mlx4_en_rx_ring *ring, 867 enum mlx4_qp_state *state, 868 struct mlx4_qp *qp) 869 { 870 struct mlx4_en_dev *mdev = priv->mdev; 871 struct mlx4_qp_context *context; 872 int err = 0; 873 874 context = kmalloc(sizeof(*context), GFP_KERNEL); 875 if (!context) 876 return -ENOMEM; 877 878 err = mlx4_qp_alloc(mdev->dev, qpn, qp); 879 if (err) { 880 en_err(priv, "Failed to allocate qp #%x\n", qpn); 881 goto out; 882 } 883 qp->event = mlx4_en_sqp_event; 884 885 memset(context, 0, sizeof *context); 886 mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0, 887 qpn, ring->cqn, -1, context); 888 context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma); 889 890 /* Cancel FCS removal if FW allows */ 891 if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) { 892 context->param3 |= cpu_to_be32(1 << 29); 893 ring->fcs_del = ETH_FCS_LEN; 894 } else 895 ring->fcs_del = 0; 896 897 err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state); 898 if (err) { 899 mlx4_qp_remove(mdev->dev, qp); 900 mlx4_qp_free(mdev->dev, qp); 901 } 902 mlx4_en_update_rx_prod_db(ring); 903 out: 904 kfree(context); 905 return err; 906 } 907 908 int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv) 909 { 910 int err; 911 u32 qpn; 912 913 err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn); 914 if (err) { 915 en_err(priv, "Failed reserving drop qpn\n"); 916 return err; 917 } 918 err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp); 919 if (err) { 920 en_err(priv, "Failed allocating drop qp\n"); 921 mlx4_qp_release_range(priv->mdev->dev, qpn, 1); 922 return err; 923 } 924 925 return 0; 926 } 927 928 void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv) 929 { 930 u32 qpn; 931 932 qpn = priv->drop_qp.qpn; 933 mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp); 934 mlx4_qp_free(priv->mdev->dev, &priv->drop_qp); 935 mlx4_qp_release_range(priv->mdev->dev, qpn, 1); 936 } 937 938 /* Allocate rx qp's and configure them according to rss map */ 939 int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) 940 { 941 struct mlx4_en_dev *mdev = priv->mdev; 942 struct mlx4_en_rss_map *rss_map = &priv->rss_map; 943 struct mlx4_qp_context context; 944 struct mlx4_rss_context *rss_context; 945 int rss_rings; 946 void *ptr; 947 u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 | 948 MLX4_RSS_TCP_IPV6); 949 int i, qpn; 950 int err = 0; 951 int good_qps = 0; 952 static const u32 rsskey[10] = { 0xD181C62C, 0xF7F4DB5B, 0x1983A2FC, 953 0x943E1ADB, 0xD9389E6B, 0xD1039C2C, 0xA74499AD, 954 0x593D56D9, 0xF3253C06, 0x2ADC1FFC}; 955 956 en_dbg(DRV, priv, "Configuring rss steering\n"); 957 err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num, 958 priv->rx_ring_num, 959 &rss_map->base_qpn); 960 if (err) { 961 en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num); 962 return err; 963 } 964 965 for (i = 0; i < priv->rx_ring_num; i++) { 966 qpn = rss_map->base_qpn + i; 967 err = mlx4_en_config_rss_qp(priv, qpn, &priv->rx_ring[i], 968 &rss_map->state[i], 969 &rss_map->qps[i]); 970 if (err) 971 goto rss_err; 972 973 ++good_qps; 974 } 975 976 /* Configure RSS indirection qp */ 977 err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp); 978 if (err) { 979 en_err(priv, "Failed to allocate RSS indirection QP\n"); 980 goto rss_err; 981 } 982 rss_map->indir_qp.event = mlx4_en_sqp_event; 983 mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn, 984 priv->rx_ring[0].cqn, -1, &context); 985 986 if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num) 987 rss_rings = priv->rx_ring_num; 988 else 989 rss_rings = priv->prof->rss_rings; 990 991 ptr = ((void *) &context) + offsetof(struct mlx4_qp_context, pri_path) 992 + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH; 993 rss_context = ptr; 994 rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 | 995 (rss_map->base_qpn)); 996 rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn); 997 if (priv->mdev->profile.udp_rss) { 998 rss_mask |= MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6; 999 rss_context->base_qpn_udp = rss_context->default_qpn; 1000 } 1001 rss_context->flags = rss_mask; 1002 rss_context->hash_fn = MLX4_RSS_HASH_TOP; 1003 for (i = 0; i < 10; i++) 1004 rss_context->rss_key[i] = cpu_to_be32(rsskey[i]); 1005 1006 err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context, 1007 &rss_map->indir_qp, &rss_map->indir_state); 1008 if (err) 1009 goto indir_err; 1010 1011 return 0; 1012 1013 indir_err: 1014 mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, 1015 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); 1016 mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); 1017 mlx4_qp_free(mdev->dev, &rss_map->indir_qp); 1018 rss_err: 1019 for (i = 0; i < good_qps; i++) { 1020 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], 1021 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); 1022 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); 1023 mlx4_qp_free(mdev->dev, &rss_map->qps[i]); 1024 } 1025 mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); 1026 return err; 1027 } 1028 1029 void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv) 1030 { 1031 struct mlx4_en_dev *mdev = priv->mdev; 1032 struct mlx4_en_rss_map *rss_map = &priv->rss_map; 1033 int i; 1034 1035 mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, 1036 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); 1037 mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); 1038 mlx4_qp_free(mdev->dev, &rss_map->indir_qp); 1039 1040 for (i = 0; i < priv->rx_ring_num; i++) { 1041 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], 1042 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); 1043 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); 1044 mlx4_qp_free(mdev->dev, &rss_map->qps[i]); 1045 } 1046 mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); 1047 } 1048