1 /* 2 * Linux driver for VMware's vmxnet3 ethernet NIC. 3 * 4 * Copyright (C) 2008-2022, VMware, Inc. All Rights Reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the 8 * Free Software Foundation; version 2 of the License and no later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 13 * NON INFRINGEMENT. See the GNU General Public License for more 14 * details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * The full GNU General Public License is included in this distribution in 21 * the file called "COPYING". 22 * 23 * Maintained by: pv-drivers@vmware.com 24 * 25 */ 26 27 #include <linux/module.h> 28 #include <net/ip6_checksum.h> 29 30 #include "vmxnet3_int.h" 31 32 char vmxnet3_driver_name[] = "vmxnet3"; 33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver" 34 35 /* 36 * PCI Device ID Table 37 * Last entry must be all 0s 38 */ 39 static const struct pci_device_id vmxnet3_pciid_table[] = { 40 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)}, 41 {0} 42 }; 43 44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table); 45 46 static int enable_mq = 1; 47 48 static void 49 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac); 50 51 /* 52 * Enable/Disable the given intr 53 */ 54 static void 55 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 56 { 57 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0); 58 } 59 60 61 static void 62 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 63 { 64 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1); 65 } 66 67 68 /* 69 * Enable/Disable all intrs used by the device 70 */ 71 static void 72 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter) 73 { 74 int i; 75 76 for (i = 0; i < adapter->intr.num_intrs; i++) 77 vmxnet3_enable_intr(adapter, i); 78 if (!VMXNET3_VERSION_GE_6(adapter) || 79 !adapter->queuesExtEnabled) { 80 adapter->shared->devRead.intrConf.intrCtrl &= 81 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 82 } else { 83 adapter->shared->devReadExt.intrConfExt.intrCtrl &= 84 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 85 } 86 } 87 88 89 static void 90 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter) 91 { 92 int i; 93 94 if (!VMXNET3_VERSION_GE_6(adapter) || 95 !adapter->queuesExtEnabled) { 96 adapter->shared->devRead.intrConf.intrCtrl |= 97 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 98 } else { 99 adapter->shared->devReadExt.intrConfExt.intrCtrl |= 100 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 101 } 102 for (i = 0; i < adapter->intr.num_intrs; i++) 103 vmxnet3_disable_intr(adapter, i); 104 } 105 106 107 static void 108 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events) 109 { 110 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events); 111 } 112 113 114 static bool 115 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 116 { 117 return tq->stopped; 118 } 119 120 121 static void 122 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 123 { 124 tq->stopped = false; 125 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue); 126 } 127 128 129 static void 130 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 131 { 132 tq->stopped = false; 133 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 134 } 135 136 137 static void 138 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 139 { 140 tq->stopped = true; 141 tq->num_stop++; 142 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 143 } 144 145 /* Check if capability is supported by UPT device or 146 * UPT is even requested 147 */ 148 bool 149 vmxnet3_check_ptcapability(u32 cap_supported, u32 cap) 150 { 151 if (cap_supported & (1UL << VMXNET3_DCR_ERROR) || 152 cap_supported & (1UL << cap)) { 153 return true; 154 } 155 156 return false; 157 } 158 159 160 /* 161 * Check the link state. This may start or stop the tx queue. 162 */ 163 static void 164 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue) 165 { 166 u32 ret; 167 int i; 168 unsigned long flags; 169 170 spin_lock_irqsave(&adapter->cmd_lock, flags); 171 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 172 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 173 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 174 175 adapter->link_speed = ret >> 16; 176 if (ret & 1) { /* Link is up. */ 177 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n", 178 adapter->link_speed); 179 netif_carrier_on(adapter->netdev); 180 181 if (affectTxQueue) { 182 for (i = 0; i < adapter->num_tx_queues; i++) 183 vmxnet3_tq_start(&adapter->tx_queue[i], 184 adapter); 185 } 186 } else { 187 netdev_info(adapter->netdev, "NIC Link is Down\n"); 188 netif_carrier_off(adapter->netdev); 189 190 if (affectTxQueue) { 191 for (i = 0; i < adapter->num_tx_queues; i++) 192 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter); 193 } 194 } 195 } 196 197 static void 198 vmxnet3_process_events(struct vmxnet3_adapter *adapter) 199 { 200 int i; 201 unsigned long flags; 202 u32 events = le32_to_cpu(adapter->shared->ecr); 203 if (!events) 204 return; 205 206 vmxnet3_ack_events(adapter, events); 207 208 /* Check if link state has changed */ 209 if (events & VMXNET3_ECR_LINK) 210 vmxnet3_check_link(adapter, true); 211 212 /* Check if there is an error on xmit/recv queues */ 213 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { 214 spin_lock_irqsave(&adapter->cmd_lock, flags); 215 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 216 VMXNET3_CMD_GET_QUEUE_STATUS); 217 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 218 219 for (i = 0; i < adapter->num_tx_queues; i++) 220 if (adapter->tqd_start[i].status.stopped) 221 dev_err(&adapter->netdev->dev, 222 "%s: tq[%d] error 0x%x\n", 223 adapter->netdev->name, i, le32_to_cpu( 224 adapter->tqd_start[i].status.error)); 225 for (i = 0; i < adapter->num_rx_queues; i++) 226 if (adapter->rqd_start[i].status.stopped) 227 dev_err(&adapter->netdev->dev, 228 "%s: rq[%d] error 0x%x\n", 229 adapter->netdev->name, i, 230 adapter->rqd_start[i].status.error); 231 232 schedule_work(&adapter->work); 233 } 234 } 235 236 #ifdef __BIG_ENDIAN_BITFIELD 237 /* 238 * The device expects the bitfields in shared structures to be written in 239 * little endian. When CPU is big endian, the following routines are used to 240 * correctly read and write into ABI. 241 * The general technique used here is : double word bitfields are defined in 242 * opposite order for big endian architecture. Then before reading them in 243 * driver the complete double word is translated using le32_to_cpu. Similarly 244 * After the driver writes into bitfields, cpu_to_le32 is used to translate the 245 * double words into required format. 246 * In order to avoid touching bits in shared structure more than once, temporary 247 * descriptors are used. These are passed as srcDesc to following functions. 248 */ 249 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc, 250 struct Vmxnet3_RxDesc *dstDesc) 251 { 252 u32 *src = (u32 *)srcDesc + 2; 253 u32 *dst = (u32 *)dstDesc + 2; 254 dstDesc->addr = le64_to_cpu(srcDesc->addr); 255 *dst = le32_to_cpu(*src); 256 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1); 257 } 258 259 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc, 260 struct Vmxnet3_TxDesc *dstDesc) 261 { 262 int i; 263 u32 *src = (u32 *)(srcDesc + 1); 264 u32 *dst = (u32 *)(dstDesc + 1); 265 266 /* Working backwards so that the gen bit is set at the end. */ 267 for (i = 2; i > 0; i--) { 268 src--; 269 dst--; 270 *dst = cpu_to_le32(*src); 271 } 272 } 273 274 275 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc, 276 struct Vmxnet3_RxCompDesc *dstDesc) 277 { 278 int i = 0; 279 u32 *src = (u32 *)srcDesc; 280 u32 *dst = (u32 *)dstDesc; 281 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) { 282 *dst = le32_to_cpu(*src); 283 src++; 284 dst++; 285 } 286 } 287 288 289 /* Used to read bitfield values from double words. */ 290 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size) 291 { 292 u32 temp = le32_to_cpu(*bitfield); 293 u32 mask = ((1 << size) - 1) << pos; 294 temp &= mask; 295 temp >>= pos; 296 return temp; 297 } 298 299 300 301 #endif /* __BIG_ENDIAN_BITFIELD */ 302 303 #ifdef __BIG_ENDIAN_BITFIELD 304 305 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \ 306 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \ 307 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE) 308 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \ 309 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \ 310 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE) 311 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \ 312 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \ 313 VMXNET3_TCD_GEN_SIZE) 314 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \ 315 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE) 316 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \ 317 (dstrcd) = (tmp); \ 318 vmxnet3_RxCompToCPU((rcd), (tmp)); \ 319 } while (0) 320 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \ 321 (dstrxd) = (tmp); \ 322 vmxnet3_RxDescToCPU((rxd), (tmp)); \ 323 } while (0) 324 325 #else 326 327 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen) 328 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop) 329 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen) 330 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx) 331 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd) 332 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd) 333 334 #endif /* __BIG_ENDIAN_BITFIELD */ 335 336 337 static void 338 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi, 339 struct pci_dev *pdev) 340 { 341 if (tbi->map_type == VMXNET3_MAP_SINGLE) 342 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len, 343 DMA_TO_DEVICE); 344 else if (tbi->map_type == VMXNET3_MAP_PAGE) 345 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len, 346 DMA_TO_DEVICE); 347 else 348 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE); 349 350 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */ 351 } 352 353 354 static int 355 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq, 356 struct pci_dev *pdev, struct vmxnet3_adapter *adapter) 357 { 358 struct sk_buff *skb; 359 int entries = 0; 360 361 /* no out of order completion */ 362 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp); 363 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1); 364 365 skb = tq->buf_info[eop_idx].skb; 366 BUG_ON(skb == NULL); 367 tq->buf_info[eop_idx].skb = NULL; 368 369 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size); 370 371 while (tq->tx_ring.next2comp != eop_idx) { 372 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp, 373 pdev); 374 375 /* update next2comp w/o tx_lock. Since we are marking more, 376 * instead of less, tx ring entries avail, the worst case is 377 * that the tx routine incorrectly re-queues a pkt due to 378 * insufficient tx ring entries. 379 */ 380 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 381 entries++; 382 } 383 384 dev_kfree_skb_any(skb); 385 return entries; 386 } 387 388 389 static int 390 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq, 391 struct vmxnet3_adapter *adapter) 392 { 393 int completed = 0; 394 union Vmxnet3_GenericDesc *gdesc; 395 396 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 397 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) { 398 /* Prevent any &gdesc->tcd field from being (speculatively) 399 * read before (&gdesc->tcd)->gen is read. 400 */ 401 dma_rmb(); 402 403 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX( 404 &gdesc->tcd), tq, adapter->pdev, 405 adapter); 406 407 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring); 408 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 409 } 410 411 if (completed) { 412 spin_lock(&tq->tx_lock); 413 if (unlikely(vmxnet3_tq_stopped(tq, adapter) && 414 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) > 415 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) && 416 netif_carrier_ok(adapter->netdev))) { 417 vmxnet3_tq_wake(tq, adapter); 418 } 419 spin_unlock(&tq->tx_lock); 420 } 421 return completed; 422 } 423 424 425 static void 426 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq, 427 struct vmxnet3_adapter *adapter) 428 { 429 int i; 430 431 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) { 432 struct vmxnet3_tx_buf_info *tbi; 433 434 tbi = tq->buf_info + tq->tx_ring.next2comp; 435 436 vmxnet3_unmap_tx_buf(tbi, adapter->pdev); 437 if (tbi->skb) { 438 dev_kfree_skb_any(tbi->skb); 439 tbi->skb = NULL; 440 } 441 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 442 } 443 444 /* sanity check, verify all buffers are indeed unmapped and freed */ 445 for (i = 0; i < tq->tx_ring.size; i++) { 446 BUG_ON(tq->buf_info[i].skb != NULL || 447 tq->buf_info[i].map_type != VMXNET3_MAP_NONE); 448 } 449 450 tq->tx_ring.gen = VMXNET3_INIT_GEN; 451 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 452 453 tq->comp_ring.gen = VMXNET3_INIT_GEN; 454 tq->comp_ring.next2proc = 0; 455 } 456 457 458 static void 459 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, 460 struct vmxnet3_adapter *adapter) 461 { 462 if (tq->tx_ring.base) { 463 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size * 464 sizeof(struct Vmxnet3_TxDesc), 465 tq->tx_ring.base, tq->tx_ring.basePA); 466 tq->tx_ring.base = NULL; 467 } 468 if (tq->data_ring.base) { 469 dma_free_coherent(&adapter->pdev->dev, 470 tq->data_ring.size * tq->txdata_desc_size, 471 tq->data_ring.base, tq->data_ring.basePA); 472 tq->data_ring.base = NULL; 473 } 474 if (tq->comp_ring.base) { 475 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size * 476 sizeof(struct Vmxnet3_TxCompDesc), 477 tq->comp_ring.base, tq->comp_ring.basePA); 478 tq->comp_ring.base = NULL; 479 } 480 kfree(tq->buf_info); 481 tq->buf_info = NULL; 482 } 483 484 485 /* Destroy all tx queues */ 486 void 487 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter) 488 { 489 int i; 490 491 for (i = 0; i < adapter->num_tx_queues; i++) 492 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter); 493 } 494 495 496 static void 497 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq, 498 struct vmxnet3_adapter *adapter) 499 { 500 int i; 501 502 /* reset the tx ring contents to 0 and reset the tx ring states */ 503 memset(tq->tx_ring.base, 0, tq->tx_ring.size * 504 sizeof(struct Vmxnet3_TxDesc)); 505 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 506 tq->tx_ring.gen = VMXNET3_INIT_GEN; 507 508 memset(tq->data_ring.base, 0, 509 tq->data_ring.size * tq->txdata_desc_size); 510 511 /* reset the tx comp ring contents to 0 and reset comp ring states */ 512 memset(tq->comp_ring.base, 0, tq->comp_ring.size * 513 sizeof(struct Vmxnet3_TxCompDesc)); 514 tq->comp_ring.next2proc = 0; 515 tq->comp_ring.gen = VMXNET3_INIT_GEN; 516 517 /* reset the bookkeeping data */ 518 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size); 519 for (i = 0; i < tq->tx_ring.size; i++) 520 tq->buf_info[i].map_type = VMXNET3_MAP_NONE; 521 522 /* stats are not reset */ 523 } 524 525 526 static int 527 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq, 528 struct vmxnet3_adapter *adapter) 529 { 530 BUG_ON(tq->tx_ring.base || tq->data_ring.base || 531 tq->comp_ring.base || tq->buf_info); 532 533 tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 534 tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc), 535 &tq->tx_ring.basePA, GFP_KERNEL); 536 if (!tq->tx_ring.base) { 537 netdev_err(adapter->netdev, "failed to allocate tx ring\n"); 538 goto err; 539 } 540 541 tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 542 tq->data_ring.size * tq->txdata_desc_size, 543 &tq->data_ring.basePA, GFP_KERNEL); 544 if (!tq->data_ring.base) { 545 netdev_err(adapter->netdev, "failed to allocate tx data ring\n"); 546 goto err; 547 } 548 549 tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 550 tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc), 551 &tq->comp_ring.basePA, GFP_KERNEL); 552 if (!tq->comp_ring.base) { 553 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n"); 554 goto err; 555 } 556 557 tq->buf_info = kcalloc_node(tq->tx_ring.size, sizeof(tq->buf_info[0]), 558 GFP_KERNEL, 559 dev_to_node(&adapter->pdev->dev)); 560 if (!tq->buf_info) 561 goto err; 562 563 return 0; 564 565 err: 566 vmxnet3_tq_destroy(tq, adapter); 567 return -ENOMEM; 568 } 569 570 static void 571 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter) 572 { 573 int i; 574 575 for (i = 0; i < adapter->num_tx_queues; i++) 576 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter); 577 } 578 579 /* 580 * starting from ring->next2fill, allocate rx buffers for the given ring 581 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers 582 * are allocated or allocation fails 583 */ 584 585 static int 586 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx, 587 int num_to_alloc, struct vmxnet3_adapter *adapter) 588 { 589 int num_allocated = 0; 590 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx]; 591 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx]; 592 u32 val; 593 594 while (num_allocated <= num_to_alloc) { 595 struct vmxnet3_rx_buf_info *rbi; 596 union Vmxnet3_GenericDesc *gd; 597 598 rbi = rbi_base + ring->next2fill; 599 gd = ring->base + ring->next2fill; 600 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 601 602 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) { 603 if (rbi->skb == NULL) { 604 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev, 605 rbi->len, 606 GFP_KERNEL); 607 if (unlikely(rbi->skb == NULL)) { 608 rq->stats.rx_buf_alloc_failure++; 609 break; 610 } 611 612 rbi->dma_addr = dma_map_single( 613 &adapter->pdev->dev, 614 rbi->skb->data, rbi->len, 615 DMA_FROM_DEVICE); 616 if (dma_mapping_error(&adapter->pdev->dev, 617 rbi->dma_addr)) { 618 dev_kfree_skb_any(rbi->skb); 619 rbi->skb = NULL; 620 rq->stats.rx_buf_alloc_failure++; 621 break; 622 } 623 } else { 624 /* rx buffer skipped by the device */ 625 } 626 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; 627 } else { 628 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE || 629 rbi->len != PAGE_SIZE); 630 631 if (rbi->page == NULL) { 632 rbi->page = alloc_page(GFP_ATOMIC); 633 if (unlikely(rbi->page == NULL)) { 634 rq->stats.rx_buf_alloc_failure++; 635 break; 636 } 637 rbi->dma_addr = dma_map_page( 638 &adapter->pdev->dev, 639 rbi->page, 0, PAGE_SIZE, 640 DMA_FROM_DEVICE); 641 if (dma_mapping_error(&adapter->pdev->dev, 642 rbi->dma_addr)) { 643 put_page(rbi->page); 644 rbi->page = NULL; 645 rq->stats.rx_buf_alloc_failure++; 646 break; 647 } 648 } else { 649 /* rx buffers skipped by the device */ 650 } 651 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT; 652 } 653 654 gd->rxd.addr = cpu_to_le64(rbi->dma_addr); 655 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT) 656 | val | rbi->len); 657 658 /* Fill the last buffer but dont mark it ready, or else the 659 * device will think that the queue is full */ 660 if (num_allocated == num_to_alloc) { 661 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 662 break; 663 } 664 665 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT); 666 num_allocated++; 667 vmxnet3_cmd_ring_adv_next2fill(ring); 668 } 669 670 netdev_dbg(adapter->netdev, 671 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n", 672 num_allocated, ring->next2fill, ring->next2comp); 673 674 /* so that the device can distinguish a full ring and an empty ring */ 675 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp); 676 677 return num_allocated; 678 } 679 680 681 static void 682 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd, 683 struct vmxnet3_rx_buf_info *rbi) 684 { 685 skb_frag_t *frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags; 686 687 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS); 688 689 __skb_frag_set_page(frag, rbi->page); 690 skb_frag_off_set(frag, 0); 691 skb_frag_size_set(frag, rcd->len); 692 skb->data_len += rcd->len; 693 skb->truesize += PAGE_SIZE; 694 skb_shinfo(skb)->nr_frags++; 695 } 696 697 698 static int 699 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx, 700 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev, 701 struct vmxnet3_adapter *adapter) 702 { 703 u32 dw2, len; 704 unsigned long buf_offset; 705 int i; 706 union Vmxnet3_GenericDesc *gdesc; 707 struct vmxnet3_tx_buf_info *tbi = NULL; 708 709 BUG_ON(ctx->copy_size > skb_headlen(skb)); 710 711 /* use the previous gen bit for the SOP desc */ 712 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; 713 714 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill; 715 gdesc = ctx->sop_txd; /* both loops below can be skipped */ 716 717 /* no need to map the buffer if headers are copied */ 718 if (ctx->copy_size) { 719 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA + 720 tq->tx_ring.next2fill * 721 tq->txdata_desc_size); 722 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size); 723 ctx->sop_txd->dword[3] = 0; 724 725 tbi = tq->buf_info + tq->tx_ring.next2fill; 726 tbi->map_type = VMXNET3_MAP_NONE; 727 728 netdev_dbg(adapter->netdev, 729 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 730 tq->tx_ring.next2fill, 731 le64_to_cpu(ctx->sop_txd->txd.addr), 732 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]); 733 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 734 735 /* use the right gen for non-SOP desc */ 736 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 737 } 738 739 /* linear part can use multiple tx desc if it's big */ 740 len = skb_headlen(skb) - ctx->copy_size; 741 buf_offset = ctx->copy_size; 742 while (len) { 743 u32 buf_size; 744 745 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 746 buf_size = len; 747 dw2 |= len; 748 } else { 749 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 750 /* spec says that for TxDesc.len, 0 == 2^14 */ 751 } 752 753 tbi = tq->buf_info + tq->tx_ring.next2fill; 754 tbi->map_type = VMXNET3_MAP_SINGLE; 755 tbi->dma_addr = dma_map_single(&adapter->pdev->dev, 756 skb->data + buf_offset, buf_size, 757 DMA_TO_DEVICE); 758 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 759 return -EFAULT; 760 761 tbi->len = buf_size; 762 763 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 764 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 765 766 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 767 gdesc->dword[2] = cpu_to_le32(dw2); 768 gdesc->dword[3] = 0; 769 770 netdev_dbg(adapter->netdev, 771 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 772 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 773 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 774 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 775 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 776 777 len -= buf_size; 778 buf_offset += buf_size; 779 } 780 781 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 782 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 783 u32 buf_size; 784 785 buf_offset = 0; 786 len = skb_frag_size(frag); 787 while (len) { 788 tbi = tq->buf_info + tq->tx_ring.next2fill; 789 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 790 buf_size = len; 791 dw2 |= len; 792 } else { 793 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 794 /* spec says that for TxDesc.len, 0 == 2^14 */ 795 } 796 tbi->map_type = VMXNET3_MAP_PAGE; 797 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag, 798 buf_offset, buf_size, 799 DMA_TO_DEVICE); 800 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 801 return -EFAULT; 802 803 tbi->len = buf_size; 804 805 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 806 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 807 808 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 809 gdesc->dword[2] = cpu_to_le32(dw2); 810 gdesc->dword[3] = 0; 811 812 netdev_dbg(adapter->netdev, 813 "txd[%u]: 0x%llx %u %u\n", 814 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 815 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 816 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 817 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 818 819 len -= buf_size; 820 buf_offset += buf_size; 821 } 822 } 823 824 ctx->eop_txd = gdesc; 825 826 /* set the last buf_info for the pkt */ 827 tbi->skb = skb; 828 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base; 829 830 return 0; 831 } 832 833 834 /* Init all tx queues */ 835 static void 836 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter) 837 { 838 int i; 839 840 for (i = 0; i < adapter->num_tx_queues; i++) 841 vmxnet3_tq_init(&adapter->tx_queue[i], adapter); 842 } 843 844 845 /* 846 * parse relevant protocol headers: 847 * For a tso pkt, relevant headers are L2/3/4 including options 848 * For a pkt requesting csum offloading, they are L2/3 and may include L4 849 * if it's a TCP/UDP pkt 850 * 851 * Returns: 852 * -1: error happens during parsing 853 * 0: protocol headers parsed, but too big to be copied 854 * 1: protocol headers parsed and copied 855 * 856 * Other effects: 857 * 1. related *ctx fields are updated. 858 * 2. ctx->copy_size is # of bytes copied 859 * 3. the portion to be copied is guaranteed to be in the linear part 860 * 861 */ 862 static int 863 vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 864 struct vmxnet3_tx_ctx *ctx, 865 struct vmxnet3_adapter *adapter) 866 { 867 u8 protocol = 0; 868 869 if (ctx->mss) { /* TSO */ 870 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 871 ctx->l4_offset = skb_inner_transport_offset(skb); 872 ctx->l4_hdr_size = inner_tcp_hdrlen(skb); 873 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 874 } else { 875 ctx->l4_offset = skb_transport_offset(skb); 876 ctx->l4_hdr_size = tcp_hdrlen(skb); 877 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 878 } 879 } else { 880 if (skb->ip_summed == CHECKSUM_PARTIAL) { 881 /* For encap packets, skb_checksum_start_offset refers 882 * to inner L4 offset. Thus, below works for encap as 883 * well as non-encap case 884 */ 885 ctx->l4_offset = skb_checksum_start_offset(skb); 886 887 if (VMXNET3_VERSION_GE_4(adapter) && 888 skb->encapsulation) { 889 struct iphdr *iph = inner_ip_hdr(skb); 890 891 if (iph->version == 4) { 892 protocol = iph->protocol; 893 } else { 894 const struct ipv6hdr *ipv6h; 895 896 ipv6h = inner_ipv6_hdr(skb); 897 protocol = ipv6h->nexthdr; 898 } 899 } else { 900 if (ctx->ipv4) { 901 const struct iphdr *iph = ip_hdr(skb); 902 903 protocol = iph->protocol; 904 } else if (ctx->ipv6) { 905 const struct ipv6hdr *ipv6h; 906 907 ipv6h = ipv6_hdr(skb); 908 protocol = ipv6h->nexthdr; 909 } 910 } 911 912 switch (protocol) { 913 case IPPROTO_TCP: 914 ctx->l4_hdr_size = skb->encapsulation ? inner_tcp_hdrlen(skb) : 915 tcp_hdrlen(skb); 916 break; 917 case IPPROTO_UDP: 918 ctx->l4_hdr_size = sizeof(struct udphdr); 919 break; 920 default: 921 ctx->l4_hdr_size = 0; 922 break; 923 } 924 925 ctx->copy_size = min(ctx->l4_offset + 926 ctx->l4_hdr_size, skb->len); 927 } else { 928 ctx->l4_offset = 0; 929 ctx->l4_hdr_size = 0; 930 /* copy as much as allowed */ 931 ctx->copy_size = min_t(unsigned int, 932 tq->txdata_desc_size, 933 skb_headlen(skb)); 934 } 935 936 if (skb->len <= VMXNET3_HDR_COPY_SIZE) 937 ctx->copy_size = skb->len; 938 939 /* make sure headers are accessible directly */ 940 if (unlikely(!pskb_may_pull(skb, ctx->copy_size))) 941 goto err; 942 } 943 944 if (unlikely(ctx->copy_size > tq->txdata_desc_size)) { 945 tq->stats.oversized_hdr++; 946 ctx->copy_size = 0; 947 return 0; 948 } 949 950 return 1; 951 err: 952 return -1; 953 } 954 955 /* 956 * copy relevant protocol headers to the transmit ring: 957 * For a tso pkt, relevant headers are L2/3/4 including options 958 * For a pkt requesting csum offloading, they are L2/3 and may include L4 959 * if it's a TCP/UDP pkt 960 * 961 * 962 * Note that this requires that vmxnet3_parse_hdr be called first to set the 963 * appropriate bits in ctx first 964 */ 965 static void 966 vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 967 struct vmxnet3_tx_ctx *ctx, 968 struct vmxnet3_adapter *adapter) 969 { 970 struct Vmxnet3_TxDataDesc *tdd; 971 972 tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base + 973 tq->tx_ring.next2fill * 974 tq->txdata_desc_size); 975 976 memcpy(tdd->data, skb->data, ctx->copy_size); 977 netdev_dbg(adapter->netdev, 978 "copy %u bytes to dataRing[%u]\n", 979 ctx->copy_size, tq->tx_ring.next2fill); 980 } 981 982 983 static void 984 vmxnet3_prepare_inner_tso(struct sk_buff *skb, 985 struct vmxnet3_tx_ctx *ctx) 986 { 987 struct tcphdr *tcph = inner_tcp_hdr(skb); 988 struct iphdr *iph = inner_ip_hdr(skb); 989 990 if (iph->version == 4) { 991 iph->check = 0; 992 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 993 IPPROTO_TCP, 0); 994 } else { 995 struct ipv6hdr *iph = inner_ipv6_hdr(skb); 996 997 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0, 998 IPPROTO_TCP, 0); 999 } 1000 } 1001 1002 static void 1003 vmxnet3_prepare_tso(struct sk_buff *skb, 1004 struct vmxnet3_tx_ctx *ctx) 1005 { 1006 struct tcphdr *tcph = tcp_hdr(skb); 1007 1008 if (ctx->ipv4) { 1009 struct iphdr *iph = ip_hdr(skb); 1010 1011 iph->check = 0; 1012 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 1013 IPPROTO_TCP, 0); 1014 } else if (ctx->ipv6) { 1015 tcp_v6_gso_csum_prep(skb); 1016 } 1017 } 1018 1019 static int txd_estimate(const struct sk_buff *skb) 1020 { 1021 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1022 int i; 1023 1024 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1025 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1026 1027 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag)); 1028 } 1029 return count; 1030 } 1031 1032 /* 1033 * Transmits a pkt thru a given tq 1034 * Returns: 1035 * NETDEV_TX_OK: descriptors are setup successfully 1036 * NETDEV_TX_OK: error occurred, the pkt is dropped 1037 * NETDEV_TX_BUSY: tx ring is full, queue is stopped 1038 * 1039 * Side-effects: 1040 * 1. tx ring may be changed 1041 * 2. tq stats may be updated accordingly 1042 * 3. shared->txNumDeferred may be updated 1043 */ 1044 1045 static int 1046 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 1047 struct vmxnet3_adapter *adapter, struct net_device *netdev) 1048 { 1049 int ret; 1050 u32 count; 1051 int num_pkts; 1052 int tx_num_deferred; 1053 unsigned long flags; 1054 struct vmxnet3_tx_ctx ctx; 1055 union Vmxnet3_GenericDesc *gdesc; 1056 #ifdef __BIG_ENDIAN_BITFIELD 1057 /* Use temporary descriptor to avoid touching bits multiple times */ 1058 union Vmxnet3_GenericDesc tempTxDesc; 1059 #endif 1060 1061 count = txd_estimate(skb); 1062 1063 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP)); 1064 ctx.ipv6 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IPV6)); 1065 1066 ctx.mss = skb_shinfo(skb)->gso_size; 1067 if (ctx.mss) { 1068 if (skb_header_cloned(skb)) { 1069 if (unlikely(pskb_expand_head(skb, 0, 0, 1070 GFP_ATOMIC) != 0)) { 1071 tq->stats.drop_tso++; 1072 goto drop_pkt; 1073 } 1074 tq->stats.copy_skb_header++; 1075 } 1076 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1077 /* tso pkts must not use more than 1078 * VMXNET3_MAX_TSO_TXD_PER_PKT entries 1079 */ 1080 if (skb_linearize(skb) != 0) { 1081 tq->stats.drop_too_many_frags++; 1082 goto drop_pkt; 1083 } 1084 tq->stats.linearized++; 1085 1086 /* recalculate the # of descriptors to use */ 1087 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1088 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1089 tq->stats.drop_too_many_frags++; 1090 goto drop_pkt; 1091 } 1092 } 1093 if (skb->encapsulation) { 1094 vmxnet3_prepare_inner_tso(skb, &ctx); 1095 } else { 1096 vmxnet3_prepare_tso(skb, &ctx); 1097 } 1098 } else { 1099 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) { 1100 1101 /* non-tso pkts must not use more than 1102 * VMXNET3_MAX_TXD_PER_PKT entries 1103 */ 1104 if (skb_linearize(skb) != 0) { 1105 tq->stats.drop_too_many_frags++; 1106 goto drop_pkt; 1107 } 1108 tq->stats.linearized++; 1109 1110 /* recalculate the # of descriptors to use */ 1111 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1112 } 1113 } 1114 1115 ret = vmxnet3_parse_hdr(skb, tq, &ctx, adapter); 1116 if (ret >= 0) { 1117 BUG_ON(ret <= 0 && ctx.copy_size != 0); 1118 /* hdrs parsed, check against other limits */ 1119 if (ctx.mss) { 1120 if (unlikely(ctx.l4_offset + ctx.l4_hdr_size > 1121 VMXNET3_MAX_TX_BUF_SIZE)) { 1122 tq->stats.drop_oversized_hdr++; 1123 goto drop_pkt; 1124 } 1125 } else { 1126 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1127 if (unlikely(ctx.l4_offset + 1128 skb->csum_offset > 1129 VMXNET3_MAX_CSUM_OFFSET)) { 1130 tq->stats.drop_oversized_hdr++; 1131 goto drop_pkt; 1132 } 1133 } 1134 } 1135 } else { 1136 tq->stats.drop_hdr_inspect_err++; 1137 goto drop_pkt; 1138 } 1139 1140 spin_lock_irqsave(&tq->tx_lock, flags); 1141 1142 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) { 1143 tq->stats.tx_ring_full++; 1144 netdev_dbg(adapter->netdev, 1145 "tx queue stopped on %s, next2comp %u" 1146 " next2fill %u\n", adapter->netdev->name, 1147 tq->tx_ring.next2comp, tq->tx_ring.next2fill); 1148 1149 vmxnet3_tq_stop(tq, adapter); 1150 spin_unlock_irqrestore(&tq->tx_lock, flags); 1151 return NETDEV_TX_BUSY; 1152 } 1153 1154 1155 vmxnet3_copy_hdr(skb, tq, &ctx, adapter); 1156 1157 /* fill tx descs related to addr & len */ 1158 if (vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter)) 1159 goto unlock_drop_pkt; 1160 1161 /* setup the EOP desc */ 1162 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP); 1163 1164 /* setup the SOP desc */ 1165 #ifdef __BIG_ENDIAN_BITFIELD 1166 gdesc = &tempTxDesc; 1167 gdesc->dword[2] = ctx.sop_txd->dword[2]; 1168 gdesc->dword[3] = ctx.sop_txd->dword[3]; 1169 #else 1170 gdesc = ctx.sop_txd; 1171 #endif 1172 tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred); 1173 if (ctx.mss) { 1174 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 1175 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1176 if (VMXNET3_VERSION_GE_7(adapter)) { 1177 gdesc->txd.om = VMXNET3_OM_TSO; 1178 gdesc->txd.ext1 = 1; 1179 } else { 1180 gdesc->txd.om = VMXNET3_OM_ENCAP; 1181 } 1182 gdesc->txd.msscof = ctx.mss; 1183 1184 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM) 1185 gdesc->txd.oco = 1; 1186 } else { 1187 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1188 gdesc->txd.om = VMXNET3_OM_TSO; 1189 gdesc->txd.msscof = ctx.mss; 1190 } 1191 num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss; 1192 } else { 1193 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1194 if (VMXNET3_VERSION_GE_4(adapter) && 1195 skb->encapsulation) { 1196 gdesc->txd.hlen = ctx.l4_offset + 1197 ctx.l4_hdr_size; 1198 if (VMXNET3_VERSION_GE_7(adapter)) { 1199 gdesc->txd.om = VMXNET3_OM_CSUM; 1200 gdesc->txd.msscof = ctx.l4_offset + 1201 skb->csum_offset; 1202 gdesc->txd.ext1 = 1; 1203 } else { 1204 gdesc->txd.om = VMXNET3_OM_ENCAP; 1205 gdesc->txd.msscof = 0; /* Reserved */ 1206 } 1207 } else { 1208 gdesc->txd.hlen = ctx.l4_offset; 1209 gdesc->txd.om = VMXNET3_OM_CSUM; 1210 gdesc->txd.msscof = ctx.l4_offset + 1211 skb->csum_offset; 1212 } 1213 } else { 1214 gdesc->txd.om = 0; 1215 gdesc->txd.msscof = 0; 1216 } 1217 num_pkts = 1; 1218 } 1219 le32_add_cpu(&tq->shared->txNumDeferred, num_pkts); 1220 tx_num_deferred += num_pkts; 1221 1222 if (skb_vlan_tag_present(skb)) { 1223 gdesc->txd.ti = 1; 1224 gdesc->txd.tci = skb_vlan_tag_get(skb); 1225 } 1226 1227 /* Ensure that the write to (&gdesc->txd)->gen will be observed after 1228 * all other writes to &gdesc->txd. 1229 */ 1230 dma_wmb(); 1231 1232 /* finally flips the GEN bit of the SOP desc. */ 1233 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^ 1234 VMXNET3_TXD_GEN); 1235 #ifdef __BIG_ENDIAN_BITFIELD 1236 /* Finished updating in bitfields of Tx Desc, so write them in original 1237 * place. 1238 */ 1239 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc, 1240 (struct Vmxnet3_TxDesc *)ctx.sop_txd); 1241 gdesc = ctx.sop_txd; 1242 #endif 1243 netdev_dbg(adapter->netdev, 1244 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n", 1245 (u32)(ctx.sop_txd - 1246 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr), 1247 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3])); 1248 1249 spin_unlock_irqrestore(&tq->tx_lock, flags); 1250 1251 if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) { 1252 tq->shared->txNumDeferred = 0; 1253 VMXNET3_WRITE_BAR0_REG(adapter, 1254 adapter->tx_prod_offset + tq->qid * 8, 1255 tq->tx_ring.next2fill); 1256 } 1257 1258 return NETDEV_TX_OK; 1259 1260 unlock_drop_pkt: 1261 spin_unlock_irqrestore(&tq->tx_lock, flags); 1262 drop_pkt: 1263 tq->stats.drop_total++; 1264 dev_kfree_skb_any(skb); 1265 return NETDEV_TX_OK; 1266 } 1267 1268 1269 static netdev_tx_t 1270 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 1271 { 1272 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1273 1274 BUG_ON(skb->queue_mapping > adapter->num_tx_queues); 1275 return vmxnet3_tq_xmit(skb, 1276 &adapter->tx_queue[skb->queue_mapping], 1277 adapter, netdev); 1278 } 1279 1280 1281 static void 1282 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter, 1283 struct sk_buff *skb, 1284 union Vmxnet3_GenericDesc *gdesc) 1285 { 1286 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) { 1287 if (gdesc->rcd.v4 && 1288 (le32_to_cpu(gdesc->dword[3]) & 1289 VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) { 1290 skb->ip_summed = CHECKSUM_UNNECESSARY; 1291 if ((le32_to_cpu(gdesc->dword[0]) & 1292 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) { 1293 skb->csum_level = 1; 1294 } 1295 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1296 !(le32_to_cpu(gdesc->dword[0]) & 1297 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1298 WARN_ON_ONCE(gdesc->rcd.frg && 1299 !(le32_to_cpu(gdesc->dword[0]) & 1300 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1301 } else if (gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) & 1302 (1 << VMXNET3_RCD_TUC_SHIFT))) { 1303 skb->ip_summed = CHECKSUM_UNNECESSARY; 1304 if ((le32_to_cpu(gdesc->dword[0]) & 1305 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) { 1306 skb->csum_level = 1; 1307 } 1308 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1309 !(le32_to_cpu(gdesc->dword[0]) & 1310 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1311 WARN_ON_ONCE(gdesc->rcd.frg && 1312 !(le32_to_cpu(gdesc->dword[0]) & 1313 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1314 } else { 1315 if (gdesc->rcd.csum) { 1316 skb->csum = htons(gdesc->rcd.csum); 1317 skb->ip_summed = CHECKSUM_PARTIAL; 1318 } else { 1319 skb_checksum_none_assert(skb); 1320 } 1321 } 1322 } else { 1323 skb_checksum_none_assert(skb); 1324 } 1325 } 1326 1327 1328 static void 1329 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd, 1330 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter) 1331 { 1332 rq->stats.drop_err++; 1333 if (!rcd->fcs) 1334 rq->stats.drop_fcs++; 1335 1336 rq->stats.drop_total++; 1337 1338 /* 1339 * We do not unmap and chain the rx buffer to the skb. 1340 * We basically pretend this buffer is not used and will be recycled 1341 * by vmxnet3_rq_alloc_rx_buf() 1342 */ 1343 1344 /* 1345 * ctx->skb may be NULL if this is the first and the only one 1346 * desc for the pkt 1347 */ 1348 if (ctx->skb) 1349 dev_kfree_skb_irq(ctx->skb); 1350 1351 ctx->skb = NULL; 1352 } 1353 1354 1355 static u32 1356 vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb, 1357 union Vmxnet3_GenericDesc *gdesc) 1358 { 1359 u32 hlen, maplen; 1360 union { 1361 void *ptr; 1362 struct ethhdr *eth; 1363 struct vlan_ethhdr *veth; 1364 struct iphdr *ipv4; 1365 struct ipv6hdr *ipv6; 1366 struct tcphdr *tcp; 1367 } hdr; 1368 BUG_ON(gdesc->rcd.tcp == 0); 1369 1370 maplen = skb_headlen(skb); 1371 if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen)) 1372 return 0; 1373 1374 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 1375 skb->protocol == cpu_to_be16(ETH_P_8021AD)) 1376 hlen = sizeof(struct vlan_ethhdr); 1377 else 1378 hlen = sizeof(struct ethhdr); 1379 1380 hdr.eth = eth_hdr(skb); 1381 if (gdesc->rcd.v4) { 1382 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) && 1383 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP)); 1384 hdr.ptr += hlen; 1385 BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP); 1386 hlen = hdr.ipv4->ihl << 2; 1387 hdr.ptr += hdr.ipv4->ihl << 2; 1388 } else if (gdesc->rcd.v6) { 1389 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) && 1390 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6)); 1391 hdr.ptr += hlen; 1392 /* Use an estimated value, since we also need to handle 1393 * TSO case. 1394 */ 1395 if (hdr.ipv6->nexthdr != IPPROTO_TCP) 1396 return sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 1397 hlen = sizeof(struct ipv6hdr); 1398 hdr.ptr += sizeof(struct ipv6hdr); 1399 } else { 1400 /* Non-IP pkt, dont estimate header length */ 1401 return 0; 1402 } 1403 1404 if (hlen + sizeof(struct tcphdr) > maplen) 1405 return 0; 1406 1407 return (hlen + (hdr.tcp->doff << 2)); 1408 } 1409 1410 static int 1411 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, 1412 struct vmxnet3_adapter *adapter, int quota) 1413 { 1414 u32 rxprod_reg[2] = { 1415 adapter->rx_prod_offset, adapter->rx_prod2_offset 1416 }; 1417 u32 num_pkts = 0; 1418 bool skip_page_frags = false; 1419 bool encap_lro = false; 1420 struct Vmxnet3_RxCompDesc *rcd; 1421 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx; 1422 u16 segCnt = 0, mss = 0; 1423 int comp_offset, fill_offset; 1424 #ifdef __BIG_ENDIAN_BITFIELD 1425 struct Vmxnet3_RxDesc rxCmdDesc; 1426 struct Vmxnet3_RxCompDesc rxComp; 1427 #endif 1428 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, 1429 &rxComp); 1430 while (rcd->gen == rq->comp_ring.gen) { 1431 struct vmxnet3_rx_buf_info *rbi; 1432 struct sk_buff *skb, *new_skb = NULL; 1433 struct page *new_page = NULL; 1434 dma_addr_t new_dma_addr; 1435 int num_to_alloc; 1436 struct Vmxnet3_RxDesc *rxd; 1437 u32 idx, ring_idx; 1438 struct vmxnet3_cmd_ring *ring = NULL; 1439 if (num_pkts >= quota) { 1440 /* we may stop even before we see the EOP desc of 1441 * the current pkt 1442 */ 1443 break; 1444 } 1445 1446 /* Prevent any rcd field from being (speculatively) read before 1447 * rcd->gen is read. 1448 */ 1449 dma_rmb(); 1450 1451 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 && 1452 rcd->rqID != rq->dataRingQid); 1453 idx = rcd->rxdIdx; 1454 ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID); 1455 ring = rq->rx_ring + ring_idx; 1456 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd, 1457 &rxCmdDesc); 1458 rbi = rq->buf_info[ring_idx] + idx; 1459 1460 BUG_ON(rxd->addr != rbi->dma_addr || 1461 rxd->len != rbi->len); 1462 1463 if (unlikely(rcd->eop && rcd->err)) { 1464 vmxnet3_rx_error(rq, rcd, ctx, adapter); 1465 goto rcd_done; 1466 } 1467 1468 if (rcd->sop) { /* first buf of the pkt */ 1469 bool rxDataRingUsed; 1470 u16 len; 1471 1472 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD || 1473 (rcd->rqID != rq->qid && 1474 rcd->rqID != rq->dataRingQid)); 1475 1476 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB); 1477 BUG_ON(ctx->skb != NULL || rbi->skb == NULL); 1478 1479 if (unlikely(rcd->len == 0)) { 1480 /* Pretend the rx buffer is skipped. */ 1481 BUG_ON(!(rcd->sop && rcd->eop)); 1482 netdev_dbg(adapter->netdev, 1483 "rxRing[%u][%u] 0 length\n", 1484 ring_idx, idx); 1485 goto rcd_done; 1486 } 1487 1488 skip_page_frags = false; 1489 ctx->skb = rbi->skb; 1490 1491 rxDataRingUsed = 1492 VMXNET3_RX_DATA_RING(adapter, rcd->rqID); 1493 len = rxDataRingUsed ? rcd->len : rbi->len; 1494 new_skb = netdev_alloc_skb_ip_align(adapter->netdev, 1495 len); 1496 if (new_skb == NULL) { 1497 /* Skb allocation failed, do not handover this 1498 * skb to stack. Reuse it. Drop the existing pkt 1499 */ 1500 rq->stats.rx_buf_alloc_failure++; 1501 ctx->skb = NULL; 1502 rq->stats.drop_total++; 1503 skip_page_frags = true; 1504 goto rcd_done; 1505 } 1506 1507 if (rxDataRingUsed) { 1508 size_t sz; 1509 1510 BUG_ON(rcd->len > rq->data_ring.desc_size); 1511 1512 ctx->skb = new_skb; 1513 sz = rcd->rxdIdx * rq->data_ring.desc_size; 1514 memcpy(new_skb->data, 1515 &rq->data_ring.base[sz], rcd->len); 1516 } else { 1517 ctx->skb = rbi->skb; 1518 1519 new_dma_addr = 1520 dma_map_single(&adapter->pdev->dev, 1521 new_skb->data, rbi->len, 1522 DMA_FROM_DEVICE); 1523 if (dma_mapping_error(&adapter->pdev->dev, 1524 new_dma_addr)) { 1525 dev_kfree_skb(new_skb); 1526 /* Skb allocation failed, do not 1527 * handover this skb to stack. Reuse 1528 * it. Drop the existing pkt. 1529 */ 1530 rq->stats.rx_buf_alloc_failure++; 1531 ctx->skb = NULL; 1532 rq->stats.drop_total++; 1533 skip_page_frags = true; 1534 goto rcd_done; 1535 } 1536 1537 dma_unmap_single(&adapter->pdev->dev, 1538 rbi->dma_addr, 1539 rbi->len, 1540 DMA_FROM_DEVICE); 1541 1542 /* Immediate refill */ 1543 rbi->skb = new_skb; 1544 rbi->dma_addr = new_dma_addr; 1545 rxd->addr = cpu_to_le64(rbi->dma_addr); 1546 rxd->len = rbi->len; 1547 } 1548 1549 skb_record_rx_queue(ctx->skb, rq->qid); 1550 skb_put(ctx->skb, rcd->len); 1551 1552 if (VMXNET3_VERSION_GE_2(adapter) && 1553 rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) { 1554 struct Vmxnet3_RxCompDescExt *rcdlro; 1555 union Vmxnet3_GenericDesc *gdesc; 1556 1557 rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd; 1558 gdesc = (union Vmxnet3_GenericDesc *)rcd; 1559 1560 segCnt = rcdlro->segCnt; 1561 WARN_ON_ONCE(segCnt == 0); 1562 mss = rcdlro->mss; 1563 if (unlikely(segCnt <= 1)) 1564 segCnt = 0; 1565 encap_lro = (le32_to_cpu(gdesc->dword[0]) & 1566 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)); 1567 } else { 1568 segCnt = 0; 1569 } 1570 } else { 1571 BUG_ON(ctx->skb == NULL && !skip_page_frags); 1572 1573 /* non SOP buffer must be type 1 in most cases */ 1574 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE); 1575 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY); 1576 1577 /* If an sop buffer was dropped, skip all 1578 * following non-sop fragments. They will be reused. 1579 */ 1580 if (skip_page_frags) 1581 goto rcd_done; 1582 1583 if (rcd->len) { 1584 new_page = alloc_page(GFP_ATOMIC); 1585 /* Replacement page frag could not be allocated. 1586 * Reuse this page. Drop the pkt and free the 1587 * skb which contained this page as a frag. Skip 1588 * processing all the following non-sop frags. 1589 */ 1590 if (unlikely(!new_page)) { 1591 rq->stats.rx_buf_alloc_failure++; 1592 dev_kfree_skb(ctx->skb); 1593 ctx->skb = NULL; 1594 skip_page_frags = true; 1595 goto rcd_done; 1596 } 1597 new_dma_addr = dma_map_page(&adapter->pdev->dev, 1598 new_page, 1599 0, PAGE_SIZE, 1600 DMA_FROM_DEVICE); 1601 if (dma_mapping_error(&adapter->pdev->dev, 1602 new_dma_addr)) { 1603 put_page(new_page); 1604 rq->stats.rx_buf_alloc_failure++; 1605 dev_kfree_skb(ctx->skb); 1606 ctx->skb = NULL; 1607 skip_page_frags = true; 1608 goto rcd_done; 1609 } 1610 1611 dma_unmap_page(&adapter->pdev->dev, 1612 rbi->dma_addr, rbi->len, 1613 DMA_FROM_DEVICE); 1614 1615 vmxnet3_append_frag(ctx->skb, rcd, rbi); 1616 1617 /* Immediate refill */ 1618 rbi->page = new_page; 1619 rbi->dma_addr = new_dma_addr; 1620 rxd->addr = cpu_to_le64(rbi->dma_addr); 1621 rxd->len = rbi->len; 1622 } 1623 } 1624 1625 1626 skb = ctx->skb; 1627 if (rcd->eop) { 1628 u32 mtu = adapter->netdev->mtu; 1629 skb->len += skb->data_len; 1630 1631 #ifdef VMXNET3_RSS 1632 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE && 1633 (adapter->netdev->features & NETIF_F_RXHASH)) { 1634 enum pkt_hash_types hash_type; 1635 1636 switch (rcd->rssType) { 1637 case VMXNET3_RCD_RSS_TYPE_IPV4: 1638 case VMXNET3_RCD_RSS_TYPE_IPV6: 1639 hash_type = PKT_HASH_TYPE_L3; 1640 break; 1641 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 1642 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 1643 case VMXNET3_RCD_RSS_TYPE_UDPIPV4: 1644 case VMXNET3_RCD_RSS_TYPE_UDPIPV6: 1645 hash_type = PKT_HASH_TYPE_L4; 1646 break; 1647 default: 1648 hash_type = PKT_HASH_TYPE_L3; 1649 break; 1650 } 1651 skb_set_hash(skb, 1652 le32_to_cpu(rcd->rssHash), 1653 hash_type); 1654 } 1655 #endif 1656 vmxnet3_rx_csum(adapter, skb, 1657 (union Vmxnet3_GenericDesc *)rcd); 1658 skb->protocol = eth_type_trans(skb, adapter->netdev); 1659 if ((!rcd->tcp && !encap_lro) || 1660 !(adapter->netdev->features & NETIF_F_LRO)) 1661 goto not_lro; 1662 1663 if (segCnt != 0 && mss != 0) { 1664 skb_shinfo(skb)->gso_type = rcd->v4 ? 1665 SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1666 skb_shinfo(skb)->gso_size = mss; 1667 skb_shinfo(skb)->gso_segs = segCnt; 1668 } else if ((segCnt != 0 || skb->len > mtu) && !encap_lro) { 1669 u32 hlen; 1670 1671 hlen = vmxnet3_get_hdr_len(adapter, skb, 1672 (union Vmxnet3_GenericDesc *)rcd); 1673 if (hlen == 0) 1674 goto not_lro; 1675 1676 skb_shinfo(skb)->gso_type = 1677 rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1678 if (segCnt != 0) { 1679 skb_shinfo(skb)->gso_segs = segCnt; 1680 skb_shinfo(skb)->gso_size = 1681 DIV_ROUND_UP(skb->len - 1682 hlen, segCnt); 1683 } else { 1684 skb_shinfo(skb)->gso_size = mtu - hlen; 1685 } 1686 } 1687 not_lro: 1688 if (unlikely(rcd->ts)) 1689 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci); 1690 1691 /* Use GRO callback if UPT is enabled */ 1692 if ((adapter->netdev->features & NETIF_F_LRO) && 1693 !rq->shared->updateRxProd) 1694 netif_receive_skb(skb); 1695 else 1696 napi_gro_receive(&rq->napi, skb); 1697 1698 ctx->skb = NULL; 1699 encap_lro = false; 1700 num_pkts++; 1701 } 1702 1703 rcd_done: 1704 /* device may have skipped some rx descs */ 1705 ring = rq->rx_ring + ring_idx; 1706 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 1707 1708 comp_offset = vmxnet3_cmd_ring_desc_avail(ring); 1709 fill_offset = (idx > ring->next2fill ? 0 : ring->size) + 1710 idx - ring->next2fill - 1; 1711 if (!ring->isOutOfOrder || fill_offset >= comp_offset) 1712 ring->next2comp = idx; 1713 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring); 1714 1715 /* Ensure that the writes to rxd->gen bits will be observed 1716 * after all other writes to rxd objects. 1717 */ 1718 dma_wmb(); 1719 1720 while (num_to_alloc) { 1721 rbi = rq->buf_info[ring_idx] + ring->next2fill; 1722 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_OOORX_COMP))) 1723 goto refill_buf; 1724 if (ring_idx == 0) { 1725 /* ring0 Type1 buffers can get skipped; re-fill them */ 1726 if (rbi->buf_type != VMXNET3_RX_BUF_SKB) 1727 goto refill_buf; 1728 } 1729 if (rbi->comp_state == VMXNET3_RXD_COMP_DONE) { 1730 refill_buf: 1731 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd, 1732 &rxCmdDesc); 1733 WARN_ON(!rxd->addr); 1734 1735 /* Recv desc is ready to be used by the device */ 1736 rxd->gen = ring->gen; 1737 vmxnet3_cmd_ring_adv_next2fill(ring); 1738 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 1739 num_to_alloc--; 1740 } else { 1741 /* rx completion hasn't occurred */ 1742 ring->isOutOfOrder = 1; 1743 break; 1744 } 1745 } 1746 1747 if (num_to_alloc == 0) { 1748 ring->isOutOfOrder = 0; 1749 } 1750 1751 /* if needed, update the register */ 1752 if (unlikely(rq->shared->updateRxProd) && (ring->next2fill & 0xf) == 0) { 1753 VMXNET3_WRITE_BAR0_REG(adapter, 1754 rxprod_reg[ring_idx] + rq->qid * 8, 1755 ring->next2fill); 1756 } 1757 1758 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring); 1759 vmxnet3_getRxComp(rcd, 1760 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp); 1761 } 1762 1763 return num_pkts; 1764 } 1765 1766 1767 static void 1768 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq, 1769 struct vmxnet3_adapter *adapter) 1770 { 1771 u32 i, ring_idx; 1772 struct Vmxnet3_RxDesc *rxd; 1773 1774 /* ring has already been cleaned up */ 1775 if (!rq->rx_ring[0].base) 1776 return; 1777 1778 for (ring_idx = 0; ring_idx < 2; ring_idx++) { 1779 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) { 1780 #ifdef __BIG_ENDIAN_BITFIELD 1781 struct Vmxnet3_RxDesc rxDesc; 1782 #endif 1783 vmxnet3_getRxDesc(rxd, 1784 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc); 1785 1786 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && 1787 rq->buf_info[ring_idx][i].skb) { 1788 dma_unmap_single(&adapter->pdev->dev, rxd->addr, 1789 rxd->len, DMA_FROM_DEVICE); 1790 dev_kfree_skb(rq->buf_info[ring_idx][i].skb); 1791 rq->buf_info[ring_idx][i].skb = NULL; 1792 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY && 1793 rq->buf_info[ring_idx][i].page) { 1794 dma_unmap_page(&adapter->pdev->dev, rxd->addr, 1795 rxd->len, DMA_FROM_DEVICE); 1796 put_page(rq->buf_info[ring_idx][i].page); 1797 rq->buf_info[ring_idx][i].page = NULL; 1798 } 1799 } 1800 1801 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN; 1802 rq->rx_ring[ring_idx].next2fill = 1803 rq->rx_ring[ring_idx].next2comp = 0; 1804 } 1805 1806 rq->comp_ring.gen = VMXNET3_INIT_GEN; 1807 rq->comp_ring.next2proc = 0; 1808 } 1809 1810 1811 static void 1812 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter) 1813 { 1814 int i; 1815 1816 for (i = 0; i < adapter->num_rx_queues; i++) 1817 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter); 1818 } 1819 1820 1821 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, 1822 struct vmxnet3_adapter *adapter) 1823 { 1824 int i; 1825 int j; 1826 1827 /* all rx buffers must have already been freed */ 1828 for (i = 0; i < 2; i++) { 1829 if (rq->buf_info[i]) { 1830 for (j = 0; j < rq->rx_ring[i].size; j++) 1831 BUG_ON(rq->buf_info[i][j].page != NULL); 1832 } 1833 } 1834 1835 1836 for (i = 0; i < 2; i++) { 1837 if (rq->rx_ring[i].base) { 1838 dma_free_coherent(&adapter->pdev->dev, 1839 rq->rx_ring[i].size 1840 * sizeof(struct Vmxnet3_RxDesc), 1841 rq->rx_ring[i].base, 1842 rq->rx_ring[i].basePA); 1843 rq->rx_ring[i].base = NULL; 1844 } 1845 } 1846 1847 if (rq->data_ring.base) { 1848 dma_free_coherent(&adapter->pdev->dev, 1849 rq->rx_ring[0].size * rq->data_ring.desc_size, 1850 rq->data_ring.base, rq->data_ring.basePA); 1851 rq->data_ring.base = NULL; 1852 } 1853 1854 if (rq->comp_ring.base) { 1855 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size 1856 * sizeof(struct Vmxnet3_RxCompDesc), 1857 rq->comp_ring.base, rq->comp_ring.basePA); 1858 rq->comp_ring.base = NULL; 1859 } 1860 1861 kfree(rq->buf_info[0]); 1862 rq->buf_info[0] = NULL; 1863 rq->buf_info[1] = NULL; 1864 } 1865 1866 static void 1867 vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter) 1868 { 1869 int i; 1870 1871 for (i = 0; i < adapter->num_rx_queues; i++) { 1872 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 1873 1874 if (rq->data_ring.base) { 1875 dma_free_coherent(&adapter->pdev->dev, 1876 (rq->rx_ring[0].size * 1877 rq->data_ring.desc_size), 1878 rq->data_ring.base, 1879 rq->data_ring.basePA); 1880 rq->data_ring.base = NULL; 1881 rq->data_ring.desc_size = 0; 1882 } 1883 } 1884 } 1885 1886 static int 1887 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq, 1888 struct vmxnet3_adapter *adapter) 1889 { 1890 int i; 1891 1892 /* initialize buf_info */ 1893 for (i = 0; i < rq->rx_ring[0].size; i++) { 1894 1895 /* 1st buf for a pkt is skbuff */ 1896 if (i % adapter->rx_buf_per_pkt == 0) { 1897 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB; 1898 rq->buf_info[0][i].len = adapter->skb_buf_size; 1899 } else { /* subsequent bufs for a pkt is frag */ 1900 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE; 1901 rq->buf_info[0][i].len = PAGE_SIZE; 1902 } 1903 } 1904 for (i = 0; i < rq->rx_ring[1].size; i++) { 1905 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE; 1906 rq->buf_info[1][i].len = PAGE_SIZE; 1907 } 1908 1909 /* reset internal state and allocate buffers for both rings */ 1910 for (i = 0; i < 2; i++) { 1911 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0; 1912 1913 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size * 1914 sizeof(struct Vmxnet3_RxDesc)); 1915 rq->rx_ring[i].gen = VMXNET3_INIT_GEN; 1916 rq->rx_ring[i].isOutOfOrder = 0; 1917 } 1918 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1, 1919 adapter) == 0) { 1920 /* at least has 1 rx buffer for the 1st ring */ 1921 return -ENOMEM; 1922 } 1923 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter); 1924 1925 /* reset the comp ring */ 1926 rq->comp_ring.next2proc = 0; 1927 memset(rq->comp_ring.base, 0, rq->comp_ring.size * 1928 sizeof(struct Vmxnet3_RxCompDesc)); 1929 rq->comp_ring.gen = VMXNET3_INIT_GEN; 1930 1931 /* reset rxctx */ 1932 rq->rx_ctx.skb = NULL; 1933 1934 /* stats are not reset */ 1935 return 0; 1936 } 1937 1938 1939 static int 1940 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter) 1941 { 1942 int i, err = 0; 1943 1944 for (i = 0; i < adapter->num_rx_queues; i++) { 1945 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter); 1946 if (unlikely(err)) { 1947 dev_err(&adapter->netdev->dev, "%s: failed to " 1948 "initialize rx queue%i\n", 1949 adapter->netdev->name, i); 1950 break; 1951 } 1952 } 1953 return err; 1954 1955 } 1956 1957 1958 static int 1959 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter) 1960 { 1961 int i; 1962 size_t sz; 1963 struct vmxnet3_rx_buf_info *bi; 1964 1965 for (i = 0; i < 2; i++) { 1966 1967 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc); 1968 rq->rx_ring[i].base = dma_alloc_coherent( 1969 &adapter->pdev->dev, sz, 1970 &rq->rx_ring[i].basePA, 1971 GFP_KERNEL); 1972 if (!rq->rx_ring[i].base) { 1973 netdev_err(adapter->netdev, 1974 "failed to allocate rx ring %d\n", i); 1975 goto err; 1976 } 1977 } 1978 1979 if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) { 1980 sz = rq->rx_ring[0].size * rq->data_ring.desc_size; 1981 rq->data_ring.base = 1982 dma_alloc_coherent(&adapter->pdev->dev, sz, 1983 &rq->data_ring.basePA, 1984 GFP_KERNEL); 1985 if (!rq->data_ring.base) { 1986 netdev_err(adapter->netdev, 1987 "rx data ring will be disabled\n"); 1988 adapter->rxdataring_enabled = false; 1989 } 1990 } else { 1991 rq->data_ring.base = NULL; 1992 rq->data_ring.desc_size = 0; 1993 } 1994 1995 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc); 1996 rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz, 1997 &rq->comp_ring.basePA, 1998 GFP_KERNEL); 1999 if (!rq->comp_ring.base) { 2000 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n"); 2001 goto err; 2002 } 2003 2004 bi = kcalloc_node(rq->rx_ring[0].size + rq->rx_ring[1].size, 2005 sizeof(rq->buf_info[0][0]), GFP_KERNEL, 2006 dev_to_node(&adapter->pdev->dev)); 2007 if (!bi) 2008 goto err; 2009 2010 rq->buf_info[0] = bi; 2011 rq->buf_info[1] = bi + rq->rx_ring[0].size; 2012 2013 return 0; 2014 2015 err: 2016 vmxnet3_rq_destroy(rq, adapter); 2017 return -ENOMEM; 2018 } 2019 2020 2021 static int 2022 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter) 2023 { 2024 int i, err = 0; 2025 2026 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 2027 2028 for (i = 0; i < adapter->num_rx_queues; i++) { 2029 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter); 2030 if (unlikely(err)) { 2031 dev_err(&adapter->netdev->dev, 2032 "%s: failed to create rx queue%i\n", 2033 adapter->netdev->name, i); 2034 goto err_out; 2035 } 2036 } 2037 2038 if (!adapter->rxdataring_enabled) 2039 vmxnet3_rq_destroy_all_rxdataring(adapter); 2040 2041 return err; 2042 err_out: 2043 vmxnet3_rq_destroy_all(adapter); 2044 return err; 2045 2046 } 2047 2048 /* Multiple queue aware polling function for tx and rx */ 2049 2050 static int 2051 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget) 2052 { 2053 int rcd_done = 0, i; 2054 if (unlikely(adapter->shared->ecr)) 2055 vmxnet3_process_events(adapter); 2056 for (i = 0; i < adapter->num_tx_queues; i++) 2057 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter); 2058 2059 for (i = 0; i < adapter->num_rx_queues; i++) 2060 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i], 2061 adapter, budget); 2062 return rcd_done; 2063 } 2064 2065 2066 static int 2067 vmxnet3_poll(struct napi_struct *napi, int budget) 2068 { 2069 struct vmxnet3_rx_queue *rx_queue = container_of(napi, 2070 struct vmxnet3_rx_queue, napi); 2071 int rxd_done; 2072 2073 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget); 2074 2075 if (rxd_done < budget) { 2076 napi_complete_done(napi, rxd_done); 2077 vmxnet3_enable_all_intrs(rx_queue->adapter); 2078 } 2079 return rxd_done; 2080 } 2081 2082 /* 2083 * NAPI polling function for MSI-X mode with multiple Rx queues 2084 * Returns the # of the NAPI credit consumed (# of rx descriptors processed) 2085 */ 2086 2087 static int 2088 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget) 2089 { 2090 struct vmxnet3_rx_queue *rq = container_of(napi, 2091 struct vmxnet3_rx_queue, napi); 2092 struct vmxnet3_adapter *adapter = rq->adapter; 2093 int rxd_done; 2094 2095 /* When sharing interrupt with corresponding tx queue, process 2096 * tx completions in that queue as well 2097 */ 2098 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) { 2099 struct vmxnet3_tx_queue *tq = 2100 &adapter->tx_queue[rq - adapter->rx_queue]; 2101 vmxnet3_tq_tx_complete(tq, adapter); 2102 } 2103 2104 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget); 2105 2106 if (rxd_done < budget) { 2107 napi_complete_done(napi, rxd_done); 2108 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx); 2109 } 2110 return rxd_done; 2111 } 2112 2113 2114 #ifdef CONFIG_PCI_MSI 2115 2116 /* 2117 * Handle completion interrupts on tx queues 2118 * Returns whether or not the intr is handled 2119 */ 2120 2121 static irqreturn_t 2122 vmxnet3_msix_tx(int irq, void *data) 2123 { 2124 struct vmxnet3_tx_queue *tq = data; 2125 struct vmxnet3_adapter *adapter = tq->adapter; 2126 2127 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2128 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx); 2129 2130 /* Handle the case where only one irq is allocate for all tx queues */ 2131 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2132 int i; 2133 for (i = 0; i < adapter->num_tx_queues; i++) { 2134 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i]; 2135 vmxnet3_tq_tx_complete(txq, adapter); 2136 } 2137 } else { 2138 vmxnet3_tq_tx_complete(tq, adapter); 2139 } 2140 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx); 2141 2142 return IRQ_HANDLED; 2143 } 2144 2145 2146 /* 2147 * Handle completion interrupts on rx queues. Returns whether or not the 2148 * intr is handled 2149 */ 2150 2151 static irqreturn_t 2152 vmxnet3_msix_rx(int irq, void *data) 2153 { 2154 struct vmxnet3_rx_queue *rq = data; 2155 struct vmxnet3_adapter *adapter = rq->adapter; 2156 2157 /* disable intr if needed */ 2158 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2159 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx); 2160 napi_schedule(&rq->napi); 2161 2162 return IRQ_HANDLED; 2163 } 2164 2165 /* 2166 *---------------------------------------------------------------------------- 2167 * 2168 * vmxnet3_msix_event -- 2169 * 2170 * vmxnet3 msix event intr handler 2171 * 2172 * Result: 2173 * whether or not the intr is handled 2174 * 2175 *---------------------------------------------------------------------------- 2176 */ 2177 2178 static irqreturn_t 2179 vmxnet3_msix_event(int irq, void *data) 2180 { 2181 struct net_device *dev = data; 2182 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2183 2184 /* disable intr if needed */ 2185 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2186 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx); 2187 2188 if (adapter->shared->ecr) 2189 vmxnet3_process_events(adapter); 2190 2191 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx); 2192 2193 return IRQ_HANDLED; 2194 } 2195 2196 #endif /* CONFIG_PCI_MSI */ 2197 2198 2199 /* Interrupt handler for vmxnet3 */ 2200 static irqreturn_t 2201 vmxnet3_intr(int irq, void *dev_id) 2202 { 2203 struct net_device *dev = dev_id; 2204 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2205 2206 if (adapter->intr.type == VMXNET3_IT_INTX) { 2207 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR); 2208 if (unlikely(icr == 0)) 2209 /* not ours */ 2210 return IRQ_NONE; 2211 } 2212 2213 2214 /* disable intr if needed */ 2215 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2216 vmxnet3_disable_all_intrs(adapter); 2217 2218 napi_schedule(&adapter->rx_queue[0].napi); 2219 2220 return IRQ_HANDLED; 2221 } 2222 2223 #ifdef CONFIG_NET_POLL_CONTROLLER 2224 2225 /* netpoll callback. */ 2226 static void 2227 vmxnet3_netpoll(struct net_device *netdev) 2228 { 2229 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2230 2231 switch (adapter->intr.type) { 2232 #ifdef CONFIG_PCI_MSI 2233 case VMXNET3_IT_MSIX: { 2234 int i; 2235 for (i = 0; i < adapter->num_rx_queues; i++) 2236 vmxnet3_msix_rx(0, &adapter->rx_queue[i]); 2237 break; 2238 } 2239 #endif 2240 case VMXNET3_IT_MSI: 2241 default: 2242 vmxnet3_intr(0, adapter->netdev); 2243 break; 2244 } 2245 2246 } 2247 #endif /* CONFIG_NET_POLL_CONTROLLER */ 2248 2249 static int 2250 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter) 2251 { 2252 struct vmxnet3_intr *intr = &adapter->intr; 2253 int err = 0, i; 2254 int vector = 0; 2255 2256 #ifdef CONFIG_PCI_MSI 2257 if (adapter->intr.type == VMXNET3_IT_MSIX) { 2258 for (i = 0; i < adapter->num_tx_queues; i++) { 2259 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2260 sprintf(adapter->tx_queue[i].name, "%s-tx-%d", 2261 adapter->netdev->name, vector); 2262 err = request_irq( 2263 intr->msix_entries[vector].vector, 2264 vmxnet3_msix_tx, 0, 2265 adapter->tx_queue[i].name, 2266 &adapter->tx_queue[i]); 2267 } else { 2268 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d", 2269 adapter->netdev->name, vector); 2270 } 2271 if (err) { 2272 dev_err(&adapter->netdev->dev, 2273 "Failed to request irq for MSIX, %s, " 2274 "error %d\n", 2275 adapter->tx_queue[i].name, err); 2276 return err; 2277 } 2278 2279 /* Handle the case where only 1 MSIx was allocated for 2280 * all tx queues */ 2281 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2282 for (; i < adapter->num_tx_queues; i++) 2283 adapter->tx_queue[i].comp_ring.intr_idx 2284 = vector; 2285 vector++; 2286 break; 2287 } else { 2288 adapter->tx_queue[i].comp_ring.intr_idx 2289 = vector++; 2290 } 2291 } 2292 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) 2293 vector = 0; 2294 2295 for (i = 0; i < adapter->num_rx_queues; i++) { 2296 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) 2297 sprintf(adapter->rx_queue[i].name, "%s-rx-%d", 2298 adapter->netdev->name, vector); 2299 else 2300 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d", 2301 adapter->netdev->name, vector); 2302 err = request_irq(intr->msix_entries[vector].vector, 2303 vmxnet3_msix_rx, 0, 2304 adapter->rx_queue[i].name, 2305 &(adapter->rx_queue[i])); 2306 if (err) { 2307 netdev_err(adapter->netdev, 2308 "Failed to request irq for MSIX, " 2309 "%s, error %d\n", 2310 adapter->rx_queue[i].name, err); 2311 return err; 2312 } 2313 2314 adapter->rx_queue[i].comp_ring.intr_idx = vector++; 2315 } 2316 2317 sprintf(intr->event_msi_vector_name, "%s-event-%d", 2318 adapter->netdev->name, vector); 2319 err = request_irq(intr->msix_entries[vector].vector, 2320 vmxnet3_msix_event, 0, 2321 intr->event_msi_vector_name, adapter->netdev); 2322 intr->event_intr_idx = vector; 2323 2324 } else if (intr->type == VMXNET3_IT_MSI) { 2325 adapter->num_rx_queues = 1; 2326 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0, 2327 adapter->netdev->name, adapter->netdev); 2328 } else { 2329 #endif 2330 adapter->num_rx_queues = 1; 2331 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 2332 IRQF_SHARED, adapter->netdev->name, 2333 adapter->netdev); 2334 #ifdef CONFIG_PCI_MSI 2335 } 2336 #endif 2337 intr->num_intrs = vector + 1; 2338 if (err) { 2339 netdev_err(adapter->netdev, 2340 "Failed to request irq (intr type:%d), error %d\n", 2341 intr->type, err); 2342 } else { 2343 /* Number of rx queues will not change after this */ 2344 for (i = 0; i < adapter->num_rx_queues; i++) { 2345 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2346 rq->qid = i; 2347 rq->qid2 = i + adapter->num_rx_queues; 2348 rq->dataRingQid = i + 2 * adapter->num_rx_queues; 2349 } 2350 2351 /* init our intr settings */ 2352 for (i = 0; i < intr->num_intrs; i++) 2353 intr->mod_levels[i] = UPT1_IML_ADAPTIVE; 2354 if (adapter->intr.type != VMXNET3_IT_MSIX) { 2355 adapter->intr.event_intr_idx = 0; 2356 for (i = 0; i < adapter->num_tx_queues; i++) 2357 adapter->tx_queue[i].comp_ring.intr_idx = 0; 2358 adapter->rx_queue[0].comp_ring.intr_idx = 0; 2359 } 2360 2361 netdev_info(adapter->netdev, 2362 "intr type %u, mode %u, %u vectors allocated\n", 2363 intr->type, intr->mask_mode, intr->num_intrs); 2364 } 2365 2366 return err; 2367 } 2368 2369 2370 static void 2371 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter) 2372 { 2373 struct vmxnet3_intr *intr = &adapter->intr; 2374 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0); 2375 2376 switch (intr->type) { 2377 #ifdef CONFIG_PCI_MSI 2378 case VMXNET3_IT_MSIX: 2379 { 2380 int i, vector = 0; 2381 2382 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2383 for (i = 0; i < adapter->num_tx_queues; i++) { 2384 free_irq(intr->msix_entries[vector++].vector, 2385 &(adapter->tx_queue[i])); 2386 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) 2387 break; 2388 } 2389 } 2390 2391 for (i = 0; i < adapter->num_rx_queues; i++) { 2392 free_irq(intr->msix_entries[vector++].vector, 2393 &(adapter->rx_queue[i])); 2394 } 2395 2396 free_irq(intr->msix_entries[vector].vector, 2397 adapter->netdev); 2398 BUG_ON(vector >= intr->num_intrs); 2399 break; 2400 } 2401 #endif 2402 case VMXNET3_IT_MSI: 2403 free_irq(adapter->pdev->irq, adapter->netdev); 2404 break; 2405 case VMXNET3_IT_INTX: 2406 free_irq(adapter->pdev->irq, adapter->netdev); 2407 break; 2408 default: 2409 BUG(); 2410 } 2411 } 2412 2413 2414 static void 2415 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter) 2416 { 2417 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2418 u16 vid; 2419 2420 /* allow untagged pkts */ 2421 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0); 2422 2423 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2424 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2425 } 2426 2427 2428 static int 2429 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 2430 { 2431 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2432 2433 if (!(netdev->flags & IFF_PROMISC)) { 2434 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2435 unsigned long flags; 2436 2437 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2438 spin_lock_irqsave(&adapter->cmd_lock, flags); 2439 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2440 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2441 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2442 } 2443 2444 set_bit(vid, adapter->active_vlans); 2445 2446 return 0; 2447 } 2448 2449 2450 static int 2451 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 2452 { 2453 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2454 2455 if (!(netdev->flags & IFF_PROMISC)) { 2456 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2457 unsigned long flags; 2458 2459 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid); 2460 spin_lock_irqsave(&adapter->cmd_lock, flags); 2461 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2462 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2463 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2464 } 2465 2466 clear_bit(vid, adapter->active_vlans); 2467 2468 return 0; 2469 } 2470 2471 2472 static u8 * 2473 vmxnet3_copy_mc(struct net_device *netdev) 2474 { 2475 u8 *buf = NULL; 2476 u32 sz = netdev_mc_count(netdev) * ETH_ALEN; 2477 2478 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */ 2479 if (sz <= 0xffff) { 2480 /* We may be called with BH disabled */ 2481 buf = kmalloc(sz, GFP_ATOMIC); 2482 if (buf) { 2483 struct netdev_hw_addr *ha; 2484 int i = 0; 2485 2486 netdev_for_each_mc_addr(ha, netdev) 2487 memcpy(buf + i++ * ETH_ALEN, ha->addr, 2488 ETH_ALEN); 2489 } 2490 } 2491 return buf; 2492 } 2493 2494 2495 static void 2496 vmxnet3_set_mc(struct net_device *netdev) 2497 { 2498 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2499 unsigned long flags; 2500 struct Vmxnet3_RxFilterConf *rxConf = 2501 &adapter->shared->devRead.rxFilterConf; 2502 u8 *new_table = NULL; 2503 dma_addr_t new_table_pa = 0; 2504 bool new_table_pa_valid = false; 2505 u32 new_mode = VMXNET3_RXM_UCAST; 2506 2507 if (netdev->flags & IFF_PROMISC) { 2508 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2509 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable)); 2510 2511 new_mode |= VMXNET3_RXM_PROMISC; 2512 } else { 2513 vmxnet3_restore_vlan(adapter); 2514 } 2515 2516 if (netdev->flags & IFF_BROADCAST) 2517 new_mode |= VMXNET3_RXM_BCAST; 2518 2519 if (netdev->flags & IFF_ALLMULTI) 2520 new_mode |= VMXNET3_RXM_ALL_MULTI; 2521 else 2522 if (!netdev_mc_empty(netdev)) { 2523 new_table = vmxnet3_copy_mc(netdev); 2524 if (new_table) { 2525 size_t sz = netdev_mc_count(netdev) * ETH_ALEN; 2526 2527 rxConf->mfTableLen = cpu_to_le16(sz); 2528 new_table_pa = dma_map_single( 2529 &adapter->pdev->dev, 2530 new_table, 2531 sz, 2532 DMA_TO_DEVICE); 2533 if (!dma_mapping_error(&adapter->pdev->dev, 2534 new_table_pa)) { 2535 new_mode |= VMXNET3_RXM_MCAST; 2536 new_table_pa_valid = true; 2537 rxConf->mfTablePA = cpu_to_le64( 2538 new_table_pa); 2539 } 2540 } 2541 if (!new_table_pa_valid) { 2542 netdev_info(netdev, 2543 "failed to copy mcast list, setting ALL_MULTI\n"); 2544 new_mode |= VMXNET3_RXM_ALL_MULTI; 2545 } 2546 } 2547 2548 if (!(new_mode & VMXNET3_RXM_MCAST)) { 2549 rxConf->mfTableLen = 0; 2550 rxConf->mfTablePA = 0; 2551 } 2552 2553 spin_lock_irqsave(&adapter->cmd_lock, flags); 2554 if (new_mode != rxConf->rxMode) { 2555 rxConf->rxMode = cpu_to_le32(new_mode); 2556 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2557 VMXNET3_CMD_UPDATE_RX_MODE); 2558 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2559 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2560 } 2561 2562 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2563 VMXNET3_CMD_UPDATE_MAC_FILTERS); 2564 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2565 2566 if (new_table_pa_valid) 2567 dma_unmap_single(&adapter->pdev->dev, new_table_pa, 2568 rxConf->mfTableLen, DMA_TO_DEVICE); 2569 kfree(new_table); 2570 } 2571 2572 void 2573 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter) 2574 { 2575 int i; 2576 2577 for (i = 0; i < adapter->num_rx_queues; i++) 2578 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter); 2579 } 2580 2581 2582 /* 2583 * Set up driver_shared based on settings in adapter. 2584 */ 2585 2586 static void 2587 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter) 2588 { 2589 struct Vmxnet3_DriverShared *shared = adapter->shared; 2590 struct Vmxnet3_DSDevRead *devRead = &shared->devRead; 2591 struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt; 2592 struct Vmxnet3_TxQueueConf *tqc; 2593 struct Vmxnet3_RxQueueConf *rqc; 2594 int i; 2595 2596 memset(shared, 0, sizeof(*shared)); 2597 2598 /* driver settings */ 2599 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC); 2600 devRead->misc.driverInfo.version = cpu_to_le32( 2601 VMXNET3_DRIVER_VERSION_NUM); 2602 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ? 2603 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64); 2604 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; 2605 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32( 2606 *((u32 *)&devRead->misc.driverInfo.gos)); 2607 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1); 2608 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1); 2609 2610 devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa); 2611 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter)); 2612 2613 /* set up feature flags */ 2614 if (adapter->netdev->features & NETIF_F_RXCSUM) 2615 devRead->misc.uptFeatures |= UPT1_F_RXCSUM; 2616 2617 if (adapter->netdev->features & NETIF_F_LRO) { 2618 devRead->misc.uptFeatures |= UPT1_F_LRO; 2619 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS); 2620 } 2621 if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2622 devRead->misc.uptFeatures |= UPT1_F_RXVLAN; 2623 2624 if (adapter->netdev->features & (NETIF_F_GSO_UDP_TUNNEL | 2625 NETIF_F_GSO_UDP_TUNNEL_CSUM)) 2626 devRead->misc.uptFeatures |= UPT1_F_RXINNEROFLD; 2627 2628 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu); 2629 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa); 2630 devRead->misc.queueDescLen = cpu_to_le32( 2631 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + 2632 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc)); 2633 2634 /* tx queue settings */ 2635 devRead->misc.numTxQueues = adapter->num_tx_queues; 2636 for (i = 0; i < adapter->num_tx_queues; i++) { 2637 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 2638 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL); 2639 tqc = &adapter->tqd_start[i].conf; 2640 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA); 2641 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA); 2642 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA); 2643 tqc->ddPA = cpu_to_le64(~0ULL); 2644 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size); 2645 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size); 2646 tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size); 2647 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size); 2648 tqc->ddLen = cpu_to_le32(0); 2649 tqc->intrIdx = tq->comp_ring.intr_idx; 2650 } 2651 2652 /* rx queue settings */ 2653 devRead->misc.numRxQueues = adapter->num_rx_queues; 2654 for (i = 0; i < adapter->num_rx_queues; i++) { 2655 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2656 rqc = &adapter->rqd_start[i].conf; 2657 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA); 2658 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA); 2659 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA); 2660 rqc->ddPA = cpu_to_le64(~0ULL); 2661 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size); 2662 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size); 2663 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size); 2664 rqc->ddLen = cpu_to_le32(0); 2665 rqc->intrIdx = rq->comp_ring.intr_idx; 2666 if (VMXNET3_VERSION_GE_3(adapter)) { 2667 rqc->rxDataRingBasePA = 2668 cpu_to_le64(rq->data_ring.basePA); 2669 rqc->rxDataRingDescSize = 2670 cpu_to_le16(rq->data_ring.desc_size); 2671 } 2672 } 2673 2674 #ifdef VMXNET3_RSS 2675 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf)); 2676 2677 if (adapter->rss) { 2678 struct UPT1_RSSConf *rssConf = adapter->rss_conf; 2679 2680 devRead->misc.uptFeatures |= UPT1_F_RSS; 2681 devRead->misc.numRxQueues = adapter->num_rx_queues; 2682 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 | 2683 UPT1_RSS_HASH_TYPE_IPV4 | 2684 UPT1_RSS_HASH_TYPE_TCP_IPV6 | 2685 UPT1_RSS_HASH_TYPE_IPV6; 2686 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ; 2687 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE; 2688 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE; 2689 netdev_rss_key_fill(rssConf->hashKey, sizeof(rssConf->hashKey)); 2690 2691 for (i = 0; i < rssConf->indTableSize; i++) 2692 rssConf->indTable[i] = ethtool_rxfh_indir_default( 2693 i, adapter->num_rx_queues); 2694 2695 devRead->rssConfDesc.confVer = 1; 2696 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf)); 2697 devRead->rssConfDesc.confPA = 2698 cpu_to_le64(adapter->rss_conf_pa); 2699 } 2700 2701 #endif /* VMXNET3_RSS */ 2702 2703 /* intr settings */ 2704 if (!VMXNET3_VERSION_GE_6(adapter) || 2705 !adapter->queuesExtEnabled) { 2706 devRead->intrConf.autoMask = adapter->intr.mask_mode == 2707 VMXNET3_IMM_AUTO; 2708 devRead->intrConf.numIntrs = adapter->intr.num_intrs; 2709 for (i = 0; i < adapter->intr.num_intrs; i++) 2710 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i]; 2711 2712 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx; 2713 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 2714 } else { 2715 devReadExt->intrConfExt.autoMask = adapter->intr.mask_mode == 2716 VMXNET3_IMM_AUTO; 2717 devReadExt->intrConfExt.numIntrs = adapter->intr.num_intrs; 2718 for (i = 0; i < adapter->intr.num_intrs; i++) 2719 devReadExt->intrConfExt.modLevels[i] = adapter->intr.mod_levels[i]; 2720 2721 devReadExt->intrConfExt.eventIntrIdx = adapter->intr.event_intr_idx; 2722 devReadExt->intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 2723 } 2724 2725 /* rx filter settings */ 2726 devRead->rxFilterConf.rxMode = 0; 2727 vmxnet3_restore_vlan(adapter); 2728 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr); 2729 2730 /* the rest are already zeroed */ 2731 } 2732 2733 static void 2734 vmxnet3_init_bufsize(struct vmxnet3_adapter *adapter) 2735 { 2736 struct Vmxnet3_DriverShared *shared = adapter->shared; 2737 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 2738 unsigned long flags; 2739 2740 if (!VMXNET3_VERSION_GE_7(adapter)) 2741 return; 2742 2743 cmdInfo->ringBufSize = adapter->ringBufSize; 2744 spin_lock_irqsave(&adapter->cmd_lock, flags); 2745 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2746 VMXNET3_CMD_SET_RING_BUFFER_SIZE); 2747 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2748 } 2749 2750 static void 2751 vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter) 2752 { 2753 struct Vmxnet3_DriverShared *shared = adapter->shared; 2754 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 2755 unsigned long flags; 2756 2757 if (!VMXNET3_VERSION_GE_3(adapter)) 2758 return; 2759 2760 spin_lock_irqsave(&adapter->cmd_lock, flags); 2761 cmdInfo->varConf.confVer = 1; 2762 cmdInfo->varConf.confLen = 2763 cpu_to_le32(sizeof(*adapter->coal_conf)); 2764 cmdInfo->varConf.confPA = cpu_to_le64(adapter->coal_conf_pa); 2765 2766 if (adapter->default_coal_mode) { 2767 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2768 VMXNET3_CMD_GET_COALESCE); 2769 } else { 2770 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2771 VMXNET3_CMD_SET_COALESCE); 2772 } 2773 2774 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2775 } 2776 2777 static void 2778 vmxnet3_init_rssfields(struct vmxnet3_adapter *adapter) 2779 { 2780 struct Vmxnet3_DriverShared *shared = adapter->shared; 2781 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 2782 unsigned long flags; 2783 2784 if (!VMXNET3_VERSION_GE_4(adapter)) 2785 return; 2786 2787 spin_lock_irqsave(&adapter->cmd_lock, flags); 2788 2789 if (adapter->default_rss_fields) { 2790 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2791 VMXNET3_CMD_GET_RSS_FIELDS); 2792 adapter->rss_fields = 2793 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2794 } else { 2795 if (VMXNET3_VERSION_GE_7(adapter)) { 2796 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 || 2797 adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) && 2798 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 2799 VMXNET3_CAP_UDP_RSS)) { 2800 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS; 2801 } else { 2802 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS); 2803 } 2804 2805 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) && 2806 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 2807 VMXNET3_CAP_ESP_RSS_IPV4)) { 2808 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4; 2809 } else { 2810 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4); 2811 } 2812 2813 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) && 2814 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 2815 VMXNET3_CAP_ESP_RSS_IPV6)) { 2816 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6; 2817 } else { 2818 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6); 2819 } 2820 2821 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 2822 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 2823 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2824 } 2825 cmdInfo->setRssFields = adapter->rss_fields; 2826 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2827 VMXNET3_CMD_SET_RSS_FIELDS); 2828 /* Not all requested RSS may get applied, so get and 2829 * cache what was actually applied. 2830 */ 2831 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2832 VMXNET3_CMD_GET_RSS_FIELDS); 2833 adapter->rss_fields = 2834 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2835 } 2836 2837 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2838 } 2839 2840 int 2841 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter) 2842 { 2843 int err, i; 2844 u32 ret; 2845 unsigned long flags; 2846 2847 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d," 2848 " ring sizes %u %u %u\n", adapter->netdev->name, 2849 adapter->skb_buf_size, adapter->rx_buf_per_pkt, 2850 adapter->tx_queue[0].tx_ring.size, 2851 adapter->rx_queue[0].rx_ring[0].size, 2852 adapter->rx_queue[0].rx_ring[1].size); 2853 2854 vmxnet3_tq_init_all(adapter); 2855 err = vmxnet3_rq_init_all(adapter); 2856 if (err) { 2857 netdev_err(adapter->netdev, 2858 "Failed to init rx queue error %d\n", err); 2859 goto rq_err; 2860 } 2861 2862 err = vmxnet3_request_irqs(adapter); 2863 if (err) { 2864 netdev_err(adapter->netdev, 2865 "Failed to setup irq for error %d\n", err); 2866 goto irq_err; 2867 } 2868 2869 vmxnet3_setup_driver_shared(adapter); 2870 2871 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO( 2872 adapter->shared_pa)); 2873 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI( 2874 adapter->shared_pa)); 2875 spin_lock_irqsave(&adapter->cmd_lock, flags); 2876 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2877 VMXNET3_CMD_ACTIVATE_DEV); 2878 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2879 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2880 2881 if (ret != 0) { 2882 netdev_err(adapter->netdev, 2883 "Failed to activate dev: error %u\n", ret); 2884 err = -EINVAL; 2885 goto activate_err; 2886 } 2887 2888 vmxnet3_init_bufsize(adapter); 2889 vmxnet3_init_coalesce(adapter); 2890 vmxnet3_init_rssfields(adapter); 2891 2892 for (i = 0; i < adapter->num_rx_queues; i++) { 2893 VMXNET3_WRITE_BAR0_REG(adapter, 2894 adapter->rx_prod_offset + i * VMXNET3_REG_ALIGN, 2895 adapter->rx_queue[i].rx_ring[0].next2fill); 2896 VMXNET3_WRITE_BAR0_REG(adapter, (adapter->rx_prod2_offset + 2897 (i * VMXNET3_REG_ALIGN)), 2898 adapter->rx_queue[i].rx_ring[1].next2fill); 2899 } 2900 2901 /* Apply the rx filter settins last. */ 2902 vmxnet3_set_mc(adapter->netdev); 2903 2904 /* 2905 * Check link state when first activating device. It will start the 2906 * tx queue if the link is up. 2907 */ 2908 vmxnet3_check_link(adapter, true); 2909 netif_tx_wake_all_queues(adapter->netdev); 2910 for (i = 0; i < adapter->num_rx_queues; i++) 2911 napi_enable(&adapter->rx_queue[i].napi); 2912 vmxnet3_enable_all_intrs(adapter); 2913 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 2914 return 0; 2915 2916 activate_err: 2917 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0); 2918 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0); 2919 vmxnet3_free_irqs(adapter); 2920 irq_err: 2921 rq_err: 2922 /* free up buffers we allocated */ 2923 vmxnet3_rq_cleanup_all(adapter); 2924 return err; 2925 } 2926 2927 2928 void 2929 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter) 2930 { 2931 unsigned long flags; 2932 spin_lock_irqsave(&adapter->cmd_lock, flags); 2933 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); 2934 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2935 } 2936 2937 2938 int 2939 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter) 2940 { 2941 int i; 2942 unsigned long flags; 2943 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state)) 2944 return 0; 2945 2946 2947 spin_lock_irqsave(&adapter->cmd_lock, flags); 2948 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2949 VMXNET3_CMD_QUIESCE_DEV); 2950 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2951 vmxnet3_disable_all_intrs(adapter); 2952 2953 for (i = 0; i < adapter->num_rx_queues; i++) 2954 napi_disable(&adapter->rx_queue[i].napi); 2955 netif_tx_disable(adapter->netdev); 2956 adapter->link_speed = 0; 2957 netif_carrier_off(adapter->netdev); 2958 2959 vmxnet3_tq_cleanup_all(adapter); 2960 vmxnet3_rq_cleanup_all(adapter); 2961 vmxnet3_free_irqs(adapter); 2962 return 0; 2963 } 2964 2965 2966 static void 2967 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac) 2968 { 2969 u32 tmp; 2970 2971 tmp = *(u32 *)mac; 2972 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp); 2973 2974 tmp = (mac[5] << 8) | mac[4]; 2975 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp); 2976 } 2977 2978 2979 static int 2980 vmxnet3_set_mac_addr(struct net_device *netdev, void *p) 2981 { 2982 struct sockaddr *addr = p; 2983 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2984 2985 dev_addr_set(netdev, addr->sa_data); 2986 vmxnet3_write_mac_addr(adapter, addr->sa_data); 2987 2988 return 0; 2989 } 2990 2991 2992 /* ==================== initialization and cleanup routines ============ */ 2993 2994 static int 2995 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter) 2996 { 2997 int err; 2998 unsigned long mmio_start, mmio_len; 2999 struct pci_dev *pdev = adapter->pdev; 3000 3001 err = pci_enable_device(pdev); 3002 if (err) { 3003 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err); 3004 return err; 3005 } 3006 3007 err = pci_request_selected_regions(pdev, (1 << 2) - 1, 3008 vmxnet3_driver_name); 3009 if (err) { 3010 dev_err(&pdev->dev, 3011 "Failed to request region for adapter: error %d\n", err); 3012 goto err_enable_device; 3013 } 3014 3015 pci_set_master(pdev); 3016 3017 mmio_start = pci_resource_start(pdev, 0); 3018 mmio_len = pci_resource_len(pdev, 0); 3019 adapter->hw_addr0 = ioremap(mmio_start, mmio_len); 3020 if (!adapter->hw_addr0) { 3021 dev_err(&pdev->dev, "Failed to map bar0\n"); 3022 err = -EIO; 3023 goto err_ioremap; 3024 } 3025 3026 mmio_start = pci_resource_start(pdev, 1); 3027 mmio_len = pci_resource_len(pdev, 1); 3028 adapter->hw_addr1 = ioremap(mmio_start, mmio_len); 3029 if (!adapter->hw_addr1) { 3030 dev_err(&pdev->dev, "Failed to map bar1\n"); 3031 err = -EIO; 3032 goto err_bar1; 3033 } 3034 return 0; 3035 3036 err_bar1: 3037 iounmap(adapter->hw_addr0); 3038 err_ioremap: 3039 pci_release_selected_regions(pdev, (1 << 2) - 1); 3040 err_enable_device: 3041 pci_disable_device(pdev); 3042 return err; 3043 } 3044 3045 3046 static void 3047 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter) 3048 { 3049 BUG_ON(!adapter->pdev); 3050 3051 iounmap(adapter->hw_addr0); 3052 iounmap(adapter->hw_addr1); 3053 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1); 3054 pci_disable_device(adapter->pdev); 3055 } 3056 3057 3058 static void 3059 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter) 3060 { 3061 size_t sz, i, ring0_size, ring1_size, comp_size; 3062 /* With version7 ring1 will have only T0 buffers */ 3063 if (!VMXNET3_VERSION_GE_7(adapter)) { 3064 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE - 3065 VMXNET3_MAX_ETH_HDR_SIZE) { 3066 adapter->skb_buf_size = adapter->netdev->mtu + 3067 VMXNET3_MAX_ETH_HDR_SIZE; 3068 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE) 3069 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE; 3070 3071 adapter->rx_buf_per_pkt = 1; 3072 } else { 3073 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE; 3074 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE + 3075 VMXNET3_MAX_ETH_HDR_SIZE; 3076 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE; 3077 } 3078 } else { 3079 adapter->skb_buf_size = min((int)adapter->netdev->mtu + VMXNET3_MAX_ETH_HDR_SIZE, 3080 VMXNET3_MAX_SKB_BUF_SIZE); 3081 adapter->rx_buf_per_pkt = 1; 3082 adapter->ringBufSize.ring1BufSizeType0 = cpu_to_le16(adapter->skb_buf_size); 3083 adapter->ringBufSize.ring1BufSizeType1 = 0; 3084 adapter->ringBufSize.ring2BufSizeType1 = cpu_to_le16(PAGE_SIZE); 3085 } 3086 3087 /* 3088 * for simplicity, force the ring0 size to be a multiple of 3089 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN 3090 */ 3091 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN; 3092 ring0_size = adapter->rx_queue[0].rx_ring[0].size; 3093 ring0_size = (ring0_size + sz - 1) / sz * sz; 3094 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE / 3095 sz * sz); 3096 ring1_size = adapter->rx_queue[0].rx_ring[1].size; 3097 ring1_size = (ring1_size + sz - 1) / sz * sz; 3098 ring1_size = min_t(u32, ring1_size, VMXNET3_RX_RING2_MAX_SIZE / 3099 sz * sz); 3100 /* For v7 and later, keep ring size power of 2 for UPT */ 3101 if (VMXNET3_VERSION_GE_7(adapter)) { 3102 ring0_size = rounddown_pow_of_two(ring0_size); 3103 ring1_size = rounddown_pow_of_two(ring1_size); 3104 } 3105 comp_size = ring0_size + ring1_size; 3106 3107 for (i = 0; i < adapter->num_rx_queues; i++) { 3108 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3109 3110 rq->rx_ring[0].size = ring0_size; 3111 rq->rx_ring[1].size = ring1_size; 3112 rq->comp_ring.size = comp_size; 3113 } 3114 } 3115 3116 3117 int 3118 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size, 3119 u32 rx_ring_size, u32 rx_ring2_size, 3120 u16 txdata_desc_size, u16 rxdata_desc_size) 3121 { 3122 int err = 0, i; 3123 3124 for (i = 0; i < adapter->num_tx_queues; i++) { 3125 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 3126 tq->tx_ring.size = tx_ring_size; 3127 tq->data_ring.size = tx_ring_size; 3128 tq->comp_ring.size = tx_ring_size; 3129 tq->txdata_desc_size = txdata_desc_size; 3130 tq->shared = &adapter->tqd_start[i].ctrl; 3131 tq->stopped = true; 3132 tq->adapter = adapter; 3133 tq->qid = i; 3134 err = vmxnet3_tq_create(tq, adapter); 3135 /* 3136 * Too late to change num_tx_queues. We cannot do away with 3137 * lesser number of queues than what we asked for 3138 */ 3139 if (err) 3140 goto queue_err; 3141 } 3142 3143 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size; 3144 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size; 3145 vmxnet3_adjust_rx_ring_size(adapter); 3146 3147 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 3148 for (i = 0; i < adapter->num_rx_queues; i++) { 3149 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3150 /* qid and qid2 for rx queues will be assigned later when num 3151 * of rx queues is finalized after allocating intrs */ 3152 rq->shared = &adapter->rqd_start[i].ctrl; 3153 rq->adapter = adapter; 3154 rq->data_ring.desc_size = rxdata_desc_size; 3155 err = vmxnet3_rq_create(rq, adapter); 3156 if (err) { 3157 if (i == 0) { 3158 netdev_err(adapter->netdev, 3159 "Could not allocate any rx queues. " 3160 "Aborting.\n"); 3161 goto queue_err; 3162 } else { 3163 netdev_info(adapter->netdev, 3164 "Number of rx queues changed " 3165 "to : %d.\n", i); 3166 adapter->num_rx_queues = i; 3167 err = 0; 3168 break; 3169 } 3170 } 3171 } 3172 3173 if (!adapter->rxdataring_enabled) 3174 vmxnet3_rq_destroy_all_rxdataring(adapter); 3175 3176 return err; 3177 queue_err: 3178 vmxnet3_tq_destroy_all(adapter); 3179 return err; 3180 } 3181 3182 static int 3183 vmxnet3_open(struct net_device *netdev) 3184 { 3185 struct vmxnet3_adapter *adapter; 3186 int err, i; 3187 3188 adapter = netdev_priv(netdev); 3189 3190 for (i = 0; i < adapter->num_tx_queues; i++) 3191 spin_lock_init(&adapter->tx_queue[i].tx_lock); 3192 3193 if (VMXNET3_VERSION_GE_3(adapter)) { 3194 unsigned long flags; 3195 u16 txdata_desc_size; 3196 3197 spin_lock_irqsave(&adapter->cmd_lock, flags); 3198 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3199 VMXNET3_CMD_GET_TXDATA_DESC_SIZE); 3200 txdata_desc_size = VMXNET3_READ_BAR1_REG(adapter, 3201 VMXNET3_REG_CMD); 3202 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3203 3204 if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) || 3205 (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) || 3206 (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) { 3207 adapter->txdata_desc_size = 3208 sizeof(struct Vmxnet3_TxDataDesc); 3209 } else { 3210 adapter->txdata_desc_size = txdata_desc_size; 3211 } 3212 } else { 3213 adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc); 3214 } 3215 3216 err = vmxnet3_create_queues(adapter, 3217 adapter->tx_ring_size, 3218 adapter->rx_ring_size, 3219 adapter->rx_ring2_size, 3220 adapter->txdata_desc_size, 3221 adapter->rxdata_desc_size); 3222 if (err) 3223 goto queue_err; 3224 3225 err = vmxnet3_activate_dev(adapter); 3226 if (err) 3227 goto activate_err; 3228 3229 return 0; 3230 3231 activate_err: 3232 vmxnet3_rq_destroy_all(adapter); 3233 vmxnet3_tq_destroy_all(adapter); 3234 queue_err: 3235 return err; 3236 } 3237 3238 3239 static int 3240 vmxnet3_close(struct net_device *netdev) 3241 { 3242 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3243 3244 /* 3245 * Reset_work may be in the middle of resetting the device, wait for its 3246 * completion. 3247 */ 3248 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3249 usleep_range(1000, 2000); 3250 3251 vmxnet3_quiesce_dev(adapter); 3252 3253 vmxnet3_rq_destroy_all(adapter); 3254 vmxnet3_tq_destroy_all(adapter); 3255 3256 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3257 3258 3259 return 0; 3260 } 3261 3262 3263 void 3264 vmxnet3_force_close(struct vmxnet3_adapter *adapter) 3265 { 3266 int i; 3267 3268 /* 3269 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise 3270 * vmxnet3_close() will deadlock. 3271 */ 3272 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)); 3273 3274 /* we need to enable NAPI, otherwise dev_close will deadlock */ 3275 for (i = 0; i < adapter->num_rx_queues; i++) 3276 napi_enable(&adapter->rx_queue[i].napi); 3277 /* 3278 * Need to clear the quiesce bit to ensure that vmxnet3_close 3279 * can quiesce the device properly 3280 */ 3281 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3282 dev_close(adapter->netdev); 3283 } 3284 3285 3286 static int 3287 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu) 3288 { 3289 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3290 int err = 0; 3291 3292 netdev->mtu = new_mtu; 3293 3294 /* 3295 * Reset_work may be in the middle of resetting the device, wait for its 3296 * completion. 3297 */ 3298 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3299 usleep_range(1000, 2000); 3300 3301 if (netif_running(netdev)) { 3302 vmxnet3_quiesce_dev(adapter); 3303 vmxnet3_reset_dev(adapter); 3304 3305 /* we need to re-create the rx queue based on the new mtu */ 3306 vmxnet3_rq_destroy_all(adapter); 3307 vmxnet3_adjust_rx_ring_size(adapter); 3308 err = vmxnet3_rq_create_all(adapter); 3309 if (err) { 3310 netdev_err(netdev, 3311 "failed to re-create rx queues, " 3312 " error %d. Closing it.\n", err); 3313 goto out; 3314 } 3315 3316 err = vmxnet3_activate_dev(adapter); 3317 if (err) { 3318 netdev_err(netdev, 3319 "failed to re-activate, error %d. " 3320 "Closing it\n", err); 3321 goto out; 3322 } 3323 } 3324 3325 out: 3326 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3327 if (err) 3328 vmxnet3_force_close(adapter); 3329 3330 return err; 3331 } 3332 3333 3334 static void 3335 vmxnet3_declare_features(struct vmxnet3_adapter *adapter) 3336 { 3337 struct net_device *netdev = adapter->netdev; 3338 3339 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | 3340 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3341 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3342 NETIF_F_LRO | NETIF_F_HIGHDMA; 3343 3344 if (VMXNET3_VERSION_GE_4(adapter)) { 3345 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 3346 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3347 3348 netdev->hw_enc_features = NETIF_F_SG | NETIF_F_RXCSUM | 3349 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3350 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3351 NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL | 3352 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3353 } 3354 3355 if (VMXNET3_VERSION_GE_7(adapter)) { 3356 unsigned long flags; 3357 3358 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3359 VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) { 3360 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD; 3361 } 3362 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3363 VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) { 3364 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD; 3365 } 3366 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3367 VMXNET3_CAP_GENEVE_TSO)) { 3368 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO; 3369 } 3370 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3371 VMXNET3_CAP_VXLAN_TSO)) { 3372 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO; 3373 } 3374 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3375 VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) { 3376 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD; 3377 } 3378 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3379 VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) { 3380 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD; 3381 } 3382 3383 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3384 spin_lock_irqsave(&adapter->cmd_lock, flags); 3385 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3386 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3387 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3388 3389 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) && 3390 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) && 3391 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_TSO)) && 3392 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_TSO))) { 3393 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL; 3394 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL; 3395 } 3396 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) && 3397 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) { 3398 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3399 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3400 } 3401 } 3402 3403 netdev->vlan_features = netdev->hw_features & 3404 ~(NETIF_F_HW_VLAN_CTAG_TX | 3405 NETIF_F_HW_VLAN_CTAG_RX); 3406 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 3407 } 3408 3409 3410 static void 3411 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac) 3412 { 3413 u32 tmp; 3414 3415 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL); 3416 *(u32 *)mac = tmp; 3417 3418 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH); 3419 mac[4] = tmp & 0xff; 3420 mac[5] = (tmp >> 8) & 0xff; 3421 } 3422 3423 #ifdef CONFIG_PCI_MSI 3424 3425 /* 3426 * Enable MSIx vectors. 3427 * Returns : 3428 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required 3429 * were enabled. 3430 * number of vectors which were enabled otherwise (this number is greater 3431 * than VMXNET3_LINUX_MIN_MSIX_VECT) 3432 */ 3433 3434 static int 3435 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec) 3436 { 3437 int ret = pci_enable_msix_range(adapter->pdev, 3438 adapter->intr.msix_entries, nvec, nvec); 3439 3440 if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) { 3441 dev_err(&adapter->netdev->dev, 3442 "Failed to enable %d MSI-X, trying %d\n", 3443 nvec, VMXNET3_LINUX_MIN_MSIX_VECT); 3444 3445 ret = pci_enable_msix_range(adapter->pdev, 3446 adapter->intr.msix_entries, 3447 VMXNET3_LINUX_MIN_MSIX_VECT, 3448 VMXNET3_LINUX_MIN_MSIX_VECT); 3449 } 3450 3451 if (ret < 0) { 3452 dev_err(&adapter->netdev->dev, 3453 "Failed to enable MSI-X, error: %d\n", ret); 3454 } 3455 3456 return ret; 3457 } 3458 3459 3460 #endif /* CONFIG_PCI_MSI */ 3461 3462 static void 3463 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter) 3464 { 3465 u32 cfg; 3466 unsigned long flags; 3467 3468 /* intr settings */ 3469 spin_lock_irqsave(&adapter->cmd_lock, flags); 3470 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3471 VMXNET3_CMD_GET_CONF_INTR); 3472 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3473 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3474 adapter->intr.type = cfg & 0x3; 3475 adapter->intr.mask_mode = (cfg >> 2) & 0x3; 3476 3477 if (adapter->intr.type == VMXNET3_IT_AUTO) { 3478 adapter->intr.type = VMXNET3_IT_MSIX; 3479 } 3480 3481 #ifdef CONFIG_PCI_MSI 3482 if (adapter->intr.type == VMXNET3_IT_MSIX) { 3483 int i, nvec, nvec_allocated; 3484 3485 nvec = adapter->share_intr == VMXNET3_INTR_TXSHARE ? 3486 1 : adapter->num_tx_queues; 3487 nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ? 3488 0 : adapter->num_rx_queues; 3489 nvec += 1; /* for link event */ 3490 nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ? 3491 nvec : VMXNET3_LINUX_MIN_MSIX_VECT; 3492 3493 for (i = 0; i < nvec; i++) 3494 adapter->intr.msix_entries[i].entry = i; 3495 3496 nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec); 3497 if (nvec_allocated < 0) 3498 goto msix_err; 3499 3500 /* If we cannot allocate one MSIx vector per queue 3501 * then limit the number of rx queues to 1 3502 */ 3503 if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT && 3504 nvec != VMXNET3_LINUX_MIN_MSIX_VECT) { 3505 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE 3506 || adapter->num_rx_queues != 1) { 3507 adapter->share_intr = VMXNET3_INTR_TXSHARE; 3508 netdev_err(adapter->netdev, 3509 "Number of rx queues : 1\n"); 3510 adapter->num_rx_queues = 1; 3511 } 3512 } 3513 3514 adapter->intr.num_intrs = nvec_allocated; 3515 return; 3516 3517 msix_err: 3518 /* If we cannot allocate MSIx vectors use only one rx queue */ 3519 dev_info(&adapter->pdev->dev, 3520 "Failed to enable MSI-X, error %d. " 3521 "Limiting #rx queues to 1, try MSI.\n", nvec_allocated); 3522 3523 adapter->intr.type = VMXNET3_IT_MSI; 3524 } 3525 3526 if (adapter->intr.type == VMXNET3_IT_MSI) { 3527 if (!pci_enable_msi(adapter->pdev)) { 3528 adapter->num_rx_queues = 1; 3529 adapter->intr.num_intrs = 1; 3530 return; 3531 } 3532 } 3533 #endif /* CONFIG_PCI_MSI */ 3534 3535 adapter->num_rx_queues = 1; 3536 dev_info(&adapter->netdev->dev, 3537 "Using INTx interrupt, #Rx queues: 1.\n"); 3538 adapter->intr.type = VMXNET3_IT_INTX; 3539 3540 /* INT-X related setting */ 3541 adapter->intr.num_intrs = 1; 3542 } 3543 3544 3545 static void 3546 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter) 3547 { 3548 if (adapter->intr.type == VMXNET3_IT_MSIX) 3549 pci_disable_msix(adapter->pdev); 3550 else if (adapter->intr.type == VMXNET3_IT_MSI) 3551 pci_disable_msi(adapter->pdev); 3552 else 3553 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX); 3554 } 3555 3556 3557 static void 3558 vmxnet3_tx_timeout(struct net_device *netdev, unsigned int txqueue) 3559 { 3560 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3561 adapter->tx_timeout_count++; 3562 3563 netdev_err(adapter->netdev, "tx hang\n"); 3564 schedule_work(&adapter->work); 3565 } 3566 3567 3568 static void 3569 vmxnet3_reset_work(struct work_struct *data) 3570 { 3571 struct vmxnet3_adapter *adapter; 3572 3573 adapter = container_of(data, struct vmxnet3_adapter, work); 3574 3575 /* if another thread is resetting the device, no need to proceed */ 3576 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3577 return; 3578 3579 /* if the device is closed, we must leave it alone */ 3580 rtnl_lock(); 3581 if (netif_running(adapter->netdev)) { 3582 netdev_notice(adapter->netdev, "resetting\n"); 3583 vmxnet3_quiesce_dev(adapter); 3584 vmxnet3_reset_dev(adapter); 3585 vmxnet3_activate_dev(adapter); 3586 } else { 3587 netdev_info(adapter->netdev, "already closed\n"); 3588 } 3589 rtnl_unlock(); 3590 3591 netif_wake_queue(adapter->netdev); 3592 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3593 } 3594 3595 3596 static int 3597 vmxnet3_probe_device(struct pci_dev *pdev, 3598 const struct pci_device_id *id) 3599 { 3600 static const struct net_device_ops vmxnet3_netdev_ops = { 3601 .ndo_open = vmxnet3_open, 3602 .ndo_stop = vmxnet3_close, 3603 .ndo_start_xmit = vmxnet3_xmit_frame, 3604 .ndo_set_mac_address = vmxnet3_set_mac_addr, 3605 .ndo_change_mtu = vmxnet3_change_mtu, 3606 .ndo_fix_features = vmxnet3_fix_features, 3607 .ndo_set_features = vmxnet3_set_features, 3608 .ndo_features_check = vmxnet3_features_check, 3609 .ndo_get_stats64 = vmxnet3_get_stats64, 3610 .ndo_tx_timeout = vmxnet3_tx_timeout, 3611 .ndo_set_rx_mode = vmxnet3_set_mc, 3612 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid, 3613 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid, 3614 #ifdef CONFIG_NET_POLL_CONTROLLER 3615 .ndo_poll_controller = vmxnet3_netpoll, 3616 #endif 3617 }; 3618 int err; 3619 u32 ver; 3620 struct net_device *netdev; 3621 struct vmxnet3_adapter *adapter; 3622 u8 mac[ETH_ALEN]; 3623 int size; 3624 int num_tx_queues; 3625 int num_rx_queues; 3626 int queues; 3627 unsigned long flags; 3628 3629 if (!pci_msi_enabled()) 3630 enable_mq = 0; 3631 3632 #ifdef VMXNET3_RSS 3633 if (enable_mq) 3634 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 3635 (int)num_online_cpus()); 3636 else 3637 #endif 3638 num_rx_queues = 1; 3639 3640 if (enable_mq) 3641 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES, 3642 (int)num_online_cpus()); 3643 else 3644 num_tx_queues = 1; 3645 3646 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter), 3647 max(num_tx_queues, num_rx_queues)); 3648 if (!netdev) 3649 return -ENOMEM; 3650 3651 pci_set_drvdata(pdev, netdev); 3652 adapter = netdev_priv(netdev); 3653 adapter->netdev = netdev; 3654 adapter->pdev = pdev; 3655 3656 adapter->tx_ring_size = VMXNET3_DEF_TX_RING_SIZE; 3657 adapter->rx_ring_size = VMXNET3_DEF_RX_RING_SIZE; 3658 adapter->rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE; 3659 3660 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3661 if (err) { 3662 dev_err(&pdev->dev, "dma_set_mask failed\n"); 3663 goto err_set_mask; 3664 } 3665 3666 spin_lock_init(&adapter->cmd_lock); 3667 adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter, 3668 sizeof(struct vmxnet3_adapter), 3669 DMA_TO_DEVICE); 3670 if (dma_mapping_error(&adapter->pdev->dev, adapter->adapter_pa)) { 3671 dev_err(&pdev->dev, "Failed to map dma\n"); 3672 err = -EFAULT; 3673 goto err_set_mask; 3674 } 3675 adapter->shared = dma_alloc_coherent( 3676 &adapter->pdev->dev, 3677 sizeof(struct Vmxnet3_DriverShared), 3678 &adapter->shared_pa, GFP_KERNEL); 3679 if (!adapter->shared) { 3680 dev_err(&pdev->dev, "Failed to allocate memory\n"); 3681 err = -ENOMEM; 3682 goto err_alloc_shared; 3683 } 3684 3685 err = vmxnet3_alloc_pci_resources(adapter); 3686 if (err < 0) 3687 goto err_alloc_pci; 3688 3689 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS); 3690 if (ver & (1 << VMXNET3_REV_7)) { 3691 VMXNET3_WRITE_BAR1_REG(adapter, 3692 VMXNET3_REG_VRRS, 3693 1 << VMXNET3_REV_7); 3694 adapter->version = VMXNET3_REV_7 + 1; 3695 } else if (ver & (1 << VMXNET3_REV_6)) { 3696 VMXNET3_WRITE_BAR1_REG(adapter, 3697 VMXNET3_REG_VRRS, 3698 1 << VMXNET3_REV_6); 3699 adapter->version = VMXNET3_REV_6 + 1; 3700 } else if (ver & (1 << VMXNET3_REV_5)) { 3701 VMXNET3_WRITE_BAR1_REG(adapter, 3702 VMXNET3_REG_VRRS, 3703 1 << VMXNET3_REV_5); 3704 adapter->version = VMXNET3_REV_5 + 1; 3705 } else if (ver & (1 << VMXNET3_REV_4)) { 3706 VMXNET3_WRITE_BAR1_REG(adapter, 3707 VMXNET3_REG_VRRS, 3708 1 << VMXNET3_REV_4); 3709 adapter->version = VMXNET3_REV_4 + 1; 3710 } else if (ver & (1 << VMXNET3_REV_3)) { 3711 VMXNET3_WRITE_BAR1_REG(adapter, 3712 VMXNET3_REG_VRRS, 3713 1 << VMXNET3_REV_3); 3714 adapter->version = VMXNET3_REV_3 + 1; 3715 } else if (ver & (1 << VMXNET3_REV_2)) { 3716 VMXNET3_WRITE_BAR1_REG(adapter, 3717 VMXNET3_REG_VRRS, 3718 1 << VMXNET3_REV_2); 3719 adapter->version = VMXNET3_REV_2 + 1; 3720 } else if (ver & (1 << VMXNET3_REV_1)) { 3721 VMXNET3_WRITE_BAR1_REG(adapter, 3722 VMXNET3_REG_VRRS, 3723 1 << VMXNET3_REV_1); 3724 adapter->version = VMXNET3_REV_1 + 1; 3725 } else { 3726 dev_err(&pdev->dev, 3727 "Incompatible h/w version (0x%x) for adapter\n", ver); 3728 err = -EBUSY; 3729 goto err_ver; 3730 } 3731 dev_dbg(&pdev->dev, "Using device version %d\n", adapter->version); 3732 3733 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS); 3734 if (ver & 1) { 3735 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1); 3736 } else { 3737 dev_err(&pdev->dev, 3738 "Incompatible upt version (0x%x) for adapter\n", ver); 3739 err = -EBUSY; 3740 goto err_ver; 3741 } 3742 3743 if (VMXNET3_VERSION_GE_7(adapter)) { 3744 adapter->devcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DCR); 3745 adapter->ptcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_PTCR); 3746 if (adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 3747 adapter->dev_caps[0] = adapter->devcap_supported[0] & 3748 (1UL << VMXNET3_CAP_LARGE_BAR); 3749 } 3750 if (!(adapter->ptcap_supported[0] & (1UL << VMXNET3_DCR_ERROR)) && 3751 adapter->ptcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP) && 3752 adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) { 3753 adapter->dev_caps[0] |= adapter->devcap_supported[0] & 3754 (1UL << VMXNET3_CAP_OOORX_COMP); 3755 } 3756 if (adapter->dev_caps[0]) 3757 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3758 3759 spin_lock_irqsave(&adapter->cmd_lock, flags); 3760 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3761 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3762 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3763 } 3764 3765 if (VMXNET3_VERSION_GE_7(adapter) && 3766 adapter->dev_caps[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 3767 adapter->tx_prod_offset = VMXNET3_REG_LB_TXPROD; 3768 adapter->rx_prod_offset = VMXNET3_REG_LB_RXPROD; 3769 adapter->rx_prod2_offset = VMXNET3_REG_LB_RXPROD2; 3770 } else { 3771 adapter->tx_prod_offset = VMXNET3_REG_TXPROD; 3772 adapter->rx_prod_offset = VMXNET3_REG_RXPROD; 3773 adapter->rx_prod2_offset = VMXNET3_REG_RXPROD2; 3774 } 3775 3776 if (VMXNET3_VERSION_GE_6(adapter)) { 3777 spin_lock_irqsave(&adapter->cmd_lock, flags); 3778 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3779 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 3780 queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3781 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3782 if (queues > 0) { 3783 adapter->num_rx_queues = min(num_rx_queues, ((queues >> 8) & 0xff)); 3784 adapter->num_tx_queues = min(num_tx_queues, (queues & 0xff)); 3785 } else { 3786 adapter->num_rx_queues = min(num_rx_queues, 3787 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3788 adapter->num_tx_queues = min(num_tx_queues, 3789 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 3790 } 3791 if (adapter->num_rx_queues > VMXNET3_MAX_RX_QUEUES || 3792 adapter->num_tx_queues > VMXNET3_MAX_TX_QUEUES) { 3793 adapter->queuesExtEnabled = true; 3794 } else { 3795 adapter->queuesExtEnabled = false; 3796 } 3797 } else { 3798 adapter->queuesExtEnabled = false; 3799 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 3800 num_tx_queues = rounddown_pow_of_two(num_tx_queues); 3801 adapter->num_rx_queues = min(num_rx_queues, 3802 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3803 adapter->num_tx_queues = min(num_tx_queues, 3804 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 3805 } 3806 dev_info(&pdev->dev, 3807 "# of Tx queues : %d, # of Rx queues : %d\n", 3808 adapter->num_tx_queues, adapter->num_rx_queues); 3809 3810 adapter->rx_buf_per_pkt = 1; 3811 3812 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 3813 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues; 3814 adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size, 3815 &adapter->queue_desc_pa, 3816 GFP_KERNEL); 3817 3818 if (!adapter->tqd_start) { 3819 dev_err(&pdev->dev, "Failed to allocate memory\n"); 3820 err = -ENOMEM; 3821 goto err_ver; 3822 } 3823 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start + 3824 adapter->num_tx_queues); 3825 3826 adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev, 3827 sizeof(struct Vmxnet3_PMConf), 3828 &adapter->pm_conf_pa, 3829 GFP_KERNEL); 3830 if (adapter->pm_conf == NULL) { 3831 err = -ENOMEM; 3832 goto err_alloc_pm; 3833 } 3834 3835 #ifdef VMXNET3_RSS 3836 3837 adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev, 3838 sizeof(struct UPT1_RSSConf), 3839 &adapter->rss_conf_pa, 3840 GFP_KERNEL); 3841 if (adapter->rss_conf == NULL) { 3842 err = -ENOMEM; 3843 goto err_alloc_rss; 3844 } 3845 #endif /* VMXNET3_RSS */ 3846 3847 if (VMXNET3_VERSION_GE_3(adapter)) { 3848 adapter->coal_conf = 3849 dma_alloc_coherent(&adapter->pdev->dev, 3850 sizeof(struct Vmxnet3_CoalesceScheme) 3851 , 3852 &adapter->coal_conf_pa, 3853 GFP_KERNEL); 3854 if (!adapter->coal_conf) { 3855 err = -ENOMEM; 3856 goto err_coal_conf; 3857 } 3858 adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED; 3859 adapter->default_coal_mode = true; 3860 } 3861 3862 if (VMXNET3_VERSION_GE_4(adapter)) { 3863 adapter->default_rss_fields = true; 3864 adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT; 3865 } 3866 3867 SET_NETDEV_DEV(netdev, &pdev->dev); 3868 vmxnet3_declare_features(adapter); 3869 3870 adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ? 3871 VMXNET3_DEF_RXDATA_DESC_SIZE : 0; 3872 3873 if (adapter->num_tx_queues == adapter->num_rx_queues) 3874 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE; 3875 else 3876 adapter->share_intr = VMXNET3_INTR_DONTSHARE; 3877 3878 vmxnet3_alloc_intr_resources(adapter); 3879 3880 #ifdef VMXNET3_RSS 3881 if (adapter->num_rx_queues > 1 && 3882 adapter->intr.type == VMXNET3_IT_MSIX) { 3883 adapter->rss = true; 3884 netdev->hw_features |= NETIF_F_RXHASH; 3885 netdev->features |= NETIF_F_RXHASH; 3886 dev_dbg(&pdev->dev, "RSS is enabled.\n"); 3887 } else { 3888 adapter->rss = false; 3889 } 3890 #endif 3891 3892 vmxnet3_read_mac_addr(adapter, mac); 3893 dev_addr_set(netdev, mac); 3894 3895 netdev->netdev_ops = &vmxnet3_netdev_ops; 3896 vmxnet3_set_ethtool_ops(netdev); 3897 netdev->watchdog_timeo = 5 * HZ; 3898 3899 /* MTU range: 60 - 9190 */ 3900 netdev->min_mtu = VMXNET3_MIN_MTU; 3901 if (VMXNET3_VERSION_GE_6(adapter)) 3902 netdev->max_mtu = VMXNET3_V6_MAX_MTU; 3903 else 3904 netdev->max_mtu = VMXNET3_MAX_MTU; 3905 3906 INIT_WORK(&adapter->work, vmxnet3_reset_work); 3907 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3908 3909 if (adapter->intr.type == VMXNET3_IT_MSIX) { 3910 int i; 3911 for (i = 0; i < adapter->num_rx_queues; i++) { 3912 netif_napi_add(adapter->netdev, 3913 &adapter->rx_queue[i].napi, 3914 vmxnet3_poll_rx_only); 3915 } 3916 } else { 3917 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi, 3918 vmxnet3_poll); 3919 } 3920 3921 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues); 3922 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues); 3923 3924 netif_carrier_off(netdev); 3925 err = register_netdev(netdev); 3926 3927 if (err) { 3928 dev_err(&pdev->dev, "Failed to register adapter\n"); 3929 goto err_register; 3930 } 3931 3932 vmxnet3_check_link(adapter, false); 3933 return 0; 3934 3935 err_register: 3936 if (VMXNET3_VERSION_GE_3(adapter)) { 3937 dma_free_coherent(&adapter->pdev->dev, 3938 sizeof(struct Vmxnet3_CoalesceScheme), 3939 adapter->coal_conf, adapter->coal_conf_pa); 3940 } 3941 vmxnet3_free_intr_resources(adapter); 3942 err_coal_conf: 3943 #ifdef VMXNET3_RSS 3944 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 3945 adapter->rss_conf, adapter->rss_conf_pa); 3946 err_alloc_rss: 3947 #endif 3948 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 3949 adapter->pm_conf, adapter->pm_conf_pa); 3950 err_alloc_pm: 3951 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 3952 adapter->queue_desc_pa); 3953 err_ver: 3954 vmxnet3_free_pci_resources(adapter); 3955 err_alloc_pci: 3956 dma_free_coherent(&adapter->pdev->dev, 3957 sizeof(struct Vmxnet3_DriverShared), 3958 adapter->shared, adapter->shared_pa); 3959 err_alloc_shared: 3960 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 3961 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 3962 err_set_mask: 3963 free_netdev(netdev); 3964 return err; 3965 } 3966 3967 3968 static void 3969 vmxnet3_remove_device(struct pci_dev *pdev) 3970 { 3971 struct net_device *netdev = pci_get_drvdata(pdev); 3972 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3973 int size = 0; 3974 int num_rx_queues, rx_queues; 3975 unsigned long flags; 3976 3977 #ifdef VMXNET3_RSS 3978 if (enable_mq) 3979 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 3980 (int)num_online_cpus()); 3981 else 3982 #endif 3983 num_rx_queues = 1; 3984 if (!VMXNET3_VERSION_GE_6(adapter)) { 3985 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 3986 } 3987 if (VMXNET3_VERSION_GE_6(adapter)) { 3988 spin_lock_irqsave(&adapter->cmd_lock, flags); 3989 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3990 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 3991 rx_queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3992 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3993 if (rx_queues > 0) 3994 rx_queues = (rx_queues >> 8) & 0xff; 3995 else 3996 rx_queues = min(num_rx_queues, VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3997 num_rx_queues = min(num_rx_queues, rx_queues); 3998 } else { 3999 num_rx_queues = min(num_rx_queues, 4000 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4001 } 4002 4003 cancel_work_sync(&adapter->work); 4004 4005 unregister_netdev(netdev); 4006 4007 vmxnet3_free_intr_resources(adapter); 4008 vmxnet3_free_pci_resources(adapter); 4009 if (VMXNET3_VERSION_GE_3(adapter)) { 4010 dma_free_coherent(&adapter->pdev->dev, 4011 sizeof(struct Vmxnet3_CoalesceScheme), 4012 adapter->coal_conf, adapter->coal_conf_pa); 4013 } 4014 #ifdef VMXNET3_RSS 4015 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 4016 adapter->rss_conf, adapter->rss_conf_pa); 4017 #endif 4018 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 4019 adapter->pm_conf, adapter->pm_conf_pa); 4020 4021 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 4022 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues; 4023 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 4024 adapter->queue_desc_pa); 4025 dma_free_coherent(&adapter->pdev->dev, 4026 sizeof(struct Vmxnet3_DriverShared), 4027 adapter->shared, adapter->shared_pa); 4028 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 4029 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 4030 free_netdev(netdev); 4031 } 4032 4033 static void vmxnet3_shutdown_device(struct pci_dev *pdev) 4034 { 4035 struct net_device *netdev = pci_get_drvdata(pdev); 4036 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4037 unsigned long flags; 4038 4039 /* Reset_work may be in the middle of resetting the device, wait for its 4040 * completion. 4041 */ 4042 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 4043 usleep_range(1000, 2000); 4044 4045 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, 4046 &adapter->state)) { 4047 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4048 return; 4049 } 4050 spin_lock_irqsave(&adapter->cmd_lock, flags); 4051 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4052 VMXNET3_CMD_QUIESCE_DEV); 4053 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4054 vmxnet3_disable_all_intrs(adapter); 4055 4056 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4057 } 4058 4059 4060 #ifdef CONFIG_PM 4061 4062 static int 4063 vmxnet3_suspend(struct device *device) 4064 { 4065 struct pci_dev *pdev = to_pci_dev(device); 4066 struct net_device *netdev = pci_get_drvdata(pdev); 4067 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4068 struct Vmxnet3_PMConf *pmConf; 4069 struct ethhdr *ehdr; 4070 struct arphdr *ahdr; 4071 u8 *arpreq; 4072 struct in_device *in_dev; 4073 struct in_ifaddr *ifa; 4074 unsigned long flags; 4075 int i = 0; 4076 4077 if (!netif_running(netdev)) 4078 return 0; 4079 4080 for (i = 0; i < adapter->num_rx_queues; i++) 4081 napi_disable(&adapter->rx_queue[i].napi); 4082 4083 vmxnet3_disable_all_intrs(adapter); 4084 vmxnet3_free_irqs(adapter); 4085 vmxnet3_free_intr_resources(adapter); 4086 4087 netif_device_detach(netdev); 4088 4089 /* Create wake-up filters. */ 4090 pmConf = adapter->pm_conf; 4091 memset(pmConf, 0, sizeof(*pmConf)); 4092 4093 if (adapter->wol & WAKE_UCAST) { 4094 pmConf->filters[i].patternSize = ETH_ALEN; 4095 pmConf->filters[i].maskSize = 1; 4096 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN); 4097 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */ 4098 4099 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4100 i++; 4101 } 4102 4103 if (adapter->wol & WAKE_ARP) { 4104 rcu_read_lock(); 4105 4106 in_dev = __in_dev_get_rcu(netdev); 4107 if (!in_dev) { 4108 rcu_read_unlock(); 4109 goto skip_arp; 4110 } 4111 4112 ifa = rcu_dereference(in_dev->ifa_list); 4113 if (!ifa) { 4114 rcu_read_unlock(); 4115 goto skip_arp; 4116 } 4117 4118 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/ 4119 sizeof(struct arphdr) + /* ARP header */ 4120 2 * ETH_ALEN + /* 2 Ethernet addresses*/ 4121 2 * sizeof(u32); /*2 IPv4 addresses */ 4122 pmConf->filters[i].maskSize = 4123 (pmConf->filters[i].patternSize - 1) / 8 + 1; 4124 4125 /* ETH_P_ARP in Ethernet header. */ 4126 ehdr = (struct ethhdr *)pmConf->filters[i].pattern; 4127 ehdr->h_proto = htons(ETH_P_ARP); 4128 4129 /* ARPOP_REQUEST in ARP header. */ 4130 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN]; 4131 ahdr->ar_op = htons(ARPOP_REQUEST); 4132 arpreq = (u8 *)(ahdr + 1); 4133 4134 /* The Unicast IPv4 address in 'tip' field. */ 4135 arpreq += 2 * ETH_ALEN + sizeof(u32); 4136 *(__be32 *)arpreq = ifa->ifa_address; 4137 4138 rcu_read_unlock(); 4139 4140 /* The mask for the relevant bits. */ 4141 pmConf->filters[i].mask[0] = 0x00; 4142 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */ 4143 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */ 4144 pmConf->filters[i].mask[3] = 0x00; 4145 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */ 4146 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */ 4147 4148 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4149 i++; 4150 } 4151 4152 skip_arp: 4153 if (adapter->wol & WAKE_MAGIC) 4154 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC; 4155 4156 pmConf->numFilters = i; 4157 4158 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1); 4159 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof( 4160 *pmConf)); 4161 adapter->shared->devRead.pmConfDesc.confPA = 4162 cpu_to_le64(adapter->pm_conf_pa); 4163 4164 spin_lock_irqsave(&adapter->cmd_lock, flags); 4165 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4166 VMXNET3_CMD_UPDATE_PMCFG); 4167 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4168 4169 pci_save_state(pdev); 4170 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND), 4171 adapter->wol); 4172 pci_disable_device(pdev); 4173 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND)); 4174 4175 return 0; 4176 } 4177 4178 4179 static int 4180 vmxnet3_resume(struct device *device) 4181 { 4182 int err; 4183 unsigned long flags; 4184 struct pci_dev *pdev = to_pci_dev(device); 4185 struct net_device *netdev = pci_get_drvdata(pdev); 4186 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4187 4188 if (!netif_running(netdev)) 4189 return 0; 4190 4191 pci_set_power_state(pdev, PCI_D0); 4192 pci_restore_state(pdev); 4193 err = pci_enable_device_mem(pdev); 4194 if (err != 0) 4195 return err; 4196 4197 pci_enable_wake(pdev, PCI_D0, 0); 4198 4199 vmxnet3_alloc_intr_resources(adapter); 4200 4201 /* During hibernate and suspend, device has to be reinitialized as the 4202 * device state need not be preserved. 4203 */ 4204 4205 /* Need not check adapter state as other reset tasks cannot run during 4206 * device resume. 4207 */ 4208 spin_lock_irqsave(&adapter->cmd_lock, flags); 4209 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4210 VMXNET3_CMD_QUIESCE_DEV); 4211 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4212 vmxnet3_tq_cleanup_all(adapter); 4213 vmxnet3_rq_cleanup_all(adapter); 4214 4215 vmxnet3_reset_dev(adapter); 4216 err = vmxnet3_activate_dev(adapter); 4217 if (err != 0) { 4218 netdev_err(netdev, 4219 "failed to re-activate on resume, error: %d", err); 4220 vmxnet3_force_close(adapter); 4221 return err; 4222 } 4223 netif_device_attach(netdev); 4224 4225 return 0; 4226 } 4227 4228 static const struct dev_pm_ops vmxnet3_pm_ops = { 4229 .suspend = vmxnet3_suspend, 4230 .resume = vmxnet3_resume, 4231 .freeze = vmxnet3_suspend, 4232 .restore = vmxnet3_resume, 4233 }; 4234 #endif 4235 4236 static struct pci_driver vmxnet3_driver = { 4237 .name = vmxnet3_driver_name, 4238 .id_table = vmxnet3_pciid_table, 4239 .probe = vmxnet3_probe_device, 4240 .remove = vmxnet3_remove_device, 4241 .shutdown = vmxnet3_shutdown_device, 4242 #ifdef CONFIG_PM 4243 .driver.pm = &vmxnet3_pm_ops, 4244 #endif 4245 }; 4246 4247 4248 static int __init 4249 vmxnet3_init_module(void) 4250 { 4251 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC, 4252 VMXNET3_DRIVER_VERSION_REPORT); 4253 return pci_register_driver(&vmxnet3_driver); 4254 } 4255 4256 module_init(vmxnet3_init_module); 4257 4258 4259 static void 4260 vmxnet3_exit_module(void) 4261 { 4262 pci_unregister_driver(&vmxnet3_driver); 4263 } 4264 4265 module_exit(vmxnet3_exit_module); 4266 4267 MODULE_AUTHOR("VMware, Inc."); 4268 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC); 4269 MODULE_LICENSE("GPL v2"); 4270 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING); 4271