1 /* 2 * Linux driver for VMware's vmxnet3 ethernet NIC. 3 * 4 * Copyright (C) 2008-2022, VMware, Inc. All Rights Reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the 8 * Free Software Foundation; version 2 of the License and no later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 13 * NON INFRINGEMENT. See the GNU General Public License for more 14 * details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * The full GNU General Public License is included in this distribution in 21 * the file called "COPYING". 22 * 23 * Maintained by: pv-drivers@vmware.com 24 * 25 */ 26 27 #include <linux/module.h> 28 #include <net/ip6_checksum.h> 29 30 #include "vmxnet3_int.h" 31 32 char vmxnet3_driver_name[] = "vmxnet3"; 33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver" 34 35 /* 36 * PCI Device ID Table 37 * Last entry must be all 0s 38 */ 39 static const struct pci_device_id vmxnet3_pciid_table[] = { 40 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)}, 41 {0} 42 }; 43 44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table); 45 46 static int enable_mq = 1; 47 48 static void 49 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac); 50 51 /* 52 * Enable/Disable the given intr 53 */ 54 static void 55 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 56 { 57 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0); 58 } 59 60 61 static void 62 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 63 { 64 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1); 65 } 66 67 68 /* 69 * Enable/Disable all intrs used by the device 70 */ 71 static void 72 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter) 73 { 74 int i; 75 76 for (i = 0; i < adapter->intr.num_intrs; i++) 77 vmxnet3_enable_intr(adapter, i); 78 adapter->shared->devRead.intrConf.intrCtrl &= 79 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 80 } 81 82 83 static void 84 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter) 85 { 86 int i; 87 88 adapter->shared->devRead.intrConf.intrCtrl |= 89 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 90 for (i = 0; i < adapter->intr.num_intrs; i++) 91 vmxnet3_disable_intr(adapter, i); 92 } 93 94 95 static void 96 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events) 97 { 98 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events); 99 } 100 101 102 static bool 103 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 104 { 105 return tq->stopped; 106 } 107 108 109 static void 110 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 111 { 112 tq->stopped = false; 113 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue); 114 } 115 116 117 static void 118 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 119 { 120 tq->stopped = false; 121 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 122 } 123 124 125 static void 126 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 127 { 128 tq->stopped = true; 129 tq->num_stop++; 130 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 131 } 132 133 /* Check if capability is supported by UPT device or 134 * UPT is even requested 135 */ 136 bool 137 vmxnet3_check_ptcapability(u32 cap_supported, u32 cap) 138 { 139 if (cap_supported & (1UL << VMXNET3_DCR_ERROR) || 140 cap_supported & (1UL << cap)) { 141 return true; 142 } 143 144 return false; 145 } 146 147 148 /* 149 * Check the link state. This may start or stop the tx queue. 150 */ 151 static void 152 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue) 153 { 154 u32 ret; 155 int i; 156 unsigned long flags; 157 158 spin_lock_irqsave(&adapter->cmd_lock, flags); 159 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 160 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 161 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 162 163 adapter->link_speed = ret >> 16; 164 if (ret & 1) { /* Link is up. */ 165 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n", 166 adapter->link_speed); 167 netif_carrier_on(adapter->netdev); 168 169 if (affectTxQueue) { 170 for (i = 0; i < adapter->num_tx_queues; i++) 171 vmxnet3_tq_start(&adapter->tx_queue[i], 172 adapter); 173 } 174 } else { 175 netdev_info(adapter->netdev, "NIC Link is Down\n"); 176 netif_carrier_off(adapter->netdev); 177 178 if (affectTxQueue) { 179 for (i = 0; i < adapter->num_tx_queues; i++) 180 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter); 181 } 182 } 183 } 184 185 static void 186 vmxnet3_process_events(struct vmxnet3_adapter *adapter) 187 { 188 int i; 189 unsigned long flags; 190 u32 events = le32_to_cpu(adapter->shared->ecr); 191 if (!events) 192 return; 193 194 vmxnet3_ack_events(adapter, events); 195 196 /* Check if link state has changed */ 197 if (events & VMXNET3_ECR_LINK) 198 vmxnet3_check_link(adapter, true); 199 200 /* Check if there is an error on xmit/recv queues */ 201 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { 202 spin_lock_irqsave(&adapter->cmd_lock, flags); 203 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 204 VMXNET3_CMD_GET_QUEUE_STATUS); 205 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 206 207 for (i = 0; i < adapter->num_tx_queues; i++) 208 if (adapter->tqd_start[i].status.stopped) 209 dev_err(&adapter->netdev->dev, 210 "%s: tq[%d] error 0x%x\n", 211 adapter->netdev->name, i, le32_to_cpu( 212 adapter->tqd_start[i].status.error)); 213 for (i = 0; i < adapter->num_rx_queues; i++) 214 if (adapter->rqd_start[i].status.stopped) 215 dev_err(&adapter->netdev->dev, 216 "%s: rq[%d] error 0x%x\n", 217 adapter->netdev->name, i, 218 adapter->rqd_start[i].status.error); 219 220 schedule_work(&adapter->work); 221 } 222 } 223 224 #ifdef __BIG_ENDIAN_BITFIELD 225 /* 226 * The device expects the bitfields in shared structures to be written in 227 * little endian. When CPU is big endian, the following routines are used to 228 * correctly read and write into ABI. 229 * The general technique used here is : double word bitfields are defined in 230 * opposite order for big endian architecture. Then before reading them in 231 * driver the complete double word is translated using le32_to_cpu. Similarly 232 * After the driver writes into bitfields, cpu_to_le32 is used to translate the 233 * double words into required format. 234 * In order to avoid touching bits in shared structure more than once, temporary 235 * descriptors are used. These are passed as srcDesc to following functions. 236 */ 237 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc, 238 struct Vmxnet3_RxDesc *dstDesc) 239 { 240 u32 *src = (u32 *)srcDesc + 2; 241 u32 *dst = (u32 *)dstDesc + 2; 242 dstDesc->addr = le64_to_cpu(srcDesc->addr); 243 *dst = le32_to_cpu(*src); 244 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1); 245 } 246 247 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc, 248 struct Vmxnet3_TxDesc *dstDesc) 249 { 250 int i; 251 u32 *src = (u32 *)(srcDesc + 1); 252 u32 *dst = (u32 *)(dstDesc + 1); 253 254 /* Working backwards so that the gen bit is set at the end. */ 255 for (i = 2; i > 0; i--) { 256 src--; 257 dst--; 258 *dst = cpu_to_le32(*src); 259 } 260 } 261 262 263 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc, 264 struct Vmxnet3_RxCompDesc *dstDesc) 265 { 266 int i = 0; 267 u32 *src = (u32 *)srcDesc; 268 u32 *dst = (u32 *)dstDesc; 269 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) { 270 *dst = le32_to_cpu(*src); 271 src++; 272 dst++; 273 } 274 } 275 276 277 /* Used to read bitfield values from double words. */ 278 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size) 279 { 280 u32 temp = le32_to_cpu(*bitfield); 281 u32 mask = ((1 << size) - 1) << pos; 282 temp &= mask; 283 temp >>= pos; 284 return temp; 285 } 286 287 288 289 #endif /* __BIG_ENDIAN_BITFIELD */ 290 291 #ifdef __BIG_ENDIAN_BITFIELD 292 293 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \ 294 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \ 295 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE) 296 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \ 297 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \ 298 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE) 299 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \ 300 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \ 301 VMXNET3_TCD_GEN_SIZE) 302 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \ 303 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE) 304 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \ 305 (dstrcd) = (tmp); \ 306 vmxnet3_RxCompToCPU((rcd), (tmp)); \ 307 } while (0) 308 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \ 309 (dstrxd) = (tmp); \ 310 vmxnet3_RxDescToCPU((rxd), (tmp)); \ 311 } while (0) 312 313 #else 314 315 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen) 316 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop) 317 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen) 318 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx) 319 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd) 320 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd) 321 322 #endif /* __BIG_ENDIAN_BITFIELD */ 323 324 325 static void 326 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi, 327 struct pci_dev *pdev) 328 { 329 if (tbi->map_type == VMXNET3_MAP_SINGLE) 330 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len, 331 DMA_TO_DEVICE); 332 else if (tbi->map_type == VMXNET3_MAP_PAGE) 333 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len, 334 DMA_TO_DEVICE); 335 else 336 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE); 337 338 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */ 339 } 340 341 342 static int 343 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq, 344 struct pci_dev *pdev, struct vmxnet3_adapter *adapter) 345 { 346 struct sk_buff *skb; 347 int entries = 0; 348 349 /* no out of order completion */ 350 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp); 351 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1); 352 353 skb = tq->buf_info[eop_idx].skb; 354 BUG_ON(skb == NULL); 355 tq->buf_info[eop_idx].skb = NULL; 356 357 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size); 358 359 while (tq->tx_ring.next2comp != eop_idx) { 360 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp, 361 pdev); 362 363 /* update next2comp w/o tx_lock. Since we are marking more, 364 * instead of less, tx ring entries avail, the worst case is 365 * that the tx routine incorrectly re-queues a pkt due to 366 * insufficient tx ring entries. 367 */ 368 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 369 entries++; 370 } 371 372 dev_kfree_skb_any(skb); 373 return entries; 374 } 375 376 377 static int 378 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq, 379 struct vmxnet3_adapter *adapter) 380 { 381 int completed = 0; 382 union Vmxnet3_GenericDesc *gdesc; 383 384 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 385 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) { 386 /* Prevent any &gdesc->tcd field from being (speculatively) 387 * read before (&gdesc->tcd)->gen is read. 388 */ 389 dma_rmb(); 390 391 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX( 392 &gdesc->tcd), tq, adapter->pdev, 393 adapter); 394 395 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring); 396 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 397 } 398 399 if (completed) { 400 spin_lock(&tq->tx_lock); 401 if (unlikely(vmxnet3_tq_stopped(tq, adapter) && 402 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) > 403 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) && 404 netif_carrier_ok(adapter->netdev))) { 405 vmxnet3_tq_wake(tq, adapter); 406 } 407 spin_unlock(&tq->tx_lock); 408 } 409 return completed; 410 } 411 412 413 static void 414 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq, 415 struct vmxnet3_adapter *adapter) 416 { 417 int i; 418 419 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) { 420 struct vmxnet3_tx_buf_info *tbi; 421 422 tbi = tq->buf_info + tq->tx_ring.next2comp; 423 424 vmxnet3_unmap_tx_buf(tbi, adapter->pdev); 425 if (tbi->skb) { 426 dev_kfree_skb_any(tbi->skb); 427 tbi->skb = NULL; 428 } 429 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 430 } 431 432 /* sanity check, verify all buffers are indeed unmapped and freed */ 433 for (i = 0; i < tq->tx_ring.size; i++) { 434 BUG_ON(tq->buf_info[i].skb != NULL || 435 tq->buf_info[i].map_type != VMXNET3_MAP_NONE); 436 } 437 438 tq->tx_ring.gen = VMXNET3_INIT_GEN; 439 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 440 441 tq->comp_ring.gen = VMXNET3_INIT_GEN; 442 tq->comp_ring.next2proc = 0; 443 } 444 445 446 static void 447 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, 448 struct vmxnet3_adapter *adapter) 449 { 450 if (tq->tx_ring.base) { 451 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size * 452 sizeof(struct Vmxnet3_TxDesc), 453 tq->tx_ring.base, tq->tx_ring.basePA); 454 tq->tx_ring.base = NULL; 455 } 456 if (tq->data_ring.base) { 457 dma_free_coherent(&adapter->pdev->dev, 458 tq->data_ring.size * tq->txdata_desc_size, 459 tq->data_ring.base, tq->data_ring.basePA); 460 tq->data_ring.base = NULL; 461 } 462 if (tq->comp_ring.base) { 463 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size * 464 sizeof(struct Vmxnet3_TxCompDesc), 465 tq->comp_ring.base, tq->comp_ring.basePA); 466 tq->comp_ring.base = NULL; 467 } 468 kfree(tq->buf_info); 469 tq->buf_info = NULL; 470 } 471 472 473 /* Destroy all tx queues */ 474 void 475 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter) 476 { 477 int i; 478 479 for (i = 0; i < adapter->num_tx_queues; i++) 480 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter); 481 } 482 483 484 static void 485 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq, 486 struct vmxnet3_adapter *adapter) 487 { 488 int i; 489 490 /* reset the tx ring contents to 0 and reset the tx ring states */ 491 memset(tq->tx_ring.base, 0, tq->tx_ring.size * 492 sizeof(struct Vmxnet3_TxDesc)); 493 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 494 tq->tx_ring.gen = VMXNET3_INIT_GEN; 495 496 memset(tq->data_ring.base, 0, 497 tq->data_ring.size * tq->txdata_desc_size); 498 499 /* reset the tx comp ring contents to 0 and reset comp ring states */ 500 memset(tq->comp_ring.base, 0, tq->comp_ring.size * 501 sizeof(struct Vmxnet3_TxCompDesc)); 502 tq->comp_ring.next2proc = 0; 503 tq->comp_ring.gen = VMXNET3_INIT_GEN; 504 505 /* reset the bookkeeping data */ 506 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size); 507 for (i = 0; i < tq->tx_ring.size; i++) 508 tq->buf_info[i].map_type = VMXNET3_MAP_NONE; 509 510 /* stats are not reset */ 511 } 512 513 514 static int 515 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq, 516 struct vmxnet3_adapter *adapter) 517 { 518 BUG_ON(tq->tx_ring.base || tq->data_ring.base || 519 tq->comp_ring.base || tq->buf_info); 520 521 tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 522 tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc), 523 &tq->tx_ring.basePA, GFP_KERNEL); 524 if (!tq->tx_ring.base) { 525 netdev_err(adapter->netdev, "failed to allocate tx ring\n"); 526 goto err; 527 } 528 529 tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 530 tq->data_ring.size * tq->txdata_desc_size, 531 &tq->data_ring.basePA, GFP_KERNEL); 532 if (!tq->data_ring.base) { 533 netdev_err(adapter->netdev, "failed to allocate tx data ring\n"); 534 goto err; 535 } 536 537 tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 538 tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc), 539 &tq->comp_ring.basePA, GFP_KERNEL); 540 if (!tq->comp_ring.base) { 541 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n"); 542 goto err; 543 } 544 545 tq->buf_info = kcalloc_node(tq->tx_ring.size, sizeof(tq->buf_info[0]), 546 GFP_KERNEL, 547 dev_to_node(&adapter->pdev->dev)); 548 if (!tq->buf_info) 549 goto err; 550 551 return 0; 552 553 err: 554 vmxnet3_tq_destroy(tq, adapter); 555 return -ENOMEM; 556 } 557 558 static void 559 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter) 560 { 561 int i; 562 563 for (i = 0; i < adapter->num_tx_queues; i++) 564 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter); 565 } 566 567 /* 568 * starting from ring->next2fill, allocate rx buffers for the given ring 569 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers 570 * are allocated or allocation fails 571 */ 572 573 static int 574 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx, 575 int num_to_alloc, struct vmxnet3_adapter *adapter) 576 { 577 int num_allocated = 0; 578 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx]; 579 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx]; 580 u32 val; 581 582 while (num_allocated <= num_to_alloc) { 583 struct vmxnet3_rx_buf_info *rbi; 584 union Vmxnet3_GenericDesc *gd; 585 586 rbi = rbi_base + ring->next2fill; 587 gd = ring->base + ring->next2fill; 588 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 589 590 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) { 591 if (rbi->skb == NULL) { 592 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev, 593 rbi->len, 594 GFP_KERNEL); 595 if (unlikely(rbi->skb == NULL)) { 596 rq->stats.rx_buf_alloc_failure++; 597 break; 598 } 599 600 rbi->dma_addr = dma_map_single( 601 &adapter->pdev->dev, 602 rbi->skb->data, rbi->len, 603 DMA_FROM_DEVICE); 604 if (dma_mapping_error(&adapter->pdev->dev, 605 rbi->dma_addr)) { 606 dev_kfree_skb_any(rbi->skb); 607 rbi->skb = NULL; 608 rq->stats.rx_buf_alloc_failure++; 609 break; 610 } 611 } else { 612 /* rx buffer skipped by the device */ 613 } 614 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; 615 } else { 616 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE || 617 rbi->len != PAGE_SIZE); 618 619 if (rbi->page == NULL) { 620 rbi->page = alloc_page(GFP_ATOMIC); 621 if (unlikely(rbi->page == NULL)) { 622 rq->stats.rx_buf_alloc_failure++; 623 break; 624 } 625 rbi->dma_addr = dma_map_page( 626 &adapter->pdev->dev, 627 rbi->page, 0, PAGE_SIZE, 628 DMA_FROM_DEVICE); 629 if (dma_mapping_error(&adapter->pdev->dev, 630 rbi->dma_addr)) { 631 put_page(rbi->page); 632 rbi->page = NULL; 633 rq->stats.rx_buf_alloc_failure++; 634 break; 635 } 636 } else { 637 /* rx buffers skipped by the device */ 638 } 639 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT; 640 } 641 642 gd->rxd.addr = cpu_to_le64(rbi->dma_addr); 643 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT) 644 | val | rbi->len); 645 646 /* Fill the last buffer but dont mark it ready, or else the 647 * device will think that the queue is full */ 648 if (num_allocated == num_to_alloc) { 649 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 650 break; 651 } 652 653 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT); 654 num_allocated++; 655 vmxnet3_cmd_ring_adv_next2fill(ring); 656 } 657 658 netdev_dbg(adapter->netdev, 659 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n", 660 num_allocated, ring->next2fill, ring->next2comp); 661 662 /* so that the device can distinguish a full ring and an empty ring */ 663 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp); 664 665 return num_allocated; 666 } 667 668 669 static void 670 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd, 671 struct vmxnet3_rx_buf_info *rbi) 672 { 673 skb_frag_t *frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags; 674 675 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS); 676 677 __skb_frag_set_page(frag, rbi->page); 678 skb_frag_off_set(frag, 0); 679 skb_frag_size_set(frag, rcd->len); 680 skb->data_len += rcd->len; 681 skb->truesize += PAGE_SIZE; 682 skb_shinfo(skb)->nr_frags++; 683 } 684 685 686 static int 687 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx, 688 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev, 689 struct vmxnet3_adapter *adapter) 690 { 691 u32 dw2, len; 692 unsigned long buf_offset; 693 int i; 694 union Vmxnet3_GenericDesc *gdesc; 695 struct vmxnet3_tx_buf_info *tbi = NULL; 696 697 BUG_ON(ctx->copy_size > skb_headlen(skb)); 698 699 /* use the previous gen bit for the SOP desc */ 700 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; 701 702 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill; 703 gdesc = ctx->sop_txd; /* both loops below can be skipped */ 704 705 /* no need to map the buffer if headers are copied */ 706 if (ctx->copy_size) { 707 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA + 708 tq->tx_ring.next2fill * 709 tq->txdata_desc_size); 710 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size); 711 ctx->sop_txd->dword[3] = 0; 712 713 tbi = tq->buf_info + tq->tx_ring.next2fill; 714 tbi->map_type = VMXNET3_MAP_NONE; 715 716 netdev_dbg(adapter->netdev, 717 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 718 tq->tx_ring.next2fill, 719 le64_to_cpu(ctx->sop_txd->txd.addr), 720 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]); 721 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 722 723 /* use the right gen for non-SOP desc */ 724 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 725 } 726 727 /* linear part can use multiple tx desc if it's big */ 728 len = skb_headlen(skb) - ctx->copy_size; 729 buf_offset = ctx->copy_size; 730 while (len) { 731 u32 buf_size; 732 733 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 734 buf_size = len; 735 dw2 |= len; 736 } else { 737 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 738 /* spec says that for TxDesc.len, 0 == 2^14 */ 739 } 740 741 tbi = tq->buf_info + tq->tx_ring.next2fill; 742 tbi->map_type = VMXNET3_MAP_SINGLE; 743 tbi->dma_addr = dma_map_single(&adapter->pdev->dev, 744 skb->data + buf_offset, buf_size, 745 DMA_TO_DEVICE); 746 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 747 return -EFAULT; 748 749 tbi->len = buf_size; 750 751 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 752 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 753 754 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 755 gdesc->dword[2] = cpu_to_le32(dw2); 756 gdesc->dword[3] = 0; 757 758 netdev_dbg(adapter->netdev, 759 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 760 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 761 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 762 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 763 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 764 765 len -= buf_size; 766 buf_offset += buf_size; 767 } 768 769 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 770 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 771 u32 buf_size; 772 773 buf_offset = 0; 774 len = skb_frag_size(frag); 775 while (len) { 776 tbi = tq->buf_info + tq->tx_ring.next2fill; 777 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 778 buf_size = len; 779 dw2 |= len; 780 } else { 781 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 782 /* spec says that for TxDesc.len, 0 == 2^14 */ 783 } 784 tbi->map_type = VMXNET3_MAP_PAGE; 785 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag, 786 buf_offset, buf_size, 787 DMA_TO_DEVICE); 788 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 789 return -EFAULT; 790 791 tbi->len = buf_size; 792 793 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 794 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 795 796 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 797 gdesc->dword[2] = cpu_to_le32(dw2); 798 gdesc->dword[3] = 0; 799 800 netdev_dbg(adapter->netdev, 801 "txd[%u]: 0x%llx %u %u\n", 802 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 803 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 804 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 805 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 806 807 len -= buf_size; 808 buf_offset += buf_size; 809 } 810 } 811 812 ctx->eop_txd = gdesc; 813 814 /* set the last buf_info for the pkt */ 815 tbi->skb = skb; 816 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base; 817 818 return 0; 819 } 820 821 822 /* Init all tx queues */ 823 static void 824 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter) 825 { 826 int i; 827 828 for (i = 0; i < adapter->num_tx_queues; i++) 829 vmxnet3_tq_init(&adapter->tx_queue[i], adapter); 830 } 831 832 833 /* 834 * parse relevant protocol headers: 835 * For a tso pkt, relevant headers are L2/3/4 including options 836 * For a pkt requesting csum offloading, they are L2/3 and may include L4 837 * if it's a TCP/UDP pkt 838 * 839 * Returns: 840 * -1: error happens during parsing 841 * 0: protocol headers parsed, but too big to be copied 842 * 1: protocol headers parsed and copied 843 * 844 * Other effects: 845 * 1. related *ctx fields are updated. 846 * 2. ctx->copy_size is # of bytes copied 847 * 3. the portion to be copied is guaranteed to be in the linear part 848 * 849 */ 850 static int 851 vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 852 struct vmxnet3_tx_ctx *ctx, 853 struct vmxnet3_adapter *adapter) 854 { 855 u8 protocol = 0; 856 857 if (ctx->mss) { /* TSO */ 858 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 859 ctx->l4_offset = skb_inner_transport_offset(skb); 860 ctx->l4_hdr_size = inner_tcp_hdrlen(skb); 861 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 862 } else { 863 ctx->l4_offset = skb_transport_offset(skb); 864 ctx->l4_hdr_size = tcp_hdrlen(skb); 865 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 866 } 867 } else { 868 if (skb->ip_summed == CHECKSUM_PARTIAL) { 869 /* For encap packets, skb_checksum_start_offset refers 870 * to inner L4 offset. Thus, below works for encap as 871 * well as non-encap case 872 */ 873 ctx->l4_offset = skb_checksum_start_offset(skb); 874 875 if (VMXNET3_VERSION_GE_4(adapter) && 876 skb->encapsulation) { 877 struct iphdr *iph = inner_ip_hdr(skb); 878 879 if (iph->version == 4) { 880 protocol = iph->protocol; 881 } else { 882 const struct ipv6hdr *ipv6h; 883 884 ipv6h = inner_ipv6_hdr(skb); 885 protocol = ipv6h->nexthdr; 886 } 887 } else { 888 if (ctx->ipv4) { 889 const struct iphdr *iph = ip_hdr(skb); 890 891 protocol = iph->protocol; 892 } else if (ctx->ipv6) { 893 const struct ipv6hdr *ipv6h; 894 895 ipv6h = ipv6_hdr(skb); 896 protocol = ipv6h->nexthdr; 897 } 898 } 899 900 switch (protocol) { 901 case IPPROTO_TCP: 902 ctx->l4_hdr_size = skb->encapsulation ? inner_tcp_hdrlen(skb) : 903 tcp_hdrlen(skb); 904 break; 905 case IPPROTO_UDP: 906 ctx->l4_hdr_size = sizeof(struct udphdr); 907 break; 908 default: 909 ctx->l4_hdr_size = 0; 910 break; 911 } 912 913 ctx->copy_size = min(ctx->l4_offset + 914 ctx->l4_hdr_size, skb->len); 915 } else { 916 ctx->l4_offset = 0; 917 ctx->l4_hdr_size = 0; 918 /* copy as much as allowed */ 919 ctx->copy_size = min_t(unsigned int, 920 tq->txdata_desc_size, 921 skb_headlen(skb)); 922 } 923 924 if (skb->len <= VMXNET3_HDR_COPY_SIZE) 925 ctx->copy_size = skb->len; 926 927 /* make sure headers are accessible directly */ 928 if (unlikely(!pskb_may_pull(skb, ctx->copy_size))) 929 goto err; 930 } 931 932 if (unlikely(ctx->copy_size > tq->txdata_desc_size)) { 933 tq->stats.oversized_hdr++; 934 ctx->copy_size = 0; 935 return 0; 936 } 937 938 return 1; 939 err: 940 return -1; 941 } 942 943 /* 944 * copy relevant protocol headers to the transmit ring: 945 * For a tso pkt, relevant headers are L2/3/4 including options 946 * For a pkt requesting csum offloading, they are L2/3 and may include L4 947 * if it's a TCP/UDP pkt 948 * 949 * 950 * Note that this requires that vmxnet3_parse_hdr be called first to set the 951 * appropriate bits in ctx first 952 */ 953 static void 954 vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 955 struct vmxnet3_tx_ctx *ctx, 956 struct vmxnet3_adapter *adapter) 957 { 958 struct Vmxnet3_TxDataDesc *tdd; 959 960 tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base + 961 tq->tx_ring.next2fill * 962 tq->txdata_desc_size); 963 964 memcpy(tdd->data, skb->data, ctx->copy_size); 965 netdev_dbg(adapter->netdev, 966 "copy %u bytes to dataRing[%u]\n", 967 ctx->copy_size, tq->tx_ring.next2fill); 968 } 969 970 971 static void 972 vmxnet3_prepare_inner_tso(struct sk_buff *skb, 973 struct vmxnet3_tx_ctx *ctx) 974 { 975 struct tcphdr *tcph = inner_tcp_hdr(skb); 976 struct iphdr *iph = inner_ip_hdr(skb); 977 978 if (iph->version == 4) { 979 iph->check = 0; 980 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 981 IPPROTO_TCP, 0); 982 } else { 983 struct ipv6hdr *iph = inner_ipv6_hdr(skb); 984 985 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0, 986 IPPROTO_TCP, 0); 987 } 988 } 989 990 static void 991 vmxnet3_prepare_tso(struct sk_buff *skb, 992 struct vmxnet3_tx_ctx *ctx) 993 { 994 struct tcphdr *tcph = tcp_hdr(skb); 995 996 if (ctx->ipv4) { 997 struct iphdr *iph = ip_hdr(skb); 998 999 iph->check = 0; 1000 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 1001 IPPROTO_TCP, 0); 1002 } else if (ctx->ipv6) { 1003 tcp_v6_gso_csum_prep(skb); 1004 } 1005 } 1006 1007 static int txd_estimate(const struct sk_buff *skb) 1008 { 1009 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1010 int i; 1011 1012 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1013 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1014 1015 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag)); 1016 } 1017 return count; 1018 } 1019 1020 /* 1021 * Transmits a pkt thru a given tq 1022 * Returns: 1023 * NETDEV_TX_OK: descriptors are setup successfully 1024 * NETDEV_TX_OK: error occurred, the pkt is dropped 1025 * NETDEV_TX_BUSY: tx ring is full, queue is stopped 1026 * 1027 * Side-effects: 1028 * 1. tx ring may be changed 1029 * 2. tq stats may be updated accordingly 1030 * 3. shared->txNumDeferred may be updated 1031 */ 1032 1033 static int 1034 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 1035 struct vmxnet3_adapter *adapter, struct net_device *netdev) 1036 { 1037 int ret; 1038 u32 count; 1039 int num_pkts; 1040 int tx_num_deferred; 1041 unsigned long flags; 1042 struct vmxnet3_tx_ctx ctx; 1043 union Vmxnet3_GenericDesc *gdesc; 1044 #ifdef __BIG_ENDIAN_BITFIELD 1045 /* Use temporary descriptor to avoid touching bits multiple times */ 1046 union Vmxnet3_GenericDesc tempTxDesc; 1047 #endif 1048 1049 count = txd_estimate(skb); 1050 1051 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP)); 1052 ctx.ipv6 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IPV6)); 1053 1054 ctx.mss = skb_shinfo(skb)->gso_size; 1055 if (ctx.mss) { 1056 if (skb_header_cloned(skb)) { 1057 if (unlikely(pskb_expand_head(skb, 0, 0, 1058 GFP_ATOMIC) != 0)) { 1059 tq->stats.drop_tso++; 1060 goto drop_pkt; 1061 } 1062 tq->stats.copy_skb_header++; 1063 } 1064 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1065 /* tso pkts must not use more than 1066 * VMXNET3_MAX_TSO_TXD_PER_PKT entries 1067 */ 1068 if (skb_linearize(skb) != 0) { 1069 tq->stats.drop_too_many_frags++; 1070 goto drop_pkt; 1071 } 1072 tq->stats.linearized++; 1073 1074 /* recalculate the # of descriptors to use */ 1075 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1076 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1077 tq->stats.drop_too_many_frags++; 1078 goto drop_pkt; 1079 } 1080 } 1081 if (skb->encapsulation) { 1082 vmxnet3_prepare_inner_tso(skb, &ctx); 1083 } else { 1084 vmxnet3_prepare_tso(skb, &ctx); 1085 } 1086 } else { 1087 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) { 1088 1089 /* non-tso pkts must not use more than 1090 * VMXNET3_MAX_TXD_PER_PKT entries 1091 */ 1092 if (skb_linearize(skb) != 0) { 1093 tq->stats.drop_too_many_frags++; 1094 goto drop_pkt; 1095 } 1096 tq->stats.linearized++; 1097 1098 /* recalculate the # of descriptors to use */ 1099 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1100 } 1101 } 1102 1103 ret = vmxnet3_parse_hdr(skb, tq, &ctx, adapter); 1104 if (ret >= 0) { 1105 BUG_ON(ret <= 0 && ctx.copy_size != 0); 1106 /* hdrs parsed, check against other limits */ 1107 if (ctx.mss) { 1108 if (unlikely(ctx.l4_offset + ctx.l4_hdr_size > 1109 VMXNET3_MAX_TX_BUF_SIZE)) { 1110 tq->stats.drop_oversized_hdr++; 1111 goto drop_pkt; 1112 } 1113 } else { 1114 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1115 if (unlikely(ctx.l4_offset + 1116 skb->csum_offset > 1117 VMXNET3_MAX_CSUM_OFFSET)) { 1118 tq->stats.drop_oversized_hdr++; 1119 goto drop_pkt; 1120 } 1121 } 1122 } 1123 } else { 1124 tq->stats.drop_hdr_inspect_err++; 1125 goto drop_pkt; 1126 } 1127 1128 spin_lock_irqsave(&tq->tx_lock, flags); 1129 1130 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) { 1131 tq->stats.tx_ring_full++; 1132 netdev_dbg(adapter->netdev, 1133 "tx queue stopped on %s, next2comp %u" 1134 " next2fill %u\n", adapter->netdev->name, 1135 tq->tx_ring.next2comp, tq->tx_ring.next2fill); 1136 1137 vmxnet3_tq_stop(tq, adapter); 1138 spin_unlock_irqrestore(&tq->tx_lock, flags); 1139 return NETDEV_TX_BUSY; 1140 } 1141 1142 1143 vmxnet3_copy_hdr(skb, tq, &ctx, adapter); 1144 1145 /* fill tx descs related to addr & len */ 1146 if (vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter)) 1147 goto unlock_drop_pkt; 1148 1149 /* setup the EOP desc */ 1150 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP); 1151 1152 /* setup the SOP desc */ 1153 #ifdef __BIG_ENDIAN_BITFIELD 1154 gdesc = &tempTxDesc; 1155 gdesc->dword[2] = ctx.sop_txd->dword[2]; 1156 gdesc->dword[3] = ctx.sop_txd->dword[3]; 1157 #else 1158 gdesc = ctx.sop_txd; 1159 #endif 1160 tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred); 1161 if (ctx.mss) { 1162 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 1163 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1164 if (VMXNET3_VERSION_GE_7(adapter)) { 1165 gdesc->txd.om = VMXNET3_OM_TSO; 1166 gdesc->txd.ext1 = 1; 1167 } else { 1168 gdesc->txd.om = VMXNET3_OM_ENCAP; 1169 } 1170 gdesc->txd.msscof = ctx.mss; 1171 1172 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM) 1173 gdesc->txd.oco = 1; 1174 } else { 1175 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1176 gdesc->txd.om = VMXNET3_OM_TSO; 1177 gdesc->txd.msscof = ctx.mss; 1178 } 1179 num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss; 1180 } else { 1181 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1182 if (VMXNET3_VERSION_GE_4(adapter) && 1183 skb->encapsulation) { 1184 gdesc->txd.hlen = ctx.l4_offset + 1185 ctx.l4_hdr_size; 1186 if (VMXNET3_VERSION_GE_7(adapter)) { 1187 gdesc->txd.om = VMXNET3_OM_CSUM; 1188 gdesc->txd.msscof = ctx.l4_offset + 1189 skb->csum_offset; 1190 gdesc->txd.ext1 = 1; 1191 } else { 1192 gdesc->txd.om = VMXNET3_OM_ENCAP; 1193 gdesc->txd.msscof = 0; /* Reserved */ 1194 } 1195 } else { 1196 gdesc->txd.hlen = ctx.l4_offset; 1197 gdesc->txd.om = VMXNET3_OM_CSUM; 1198 gdesc->txd.msscof = ctx.l4_offset + 1199 skb->csum_offset; 1200 } 1201 } else { 1202 gdesc->txd.om = 0; 1203 gdesc->txd.msscof = 0; 1204 } 1205 num_pkts = 1; 1206 } 1207 le32_add_cpu(&tq->shared->txNumDeferred, num_pkts); 1208 tx_num_deferred += num_pkts; 1209 1210 if (skb_vlan_tag_present(skb)) { 1211 gdesc->txd.ti = 1; 1212 gdesc->txd.tci = skb_vlan_tag_get(skb); 1213 } 1214 1215 /* Ensure that the write to (&gdesc->txd)->gen will be observed after 1216 * all other writes to &gdesc->txd. 1217 */ 1218 dma_wmb(); 1219 1220 /* finally flips the GEN bit of the SOP desc. */ 1221 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^ 1222 VMXNET3_TXD_GEN); 1223 #ifdef __BIG_ENDIAN_BITFIELD 1224 /* Finished updating in bitfields of Tx Desc, so write them in original 1225 * place. 1226 */ 1227 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc, 1228 (struct Vmxnet3_TxDesc *)ctx.sop_txd); 1229 gdesc = ctx.sop_txd; 1230 #endif 1231 netdev_dbg(adapter->netdev, 1232 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n", 1233 (u32)(ctx.sop_txd - 1234 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr), 1235 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3])); 1236 1237 spin_unlock_irqrestore(&tq->tx_lock, flags); 1238 1239 if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) { 1240 tq->shared->txNumDeferred = 0; 1241 VMXNET3_WRITE_BAR0_REG(adapter, 1242 adapter->tx_prod_offset + tq->qid * 8, 1243 tq->tx_ring.next2fill); 1244 } 1245 1246 return NETDEV_TX_OK; 1247 1248 unlock_drop_pkt: 1249 spin_unlock_irqrestore(&tq->tx_lock, flags); 1250 drop_pkt: 1251 tq->stats.drop_total++; 1252 dev_kfree_skb_any(skb); 1253 return NETDEV_TX_OK; 1254 } 1255 1256 1257 static netdev_tx_t 1258 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 1259 { 1260 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1261 1262 BUG_ON(skb->queue_mapping > adapter->num_tx_queues); 1263 return vmxnet3_tq_xmit(skb, 1264 &adapter->tx_queue[skb->queue_mapping], 1265 adapter, netdev); 1266 } 1267 1268 1269 static void 1270 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter, 1271 struct sk_buff *skb, 1272 union Vmxnet3_GenericDesc *gdesc) 1273 { 1274 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) { 1275 if (gdesc->rcd.v4 && 1276 (le32_to_cpu(gdesc->dword[3]) & 1277 VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) { 1278 skb->ip_summed = CHECKSUM_UNNECESSARY; 1279 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1280 !(le32_to_cpu(gdesc->dword[0]) & 1281 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1282 WARN_ON_ONCE(gdesc->rcd.frg && 1283 !(le32_to_cpu(gdesc->dword[0]) & 1284 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1285 } else if (gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) & 1286 (1 << VMXNET3_RCD_TUC_SHIFT))) { 1287 skb->ip_summed = CHECKSUM_UNNECESSARY; 1288 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1289 !(le32_to_cpu(gdesc->dword[0]) & 1290 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1291 WARN_ON_ONCE(gdesc->rcd.frg && 1292 !(le32_to_cpu(gdesc->dword[0]) & 1293 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1294 } else { 1295 if (gdesc->rcd.csum) { 1296 skb->csum = htons(gdesc->rcd.csum); 1297 skb->ip_summed = CHECKSUM_PARTIAL; 1298 } else { 1299 skb_checksum_none_assert(skb); 1300 } 1301 } 1302 } else { 1303 skb_checksum_none_assert(skb); 1304 } 1305 } 1306 1307 1308 static void 1309 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd, 1310 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter) 1311 { 1312 rq->stats.drop_err++; 1313 if (!rcd->fcs) 1314 rq->stats.drop_fcs++; 1315 1316 rq->stats.drop_total++; 1317 1318 /* 1319 * We do not unmap and chain the rx buffer to the skb. 1320 * We basically pretend this buffer is not used and will be recycled 1321 * by vmxnet3_rq_alloc_rx_buf() 1322 */ 1323 1324 /* 1325 * ctx->skb may be NULL if this is the first and the only one 1326 * desc for the pkt 1327 */ 1328 if (ctx->skb) 1329 dev_kfree_skb_irq(ctx->skb); 1330 1331 ctx->skb = NULL; 1332 } 1333 1334 1335 static u32 1336 vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb, 1337 union Vmxnet3_GenericDesc *gdesc) 1338 { 1339 u32 hlen, maplen; 1340 union { 1341 void *ptr; 1342 struct ethhdr *eth; 1343 struct vlan_ethhdr *veth; 1344 struct iphdr *ipv4; 1345 struct ipv6hdr *ipv6; 1346 struct tcphdr *tcp; 1347 } hdr; 1348 BUG_ON(gdesc->rcd.tcp == 0); 1349 1350 maplen = skb_headlen(skb); 1351 if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen)) 1352 return 0; 1353 1354 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 1355 skb->protocol == cpu_to_be16(ETH_P_8021AD)) 1356 hlen = sizeof(struct vlan_ethhdr); 1357 else 1358 hlen = sizeof(struct ethhdr); 1359 1360 hdr.eth = eth_hdr(skb); 1361 if (gdesc->rcd.v4) { 1362 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) && 1363 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP)); 1364 hdr.ptr += hlen; 1365 BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP); 1366 hlen = hdr.ipv4->ihl << 2; 1367 hdr.ptr += hdr.ipv4->ihl << 2; 1368 } else if (gdesc->rcd.v6) { 1369 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) && 1370 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6)); 1371 hdr.ptr += hlen; 1372 /* Use an estimated value, since we also need to handle 1373 * TSO case. 1374 */ 1375 if (hdr.ipv6->nexthdr != IPPROTO_TCP) 1376 return sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 1377 hlen = sizeof(struct ipv6hdr); 1378 hdr.ptr += sizeof(struct ipv6hdr); 1379 } else { 1380 /* Non-IP pkt, dont estimate header length */ 1381 return 0; 1382 } 1383 1384 if (hlen + sizeof(struct tcphdr) > maplen) 1385 return 0; 1386 1387 return (hlen + (hdr.tcp->doff << 2)); 1388 } 1389 1390 static int 1391 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, 1392 struct vmxnet3_adapter *adapter, int quota) 1393 { 1394 u32 rxprod_reg[2] = { 1395 adapter->rx_prod_offset, adapter->rx_prod2_offset 1396 }; 1397 u32 num_pkts = 0; 1398 bool skip_page_frags = false; 1399 struct Vmxnet3_RxCompDesc *rcd; 1400 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx; 1401 u16 segCnt = 0, mss = 0; 1402 int comp_offset, fill_offset; 1403 #ifdef __BIG_ENDIAN_BITFIELD 1404 struct Vmxnet3_RxDesc rxCmdDesc; 1405 struct Vmxnet3_RxCompDesc rxComp; 1406 #endif 1407 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, 1408 &rxComp); 1409 while (rcd->gen == rq->comp_ring.gen) { 1410 struct vmxnet3_rx_buf_info *rbi; 1411 struct sk_buff *skb, *new_skb = NULL; 1412 struct page *new_page = NULL; 1413 dma_addr_t new_dma_addr; 1414 int num_to_alloc; 1415 struct Vmxnet3_RxDesc *rxd; 1416 u32 idx, ring_idx; 1417 struct vmxnet3_cmd_ring *ring = NULL; 1418 if (num_pkts >= quota) { 1419 /* we may stop even before we see the EOP desc of 1420 * the current pkt 1421 */ 1422 break; 1423 } 1424 1425 /* Prevent any rcd field from being (speculatively) read before 1426 * rcd->gen is read. 1427 */ 1428 dma_rmb(); 1429 1430 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 && 1431 rcd->rqID != rq->dataRingQid); 1432 idx = rcd->rxdIdx; 1433 ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID); 1434 ring = rq->rx_ring + ring_idx; 1435 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd, 1436 &rxCmdDesc); 1437 rbi = rq->buf_info[ring_idx] + idx; 1438 1439 BUG_ON(rxd->addr != rbi->dma_addr || 1440 rxd->len != rbi->len); 1441 1442 if (unlikely(rcd->eop && rcd->err)) { 1443 vmxnet3_rx_error(rq, rcd, ctx, adapter); 1444 goto rcd_done; 1445 } 1446 1447 if (rcd->sop) { /* first buf of the pkt */ 1448 bool rxDataRingUsed; 1449 u16 len; 1450 1451 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD || 1452 (rcd->rqID != rq->qid && 1453 rcd->rqID != rq->dataRingQid)); 1454 1455 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB); 1456 BUG_ON(ctx->skb != NULL || rbi->skb == NULL); 1457 1458 if (unlikely(rcd->len == 0)) { 1459 /* Pretend the rx buffer is skipped. */ 1460 BUG_ON(!(rcd->sop && rcd->eop)); 1461 netdev_dbg(adapter->netdev, 1462 "rxRing[%u][%u] 0 length\n", 1463 ring_idx, idx); 1464 goto rcd_done; 1465 } 1466 1467 skip_page_frags = false; 1468 ctx->skb = rbi->skb; 1469 1470 rxDataRingUsed = 1471 VMXNET3_RX_DATA_RING(adapter, rcd->rqID); 1472 len = rxDataRingUsed ? rcd->len : rbi->len; 1473 new_skb = netdev_alloc_skb_ip_align(adapter->netdev, 1474 len); 1475 if (new_skb == NULL) { 1476 /* Skb allocation failed, do not handover this 1477 * skb to stack. Reuse it. Drop the existing pkt 1478 */ 1479 rq->stats.rx_buf_alloc_failure++; 1480 ctx->skb = NULL; 1481 rq->stats.drop_total++; 1482 skip_page_frags = true; 1483 goto rcd_done; 1484 } 1485 1486 if (rxDataRingUsed) { 1487 size_t sz; 1488 1489 BUG_ON(rcd->len > rq->data_ring.desc_size); 1490 1491 ctx->skb = new_skb; 1492 sz = rcd->rxdIdx * rq->data_ring.desc_size; 1493 memcpy(new_skb->data, 1494 &rq->data_ring.base[sz], rcd->len); 1495 } else { 1496 ctx->skb = rbi->skb; 1497 1498 new_dma_addr = 1499 dma_map_single(&adapter->pdev->dev, 1500 new_skb->data, rbi->len, 1501 DMA_FROM_DEVICE); 1502 if (dma_mapping_error(&adapter->pdev->dev, 1503 new_dma_addr)) { 1504 dev_kfree_skb(new_skb); 1505 /* Skb allocation failed, do not 1506 * handover this skb to stack. Reuse 1507 * it. Drop the existing pkt. 1508 */ 1509 rq->stats.rx_buf_alloc_failure++; 1510 ctx->skb = NULL; 1511 rq->stats.drop_total++; 1512 skip_page_frags = true; 1513 goto rcd_done; 1514 } 1515 1516 dma_unmap_single(&adapter->pdev->dev, 1517 rbi->dma_addr, 1518 rbi->len, 1519 DMA_FROM_DEVICE); 1520 1521 /* Immediate refill */ 1522 rbi->skb = new_skb; 1523 rbi->dma_addr = new_dma_addr; 1524 rxd->addr = cpu_to_le64(rbi->dma_addr); 1525 rxd->len = rbi->len; 1526 } 1527 1528 #ifdef VMXNET3_RSS 1529 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE && 1530 (adapter->netdev->features & NETIF_F_RXHASH)) { 1531 enum pkt_hash_types hash_type; 1532 1533 switch (rcd->rssType) { 1534 case VMXNET3_RCD_RSS_TYPE_IPV4: 1535 case VMXNET3_RCD_RSS_TYPE_IPV6: 1536 hash_type = PKT_HASH_TYPE_L3; 1537 break; 1538 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 1539 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 1540 case VMXNET3_RCD_RSS_TYPE_UDPIPV4: 1541 case VMXNET3_RCD_RSS_TYPE_UDPIPV6: 1542 hash_type = PKT_HASH_TYPE_L4; 1543 break; 1544 default: 1545 hash_type = PKT_HASH_TYPE_L3; 1546 break; 1547 } 1548 skb_set_hash(ctx->skb, 1549 le32_to_cpu(rcd->rssHash), 1550 hash_type); 1551 } 1552 #endif 1553 skb_put(ctx->skb, rcd->len); 1554 1555 if (VMXNET3_VERSION_GE_2(adapter) && 1556 rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) { 1557 struct Vmxnet3_RxCompDescExt *rcdlro; 1558 rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd; 1559 1560 segCnt = rcdlro->segCnt; 1561 WARN_ON_ONCE(segCnt == 0); 1562 mss = rcdlro->mss; 1563 if (unlikely(segCnt <= 1)) 1564 segCnt = 0; 1565 } else { 1566 segCnt = 0; 1567 } 1568 } else { 1569 BUG_ON(ctx->skb == NULL && !skip_page_frags); 1570 1571 /* non SOP buffer must be type 1 in most cases */ 1572 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE); 1573 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY); 1574 1575 /* If an sop buffer was dropped, skip all 1576 * following non-sop fragments. They will be reused. 1577 */ 1578 if (skip_page_frags) 1579 goto rcd_done; 1580 1581 if (rcd->len) { 1582 new_page = alloc_page(GFP_ATOMIC); 1583 /* Replacement page frag could not be allocated. 1584 * Reuse this page. Drop the pkt and free the 1585 * skb which contained this page as a frag. Skip 1586 * processing all the following non-sop frags. 1587 */ 1588 if (unlikely(!new_page)) { 1589 rq->stats.rx_buf_alloc_failure++; 1590 dev_kfree_skb(ctx->skb); 1591 ctx->skb = NULL; 1592 skip_page_frags = true; 1593 goto rcd_done; 1594 } 1595 new_dma_addr = dma_map_page(&adapter->pdev->dev, 1596 new_page, 1597 0, PAGE_SIZE, 1598 DMA_FROM_DEVICE); 1599 if (dma_mapping_error(&adapter->pdev->dev, 1600 new_dma_addr)) { 1601 put_page(new_page); 1602 rq->stats.rx_buf_alloc_failure++; 1603 dev_kfree_skb(ctx->skb); 1604 ctx->skb = NULL; 1605 skip_page_frags = true; 1606 goto rcd_done; 1607 } 1608 1609 dma_unmap_page(&adapter->pdev->dev, 1610 rbi->dma_addr, rbi->len, 1611 DMA_FROM_DEVICE); 1612 1613 vmxnet3_append_frag(ctx->skb, rcd, rbi); 1614 1615 /* Immediate refill */ 1616 rbi->page = new_page; 1617 rbi->dma_addr = new_dma_addr; 1618 rxd->addr = cpu_to_le64(rbi->dma_addr); 1619 rxd->len = rbi->len; 1620 } 1621 } 1622 1623 1624 skb = ctx->skb; 1625 if (rcd->eop) { 1626 u32 mtu = adapter->netdev->mtu; 1627 skb->len += skb->data_len; 1628 1629 vmxnet3_rx_csum(adapter, skb, 1630 (union Vmxnet3_GenericDesc *)rcd); 1631 skb->protocol = eth_type_trans(skb, adapter->netdev); 1632 if (!rcd->tcp || 1633 !(adapter->netdev->features & NETIF_F_LRO)) 1634 goto not_lro; 1635 1636 if (segCnt != 0 && mss != 0) { 1637 skb_shinfo(skb)->gso_type = rcd->v4 ? 1638 SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1639 skb_shinfo(skb)->gso_size = mss; 1640 skb_shinfo(skb)->gso_segs = segCnt; 1641 } else if (segCnt != 0 || skb->len > mtu) { 1642 u32 hlen; 1643 1644 hlen = vmxnet3_get_hdr_len(adapter, skb, 1645 (union Vmxnet3_GenericDesc *)rcd); 1646 if (hlen == 0) 1647 goto not_lro; 1648 1649 skb_shinfo(skb)->gso_type = 1650 rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1651 if (segCnt != 0) { 1652 skb_shinfo(skb)->gso_segs = segCnt; 1653 skb_shinfo(skb)->gso_size = 1654 DIV_ROUND_UP(skb->len - 1655 hlen, segCnt); 1656 } else { 1657 skb_shinfo(skb)->gso_size = mtu - hlen; 1658 } 1659 } 1660 not_lro: 1661 if (unlikely(rcd->ts)) 1662 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci); 1663 1664 if (adapter->netdev->features & NETIF_F_LRO) 1665 netif_receive_skb(skb); 1666 else 1667 napi_gro_receive(&rq->napi, skb); 1668 1669 ctx->skb = NULL; 1670 num_pkts++; 1671 } 1672 1673 rcd_done: 1674 /* device may have skipped some rx descs */ 1675 ring = rq->rx_ring + ring_idx; 1676 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 1677 1678 comp_offset = vmxnet3_cmd_ring_desc_avail(ring); 1679 fill_offset = (idx > ring->next2fill ? 0 : ring->size) + 1680 idx - ring->next2fill - 1; 1681 if (!ring->isOutOfOrder || fill_offset >= comp_offset) 1682 ring->next2comp = idx; 1683 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring); 1684 1685 /* Ensure that the writes to rxd->gen bits will be observed 1686 * after all other writes to rxd objects. 1687 */ 1688 dma_wmb(); 1689 1690 while (num_to_alloc) { 1691 rbi = rq->buf_info[ring_idx] + ring->next2fill; 1692 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_OOORX_COMP))) 1693 goto refill_buf; 1694 if (ring_idx == 0) { 1695 /* ring0 Type1 buffers can get skipped; re-fill them */ 1696 if (rbi->buf_type != VMXNET3_RX_BUF_SKB) 1697 goto refill_buf; 1698 } 1699 if (rbi->comp_state == VMXNET3_RXD_COMP_DONE) { 1700 refill_buf: 1701 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd, 1702 &rxCmdDesc); 1703 WARN_ON(!rxd->addr); 1704 1705 /* Recv desc is ready to be used by the device */ 1706 rxd->gen = ring->gen; 1707 vmxnet3_cmd_ring_adv_next2fill(ring); 1708 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 1709 num_to_alloc--; 1710 } else { 1711 /* rx completion hasn't occurred */ 1712 ring->isOutOfOrder = 1; 1713 break; 1714 } 1715 } 1716 1717 if (num_to_alloc == 0) { 1718 ring->isOutOfOrder = 0; 1719 } 1720 1721 /* if needed, update the register */ 1722 if (unlikely(rq->shared->updateRxProd) && (ring->next2fill & 0xf) == 0) { 1723 VMXNET3_WRITE_BAR0_REG(adapter, 1724 rxprod_reg[ring_idx] + rq->qid * 8, 1725 ring->next2fill); 1726 } 1727 1728 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring); 1729 vmxnet3_getRxComp(rcd, 1730 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp); 1731 } 1732 1733 return num_pkts; 1734 } 1735 1736 1737 static void 1738 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq, 1739 struct vmxnet3_adapter *adapter) 1740 { 1741 u32 i, ring_idx; 1742 struct Vmxnet3_RxDesc *rxd; 1743 1744 /* ring has already been cleaned up */ 1745 if (!rq->rx_ring[0].base) 1746 return; 1747 1748 for (ring_idx = 0; ring_idx < 2; ring_idx++) { 1749 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) { 1750 #ifdef __BIG_ENDIAN_BITFIELD 1751 struct Vmxnet3_RxDesc rxDesc; 1752 #endif 1753 vmxnet3_getRxDesc(rxd, 1754 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc); 1755 1756 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && 1757 rq->buf_info[ring_idx][i].skb) { 1758 dma_unmap_single(&adapter->pdev->dev, rxd->addr, 1759 rxd->len, DMA_FROM_DEVICE); 1760 dev_kfree_skb(rq->buf_info[ring_idx][i].skb); 1761 rq->buf_info[ring_idx][i].skb = NULL; 1762 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY && 1763 rq->buf_info[ring_idx][i].page) { 1764 dma_unmap_page(&adapter->pdev->dev, rxd->addr, 1765 rxd->len, DMA_FROM_DEVICE); 1766 put_page(rq->buf_info[ring_idx][i].page); 1767 rq->buf_info[ring_idx][i].page = NULL; 1768 } 1769 } 1770 1771 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN; 1772 rq->rx_ring[ring_idx].next2fill = 1773 rq->rx_ring[ring_idx].next2comp = 0; 1774 } 1775 1776 rq->comp_ring.gen = VMXNET3_INIT_GEN; 1777 rq->comp_ring.next2proc = 0; 1778 } 1779 1780 1781 static void 1782 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter) 1783 { 1784 int i; 1785 1786 for (i = 0; i < adapter->num_rx_queues; i++) 1787 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter); 1788 } 1789 1790 1791 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, 1792 struct vmxnet3_adapter *adapter) 1793 { 1794 int i; 1795 int j; 1796 1797 /* all rx buffers must have already been freed */ 1798 for (i = 0; i < 2; i++) { 1799 if (rq->buf_info[i]) { 1800 for (j = 0; j < rq->rx_ring[i].size; j++) 1801 BUG_ON(rq->buf_info[i][j].page != NULL); 1802 } 1803 } 1804 1805 1806 for (i = 0; i < 2; i++) { 1807 if (rq->rx_ring[i].base) { 1808 dma_free_coherent(&adapter->pdev->dev, 1809 rq->rx_ring[i].size 1810 * sizeof(struct Vmxnet3_RxDesc), 1811 rq->rx_ring[i].base, 1812 rq->rx_ring[i].basePA); 1813 rq->rx_ring[i].base = NULL; 1814 } 1815 } 1816 1817 if (rq->data_ring.base) { 1818 dma_free_coherent(&adapter->pdev->dev, 1819 rq->rx_ring[0].size * rq->data_ring.desc_size, 1820 rq->data_ring.base, rq->data_ring.basePA); 1821 rq->data_ring.base = NULL; 1822 } 1823 1824 if (rq->comp_ring.base) { 1825 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size 1826 * sizeof(struct Vmxnet3_RxCompDesc), 1827 rq->comp_ring.base, rq->comp_ring.basePA); 1828 rq->comp_ring.base = NULL; 1829 } 1830 1831 kfree(rq->buf_info[0]); 1832 rq->buf_info[0] = NULL; 1833 rq->buf_info[1] = NULL; 1834 } 1835 1836 static void 1837 vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter) 1838 { 1839 int i; 1840 1841 for (i = 0; i < adapter->num_rx_queues; i++) { 1842 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 1843 1844 if (rq->data_ring.base) { 1845 dma_free_coherent(&adapter->pdev->dev, 1846 (rq->rx_ring[0].size * 1847 rq->data_ring.desc_size), 1848 rq->data_ring.base, 1849 rq->data_ring.basePA); 1850 rq->data_ring.base = NULL; 1851 rq->data_ring.desc_size = 0; 1852 } 1853 } 1854 } 1855 1856 static int 1857 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq, 1858 struct vmxnet3_adapter *adapter) 1859 { 1860 int i; 1861 1862 /* initialize buf_info */ 1863 for (i = 0; i < rq->rx_ring[0].size; i++) { 1864 1865 /* 1st buf for a pkt is skbuff */ 1866 if (i % adapter->rx_buf_per_pkt == 0) { 1867 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB; 1868 rq->buf_info[0][i].len = adapter->skb_buf_size; 1869 } else { /* subsequent bufs for a pkt is frag */ 1870 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE; 1871 rq->buf_info[0][i].len = PAGE_SIZE; 1872 } 1873 } 1874 for (i = 0; i < rq->rx_ring[1].size; i++) { 1875 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE; 1876 rq->buf_info[1][i].len = PAGE_SIZE; 1877 } 1878 1879 /* reset internal state and allocate buffers for both rings */ 1880 for (i = 0; i < 2; i++) { 1881 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0; 1882 1883 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size * 1884 sizeof(struct Vmxnet3_RxDesc)); 1885 rq->rx_ring[i].gen = VMXNET3_INIT_GEN; 1886 rq->rx_ring[i].isOutOfOrder = 0; 1887 } 1888 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1, 1889 adapter) == 0) { 1890 /* at least has 1 rx buffer for the 1st ring */ 1891 return -ENOMEM; 1892 } 1893 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter); 1894 1895 /* reset the comp ring */ 1896 rq->comp_ring.next2proc = 0; 1897 memset(rq->comp_ring.base, 0, rq->comp_ring.size * 1898 sizeof(struct Vmxnet3_RxCompDesc)); 1899 rq->comp_ring.gen = VMXNET3_INIT_GEN; 1900 1901 /* reset rxctx */ 1902 rq->rx_ctx.skb = NULL; 1903 1904 /* stats are not reset */ 1905 return 0; 1906 } 1907 1908 1909 static int 1910 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter) 1911 { 1912 int i, err = 0; 1913 1914 for (i = 0; i < adapter->num_rx_queues; i++) { 1915 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter); 1916 if (unlikely(err)) { 1917 dev_err(&adapter->netdev->dev, "%s: failed to " 1918 "initialize rx queue%i\n", 1919 adapter->netdev->name, i); 1920 break; 1921 } 1922 } 1923 return err; 1924 1925 } 1926 1927 1928 static int 1929 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter) 1930 { 1931 int i; 1932 size_t sz; 1933 struct vmxnet3_rx_buf_info *bi; 1934 1935 for (i = 0; i < 2; i++) { 1936 1937 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc); 1938 rq->rx_ring[i].base = dma_alloc_coherent( 1939 &adapter->pdev->dev, sz, 1940 &rq->rx_ring[i].basePA, 1941 GFP_KERNEL); 1942 if (!rq->rx_ring[i].base) { 1943 netdev_err(adapter->netdev, 1944 "failed to allocate rx ring %d\n", i); 1945 goto err; 1946 } 1947 } 1948 1949 if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) { 1950 sz = rq->rx_ring[0].size * rq->data_ring.desc_size; 1951 rq->data_ring.base = 1952 dma_alloc_coherent(&adapter->pdev->dev, sz, 1953 &rq->data_ring.basePA, 1954 GFP_KERNEL); 1955 if (!rq->data_ring.base) { 1956 netdev_err(adapter->netdev, 1957 "rx data ring will be disabled\n"); 1958 adapter->rxdataring_enabled = false; 1959 } 1960 } else { 1961 rq->data_ring.base = NULL; 1962 rq->data_ring.desc_size = 0; 1963 } 1964 1965 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc); 1966 rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz, 1967 &rq->comp_ring.basePA, 1968 GFP_KERNEL); 1969 if (!rq->comp_ring.base) { 1970 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n"); 1971 goto err; 1972 } 1973 1974 bi = kcalloc_node(rq->rx_ring[0].size + rq->rx_ring[1].size, 1975 sizeof(rq->buf_info[0][0]), GFP_KERNEL, 1976 dev_to_node(&adapter->pdev->dev)); 1977 if (!bi) 1978 goto err; 1979 1980 rq->buf_info[0] = bi; 1981 rq->buf_info[1] = bi + rq->rx_ring[0].size; 1982 1983 return 0; 1984 1985 err: 1986 vmxnet3_rq_destroy(rq, adapter); 1987 return -ENOMEM; 1988 } 1989 1990 1991 static int 1992 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter) 1993 { 1994 int i, err = 0; 1995 1996 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 1997 1998 for (i = 0; i < adapter->num_rx_queues; i++) { 1999 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter); 2000 if (unlikely(err)) { 2001 dev_err(&adapter->netdev->dev, 2002 "%s: failed to create rx queue%i\n", 2003 adapter->netdev->name, i); 2004 goto err_out; 2005 } 2006 } 2007 2008 if (!adapter->rxdataring_enabled) 2009 vmxnet3_rq_destroy_all_rxdataring(adapter); 2010 2011 return err; 2012 err_out: 2013 vmxnet3_rq_destroy_all(adapter); 2014 return err; 2015 2016 } 2017 2018 /* Multiple queue aware polling function for tx and rx */ 2019 2020 static int 2021 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget) 2022 { 2023 int rcd_done = 0, i; 2024 if (unlikely(adapter->shared->ecr)) 2025 vmxnet3_process_events(adapter); 2026 for (i = 0; i < adapter->num_tx_queues; i++) 2027 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter); 2028 2029 for (i = 0; i < adapter->num_rx_queues; i++) 2030 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i], 2031 adapter, budget); 2032 return rcd_done; 2033 } 2034 2035 2036 static int 2037 vmxnet3_poll(struct napi_struct *napi, int budget) 2038 { 2039 struct vmxnet3_rx_queue *rx_queue = container_of(napi, 2040 struct vmxnet3_rx_queue, napi); 2041 int rxd_done; 2042 2043 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget); 2044 2045 if (rxd_done < budget) { 2046 napi_complete_done(napi, rxd_done); 2047 vmxnet3_enable_all_intrs(rx_queue->adapter); 2048 } 2049 return rxd_done; 2050 } 2051 2052 /* 2053 * NAPI polling function for MSI-X mode with multiple Rx queues 2054 * Returns the # of the NAPI credit consumed (# of rx descriptors processed) 2055 */ 2056 2057 static int 2058 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget) 2059 { 2060 struct vmxnet3_rx_queue *rq = container_of(napi, 2061 struct vmxnet3_rx_queue, napi); 2062 struct vmxnet3_adapter *adapter = rq->adapter; 2063 int rxd_done; 2064 2065 /* When sharing interrupt with corresponding tx queue, process 2066 * tx completions in that queue as well 2067 */ 2068 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) { 2069 struct vmxnet3_tx_queue *tq = 2070 &adapter->tx_queue[rq - adapter->rx_queue]; 2071 vmxnet3_tq_tx_complete(tq, adapter); 2072 } 2073 2074 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget); 2075 2076 if (rxd_done < budget) { 2077 struct Vmxnet3_RxCompDesc *rcd; 2078 #ifdef __BIG_ENDIAN_BITFIELD 2079 struct Vmxnet3_RxCompDesc rxComp; 2080 #endif 2081 napi_complete_done(napi, rxd_done); 2082 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx); 2083 /* after unmasking the interrupt, check if any descriptors were completed */ 2084 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, 2085 &rxComp); 2086 if (rcd->gen == rq->comp_ring.gen && napi_reschedule(napi)) 2087 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx); 2088 } 2089 return rxd_done; 2090 } 2091 2092 2093 #ifdef CONFIG_PCI_MSI 2094 2095 /* 2096 * Handle completion interrupts on tx queues 2097 * Returns whether or not the intr is handled 2098 */ 2099 2100 static irqreturn_t 2101 vmxnet3_msix_tx(int irq, void *data) 2102 { 2103 struct vmxnet3_tx_queue *tq = data; 2104 struct vmxnet3_adapter *adapter = tq->adapter; 2105 2106 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2107 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx); 2108 2109 /* Handle the case where only one irq is allocate for all tx queues */ 2110 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2111 int i; 2112 for (i = 0; i < adapter->num_tx_queues; i++) { 2113 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i]; 2114 vmxnet3_tq_tx_complete(txq, adapter); 2115 } 2116 } else { 2117 vmxnet3_tq_tx_complete(tq, adapter); 2118 } 2119 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx); 2120 2121 return IRQ_HANDLED; 2122 } 2123 2124 2125 /* 2126 * Handle completion interrupts on rx queues. Returns whether or not the 2127 * intr is handled 2128 */ 2129 2130 static irqreturn_t 2131 vmxnet3_msix_rx(int irq, void *data) 2132 { 2133 struct vmxnet3_rx_queue *rq = data; 2134 struct vmxnet3_adapter *adapter = rq->adapter; 2135 2136 /* disable intr if needed */ 2137 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2138 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx); 2139 napi_schedule(&rq->napi); 2140 2141 return IRQ_HANDLED; 2142 } 2143 2144 /* 2145 *---------------------------------------------------------------------------- 2146 * 2147 * vmxnet3_msix_event -- 2148 * 2149 * vmxnet3 msix event intr handler 2150 * 2151 * Result: 2152 * whether or not the intr is handled 2153 * 2154 *---------------------------------------------------------------------------- 2155 */ 2156 2157 static irqreturn_t 2158 vmxnet3_msix_event(int irq, void *data) 2159 { 2160 struct net_device *dev = data; 2161 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2162 2163 /* disable intr if needed */ 2164 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2165 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx); 2166 2167 if (adapter->shared->ecr) 2168 vmxnet3_process_events(adapter); 2169 2170 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx); 2171 2172 return IRQ_HANDLED; 2173 } 2174 2175 #endif /* CONFIG_PCI_MSI */ 2176 2177 2178 /* Interrupt handler for vmxnet3 */ 2179 static irqreturn_t 2180 vmxnet3_intr(int irq, void *dev_id) 2181 { 2182 struct net_device *dev = dev_id; 2183 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2184 2185 if (adapter->intr.type == VMXNET3_IT_INTX) { 2186 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR); 2187 if (unlikely(icr == 0)) 2188 /* not ours */ 2189 return IRQ_NONE; 2190 } 2191 2192 2193 /* disable intr if needed */ 2194 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2195 vmxnet3_disable_all_intrs(adapter); 2196 2197 napi_schedule(&adapter->rx_queue[0].napi); 2198 2199 return IRQ_HANDLED; 2200 } 2201 2202 #ifdef CONFIG_NET_POLL_CONTROLLER 2203 2204 /* netpoll callback. */ 2205 static void 2206 vmxnet3_netpoll(struct net_device *netdev) 2207 { 2208 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2209 2210 switch (adapter->intr.type) { 2211 #ifdef CONFIG_PCI_MSI 2212 case VMXNET3_IT_MSIX: { 2213 int i; 2214 for (i = 0; i < adapter->num_rx_queues; i++) 2215 vmxnet3_msix_rx(0, &adapter->rx_queue[i]); 2216 break; 2217 } 2218 #endif 2219 case VMXNET3_IT_MSI: 2220 default: 2221 vmxnet3_intr(0, adapter->netdev); 2222 break; 2223 } 2224 2225 } 2226 #endif /* CONFIG_NET_POLL_CONTROLLER */ 2227 2228 static int 2229 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter) 2230 { 2231 struct vmxnet3_intr *intr = &adapter->intr; 2232 int err = 0, i; 2233 int vector = 0; 2234 2235 #ifdef CONFIG_PCI_MSI 2236 if (adapter->intr.type == VMXNET3_IT_MSIX) { 2237 for (i = 0; i < adapter->num_tx_queues; i++) { 2238 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2239 sprintf(adapter->tx_queue[i].name, "%s-tx-%d", 2240 adapter->netdev->name, vector); 2241 err = request_irq( 2242 intr->msix_entries[vector].vector, 2243 vmxnet3_msix_tx, 0, 2244 adapter->tx_queue[i].name, 2245 &adapter->tx_queue[i]); 2246 } else { 2247 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d", 2248 adapter->netdev->name, vector); 2249 } 2250 if (err) { 2251 dev_err(&adapter->netdev->dev, 2252 "Failed to request irq for MSIX, %s, " 2253 "error %d\n", 2254 adapter->tx_queue[i].name, err); 2255 return err; 2256 } 2257 2258 /* Handle the case where only 1 MSIx was allocated for 2259 * all tx queues */ 2260 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2261 for (; i < adapter->num_tx_queues; i++) 2262 adapter->tx_queue[i].comp_ring.intr_idx 2263 = vector; 2264 vector++; 2265 break; 2266 } else { 2267 adapter->tx_queue[i].comp_ring.intr_idx 2268 = vector++; 2269 } 2270 } 2271 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) 2272 vector = 0; 2273 2274 for (i = 0; i < adapter->num_rx_queues; i++) { 2275 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) 2276 sprintf(adapter->rx_queue[i].name, "%s-rx-%d", 2277 adapter->netdev->name, vector); 2278 else 2279 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d", 2280 adapter->netdev->name, vector); 2281 err = request_irq(intr->msix_entries[vector].vector, 2282 vmxnet3_msix_rx, 0, 2283 adapter->rx_queue[i].name, 2284 &(adapter->rx_queue[i])); 2285 if (err) { 2286 netdev_err(adapter->netdev, 2287 "Failed to request irq for MSIX, " 2288 "%s, error %d\n", 2289 adapter->rx_queue[i].name, err); 2290 return err; 2291 } 2292 2293 adapter->rx_queue[i].comp_ring.intr_idx = vector++; 2294 } 2295 2296 sprintf(intr->event_msi_vector_name, "%s-event-%d", 2297 adapter->netdev->name, vector); 2298 err = request_irq(intr->msix_entries[vector].vector, 2299 vmxnet3_msix_event, 0, 2300 intr->event_msi_vector_name, adapter->netdev); 2301 intr->event_intr_idx = vector; 2302 2303 } else if (intr->type == VMXNET3_IT_MSI) { 2304 adapter->num_rx_queues = 1; 2305 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0, 2306 adapter->netdev->name, adapter->netdev); 2307 } else { 2308 #endif 2309 adapter->num_rx_queues = 1; 2310 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 2311 IRQF_SHARED, adapter->netdev->name, 2312 adapter->netdev); 2313 #ifdef CONFIG_PCI_MSI 2314 } 2315 #endif 2316 intr->num_intrs = vector + 1; 2317 if (err) { 2318 netdev_err(adapter->netdev, 2319 "Failed to request irq (intr type:%d), error %d\n", 2320 intr->type, err); 2321 } else { 2322 /* Number of rx queues will not change after this */ 2323 for (i = 0; i < adapter->num_rx_queues; i++) { 2324 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2325 rq->qid = i; 2326 rq->qid2 = i + adapter->num_rx_queues; 2327 rq->dataRingQid = i + 2 * adapter->num_rx_queues; 2328 } 2329 2330 /* init our intr settings */ 2331 for (i = 0; i < intr->num_intrs; i++) 2332 intr->mod_levels[i] = UPT1_IML_ADAPTIVE; 2333 if (adapter->intr.type != VMXNET3_IT_MSIX) { 2334 adapter->intr.event_intr_idx = 0; 2335 for (i = 0; i < adapter->num_tx_queues; i++) 2336 adapter->tx_queue[i].comp_ring.intr_idx = 0; 2337 adapter->rx_queue[0].comp_ring.intr_idx = 0; 2338 } 2339 2340 netdev_info(adapter->netdev, 2341 "intr type %u, mode %u, %u vectors allocated\n", 2342 intr->type, intr->mask_mode, intr->num_intrs); 2343 } 2344 2345 return err; 2346 } 2347 2348 2349 static void 2350 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter) 2351 { 2352 struct vmxnet3_intr *intr = &adapter->intr; 2353 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0); 2354 2355 switch (intr->type) { 2356 #ifdef CONFIG_PCI_MSI 2357 case VMXNET3_IT_MSIX: 2358 { 2359 int i, vector = 0; 2360 2361 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2362 for (i = 0; i < adapter->num_tx_queues; i++) { 2363 free_irq(intr->msix_entries[vector++].vector, 2364 &(adapter->tx_queue[i])); 2365 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) 2366 break; 2367 } 2368 } 2369 2370 for (i = 0; i < adapter->num_rx_queues; i++) { 2371 free_irq(intr->msix_entries[vector++].vector, 2372 &(adapter->rx_queue[i])); 2373 } 2374 2375 free_irq(intr->msix_entries[vector].vector, 2376 adapter->netdev); 2377 BUG_ON(vector >= intr->num_intrs); 2378 break; 2379 } 2380 #endif 2381 case VMXNET3_IT_MSI: 2382 free_irq(adapter->pdev->irq, adapter->netdev); 2383 break; 2384 case VMXNET3_IT_INTX: 2385 free_irq(adapter->pdev->irq, adapter->netdev); 2386 break; 2387 default: 2388 BUG(); 2389 } 2390 } 2391 2392 2393 static void 2394 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter) 2395 { 2396 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2397 u16 vid; 2398 2399 /* allow untagged pkts */ 2400 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0); 2401 2402 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2403 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2404 } 2405 2406 2407 static int 2408 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 2409 { 2410 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2411 2412 if (!(netdev->flags & IFF_PROMISC)) { 2413 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2414 unsigned long flags; 2415 2416 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2417 spin_lock_irqsave(&adapter->cmd_lock, flags); 2418 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2419 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2420 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2421 } 2422 2423 set_bit(vid, adapter->active_vlans); 2424 2425 return 0; 2426 } 2427 2428 2429 static int 2430 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 2431 { 2432 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2433 2434 if (!(netdev->flags & IFF_PROMISC)) { 2435 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2436 unsigned long flags; 2437 2438 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid); 2439 spin_lock_irqsave(&adapter->cmd_lock, flags); 2440 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2441 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2442 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2443 } 2444 2445 clear_bit(vid, adapter->active_vlans); 2446 2447 return 0; 2448 } 2449 2450 2451 static u8 * 2452 vmxnet3_copy_mc(struct net_device *netdev) 2453 { 2454 u8 *buf = NULL; 2455 u32 sz = netdev_mc_count(netdev) * ETH_ALEN; 2456 2457 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */ 2458 if (sz <= 0xffff) { 2459 /* We may be called with BH disabled */ 2460 buf = kmalloc(sz, GFP_ATOMIC); 2461 if (buf) { 2462 struct netdev_hw_addr *ha; 2463 int i = 0; 2464 2465 netdev_for_each_mc_addr(ha, netdev) 2466 memcpy(buf + i++ * ETH_ALEN, ha->addr, 2467 ETH_ALEN); 2468 } 2469 } 2470 return buf; 2471 } 2472 2473 2474 static void 2475 vmxnet3_set_mc(struct net_device *netdev) 2476 { 2477 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2478 unsigned long flags; 2479 struct Vmxnet3_RxFilterConf *rxConf = 2480 &adapter->shared->devRead.rxFilterConf; 2481 u8 *new_table = NULL; 2482 dma_addr_t new_table_pa = 0; 2483 bool new_table_pa_valid = false; 2484 u32 new_mode = VMXNET3_RXM_UCAST; 2485 2486 if (netdev->flags & IFF_PROMISC) { 2487 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2488 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable)); 2489 2490 new_mode |= VMXNET3_RXM_PROMISC; 2491 } else { 2492 vmxnet3_restore_vlan(adapter); 2493 } 2494 2495 if (netdev->flags & IFF_BROADCAST) 2496 new_mode |= VMXNET3_RXM_BCAST; 2497 2498 if (netdev->flags & IFF_ALLMULTI) 2499 new_mode |= VMXNET3_RXM_ALL_MULTI; 2500 else 2501 if (!netdev_mc_empty(netdev)) { 2502 new_table = vmxnet3_copy_mc(netdev); 2503 if (new_table) { 2504 size_t sz = netdev_mc_count(netdev) * ETH_ALEN; 2505 2506 rxConf->mfTableLen = cpu_to_le16(sz); 2507 new_table_pa = dma_map_single( 2508 &adapter->pdev->dev, 2509 new_table, 2510 sz, 2511 DMA_TO_DEVICE); 2512 if (!dma_mapping_error(&adapter->pdev->dev, 2513 new_table_pa)) { 2514 new_mode |= VMXNET3_RXM_MCAST; 2515 new_table_pa_valid = true; 2516 rxConf->mfTablePA = cpu_to_le64( 2517 new_table_pa); 2518 } 2519 } 2520 if (!new_table_pa_valid) { 2521 netdev_info(netdev, 2522 "failed to copy mcast list, setting ALL_MULTI\n"); 2523 new_mode |= VMXNET3_RXM_ALL_MULTI; 2524 } 2525 } 2526 2527 if (!(new_mode & VMXNET3_RXM_MCAST)) { 2528 rxConf->mfTableLen = 0; 2529 rxConf->mfTablePA = 0; 2530 } 2531 2532 spin_lock_irqsave(&adapter->cmd_lock, flags); 2533 if (new_mode != rxConf->rxMode) { 2534 rxConf->rxMode = cpu_to_le32(new_mode); 2535 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2536 VMXNET3_CMD_UPDATE_RX_MODE); 2537 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2538 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2539 } 2540 2541 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2542 VMXNET3_CMD_UPDATE_MAC_FILTERS); 2543 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2544 2545 if (new_table_pa_valid) 2546 dma_unmap_single(&adapter->pdev->dev, new_table_pa, 2547 rxConf->mfTableLen, DMA_TO_DEVICE); 2548 kfree(new_table); 2549 } 2550 2551 void 2552 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter) 2553 { 2554 int i; 2555 2556 for (i = 0; i < adapter->num_rx_queues; i++) 2557 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter); 2558 } 2559 2560 2561 /* 2562 * Set up driver_shared based on settings in adapter. 2563 */ 2564 2565 static void 2566 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter) 2567 { 2568 struct Vmxnet3_DriverShared *shared = adapter->shared; 2569 struct Vmxnet3_DSDevRead *devRead = &shared->devRead; 2570 struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt; 2571 struct Vmxnet3_TxQueueConf *tqc; 2572 struct Vmxnet3_RxQueueConf *rqc; 2573 int i; 2574 2575 memset(shared, 0, sizeof(*shared)); 2576 2577 /* driver settings */ 2578 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC); 2579 devRead->misc.driverInfo.version = cpu_to_le32( 2580 VMXNET3_DRIVER_VERSION_NUM); 2581 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ? 2582 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64); 2583 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; 2584 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32( 2585 *((u32 *)&devRead->misc.driverInfo.gos)); 2586 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1); 2587 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1); 2588 2589 devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa); 2590 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter)); 2591 2592 /* set up feature flags */ 2593 if (adapter->netdev->features & NETIF_F_RXCSUM) 2594 devRead->misc.uptFeatures |= UPT1_F_RXCSUM; 2595 2596 if (adapter->netdev->features & NETIF_F_LRO) { 2597 devRead->misc.uptFeatures |= UPT1_F_LRO; 2598 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS); 2599 } 2600 if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2601 devRead->misc.uptFeatures |= UPT1_F_RXVLAN; 2602 2603 if (adapter->netdev->features & (NETIF_F_GSO_UDP_TUNNEL | 2604 NETIF_F_GSO_UDP_TUNNEL_CSUM)) 2605 devRead->misc.uptFeatures |= UPT1_F_RXINNEROFLD; 2606 2607 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu); 2608 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa); 2609 devRead->misc.queueDescLen = cpu_to_le32( 2610 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + 2611 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc)); 2612 2613 /* tx queue settings */ 2614 devRead->misc.numTxQueues = adapter->num_tx_queues; 2615 for (i = 0; i < adapter->num_tx_queues; i++) { 2616 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 2617 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL); 2618 tqc = &adapter->tqd_start[i].conf; 2619 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA); 2620 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA); 2621 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA); 2622 tqc->ddPA = cpu_to_le64(~0ULL); 2623 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size); 2624 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size); 2625 tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size); 2626 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size); 2627 tqc->ddLen = cpu_to_le32(0); 2628 tqc->intrIdx = tq->comp_ring.intr_idx; 2629 } 2630 2631 /* rx queue settings */ 2632 devRead->misc.numRxQueues = adapter->num_rx_queues; 2633 for (i = 0; i < adapter->num_rx_queues; i++) { 2634 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2635 rqc = &adapter->rqd_start[i].conf; 2636 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA); 2637 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA); 2638 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA); 2639 rqc->ddPA = cpu_to_le64(~0ULL); 2640 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size); 2641 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size); 2642 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size); 2643 rqc->ddLen = cpu_to_le32(0); 2644 rqc->intrIdx = rq->comp_ring.intr_idx; 2645 if (VMXNET3_VERSION_GE_3(adapter)) { 2646 rqc->rxDataRingBasePA = 2647 cpu_to_le64(rq->data_ring.basePA); 2648 rqc->rxDataRingDescSize = 2649 cpu_to_le16(rq->data_ring.desc_size); 2650 } 2651 } 2652 2653 #ifdef VMXNET3_RSS 2654 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf)); 2655 2656 if (adapter->rss) { 2657 struct UPT1_RSSConf *rssConf = adapter->rss_conf; 2658 2659 devRead->misc.uptFeatures |= UPT1_F_RSS; 2660 devRead->misc.numRxQueues = adapter->num_rx_queues; 2661 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 | 2662 UPT1_RSS_HASH_TYPE_IPV4 | 2663 UPT1_RSS_HASH_TYPE_TCP_IPV6 | 2664 UPT1_RSS_HASH_TYPE_IPV6; 2665 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ; 2666 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE; 2667 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE; 2668 netdev_rss_key_fill(rssConf->hashKey, sizeof(rssConf->hashKey)); 2669 2670 for (i = 0; i < rssConf->indTableSize; i++) 2671 rssConf->indTable[i] = ethtool_rxfh_indir_default( 2672 i, adapter->num_rx_queues); 2673 2674 devRead->rssConfDesc.confVer = 1; 2675 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf)); 2676 devRead->rssConfDesc.confPA = 2677 cpu_to_le64(adapter->rss_conf_pa); 2678 } 2679 2680 #endif /* VMXNET3_RSS */ 2681 2682 /* intr settings */ 2683 if (!VMXNET3_VERSION_GE_6(adapter) || 2684 !adapter->queuesExtEnabled) { 2685 devRead->intrConf.autoMask = adapter->intr.mask_mode == 2686 VMXNET3_IMM_AUTO; 2687 devRead->intrConf.numIntrs = adapter->intr.num_intrs; 2688 for (i = 0; i < adapter->intr.num_intrs; i++) 2689 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i]; 2690 2691 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx; 2692 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 2693 } else { 2694 devReadExt->intrConfExt.autoMask = adapter->intr.mask_mode == 2695 VMXNET3_IMM_AUTO; 2696 devReadExt->intrConfExt.numIntrs = adapter->intr.num_intrs; 2697 for (i = 0; i < adapter->intr.num_intrs; i++) 2698 devReadExt->intrConfExt.modLevels[i] = adapter->intr.mod_levels[i]; 2699 2700 devReadExt->intrConfExt.eventIntrIdx = adapter->intr.event_intr_idx; 2701 devReadExt->intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 2702 } 2703 2704 /* rx filter settings */ 2705 devRead->rxFilterConf.rxMode = 0; 2706 vmxnet3_restore_vlan(adapter); 2707 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr); 2708 2709 /* the rest are already zeroed */ 2710 } 2711 2712 static void 2713 vmxnet3_init_bufsize(struct vmxnet3_adapter *adapter) 2714 { 2715 struct Vmxnet3_DriverShared *shared = adapter->shared; 2716 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 2717 unsigned long flags; 2718 2719 if (!VMXNET3_VERSION_GE_7(adapter)) 2720 return; 2721 2722 cmdInfo->ringBufSize = adapter->ringBufSize; 2723 spin_lock_irqsave(&adapter->cmd_lock, flags); 2724 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2725 VMXNET3_CMD_SET_RING_BUFFER_SIZE); 2726 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2727 } 2728 2729 static void 2730 vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter) 2731 { 2732 struct Vmxnet3_DriverShared *shared = adapter->shared; 2733 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 2734 unsigned long flags; 2735 2736 if (!VMXNET3_VERSION_GE_3(adapter)) 2737 return; 2738 2739 spin_lock_irqsave(&adapter->cmd_lock, flags); 2740 cmdInfo->varConf.confVer = 1; 2741 cmdInfo->varConf.confLen = 2742 cpu_to_le32(sizeof(*adapter->coal_conf)); 2743 cmdInfo->varConf.confPA = cpu_to_le64(adapter->coal_conf_pa); 2744 2745 if (adapter->default_coal_mode) { 2746 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2747 VMXNET3_CMD_GET_COALESCE); 2748 } else { 2749 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2750 VMXNET3_CMD_SET_COALESCE); 2751 } 2752 2753 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2754 } 2755 2756 static void 2757 vmxnet3_init_rssfields(struct vmxnet3_adapter *adapter) 2758 { 2759 struct Vmxnet3_DriverShared *shared = adapter->shared; 2760 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 2761 unsigned long flags; 2762 2763 if (!VMXNET3_VERSION_GE_4(adapter)) 2764 return; 2765 2766 spin_lock_irqsave(&adapter->cmd_lock, flags); 2767 2768 if (adapter->default_rss_fields) { 2769 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2770 VMXNET3_CMD_GET_RSS_FIELDS); 2771 adapter->rss_fields = 2772 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2773 } else { 2774 if (VMXNET3_VERSION_GE_7(adapter)) { 2775 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 || 2776 adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) && 2777 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 2778 VMXNET3_CAP_UDP_RSS)) { 2779 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS; 2780 } else { 2781 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS); 2782 } 2783 2784 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) && 2785 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 2786 VMXNET3_CAP_ESP_RSS_IPV4)) { 2787 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4; 2788 } else { 2789 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4); 2790 } 2791 2792 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) && 2793 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 2794 VMXNET3_CAP_ESP_RSS_IPV6)) { 2795 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6; 2796 } else { 2797 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6); 2798 } 2799 2800 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 2801 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 2802 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2803 } 2804 cmdInfo->setRssFields = adapter->rss_fields; 2805 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2806 VMXNET3_CMD_SET_RSS_FIELDS); 2807 /* Not all requested RSS may get applied, so get and 2808 * cache what was actually applied. 2809 */ 2810 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2811 VMXNET3_CMD_GET_RSS_FIELDS); 2812 adapter->rss_fields = 2813 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2814 } 2815 2816 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2817 } 2818 2819 int 2820 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter) 2821 { 2822 int err, i; 2823 u32 ret; 2824 unsigned long flags; 2825 2826 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d," 2827 " ring sizes %u %u %u\n", adapter->netdev->name, 2828 adapter->skb_buf_size, adapter->rx_buf_per_pkt, 2829 adapter->tx_queue[0].tx_ring.size, 2830 adapter->rx_queue[0].rx_ring[0].size, 2831 adapter->rx_queue[0].rx_ring[1].size); 2832 2833 vmxnet3_tq_init_all(adapter); 2834 err = vmxnet3_rq_init_all(adapter); 2835 if (err) { 2836 netdev_err(adapter->netdev, 2837 "Failed to init rx queue error %d\n", err); 2838 goto rq_err; 2839 } 2840 2841 err = vmxnet3_request_irqs(adapter); 2842 if (err) { 2843 netdev_err(adapter->netdev, 2844 "Failed to setup irq for error %d\n", err); 2845 goto irq_err; 2846 } 2847 2848 vmxnet3_setup_driver_shared(adapter); 2849 2850 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO( 2851 adapter->shared_pa)); 2852 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI( 2853 adapter->shared_pa)); 2854 spin_lock_irqsave(&adapter->cmd_lock, flags); 2855 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2856 VMXNET3_CMD_ACTIVATE_DEV); 2857 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2858 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2859 2860 if (ret != 0) { 2861 netdev_err(adapter->netdev, 2862 "Failed to activate dev: error %u\n", ret); 2863 err = -EINVAL; 2864 goto activate_err; 2865 } 2866 2867 vmxnet3_init_bufsize(adapter); 2868 vmxnet3_init_coalesce(adapter); 2869 vmxnet3_init_rssfields(adapter); 2870 2871 for (i = 0; i < adapter->num_rx_queues; i++) { 2872 VMXNET3_WRITE_BAR0_REG(adapter, 2873 adapter->rx_prod_offset + i * VMXNET3_REG_ALIGN, 2874 adapter->rx_queue[i].rx_ring[0].next2fill); 2875 VMXNET3_WRITE_BAR0_REG(adapter, (adapter->rx_prod2_offset + 2876 (i * VMXNET3_REG_ALIGN)), 2877 adapter->rx_queue[i].rx_ring[1].next2fill); 2878 } 2879 2880 /* Apply the rx filter settins last. */ 2881 vmxnet3_set_mc(adapter->netdev); 2882 2883 /* 2884 * Check link state when first activating device. It will start the 2885 * tx queue if the link is up. 2886 */ 2887 vmxnet3_check_link(adapter, true); 2888 netif_tx_wake_all_queues(adapter->netdev); 2889 for (i = 0; i < adapter->num_rx_queues; i++) 2890 napi_enable(&adapter->rx_queue[i].napi); 2891 vmxnet3_enable_all_intrs(adapter); 2892 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 2893 return 0; 2894 2895 activate_err: 2896 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0); 2897 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0); 2898 vmxnet3_free_irqs(adapter); 2899 irq_err: 2900 rq_err: 2901 /* free up buffers we allocated */ 2902 vmxnet3_rq_cleanup_all(adapter); 2903 return err; 2904 } 2905 2906 2907 void 2908 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter) 2909 { 2910 unsigned long flags; 2911 spin_lock_irqsave(&adapter->cmd_lock, flags); 2912 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); 2913 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2914 } 2915 2916 2917 int 2918 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter) 2919 { 2920 int i; 2921 unsigned long flags; 2922 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state)) 2923 return 0; 2924 2925 2926 spin_lock_irqsave(&adapter->cmd_lock, flags); 2927 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2928 VMXNET3_CMD_QUIESCE_DEV); 2929 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2930 vmxnet3_disable_all_intrs(adapter); 2931 2932 for (i = 0; i < adapter->num_rx_queues; i++) 2933 napi_disable(&adapter->rx_queue[i].napi); 2934 netif_tx_disable(adapter->netdev); 2935 adapter->link_speed = 0; 2936 netif_carrier_off(adapter->netdev); 2937 2938 vmxnet3_tq_cleanup_all(adapter); 2939 vmxnet3_rq_cleanup_all(adapter); 2940 vmxnet3_free_irqs(adapter); 2941 return 0; 2942 } 2943 2944 2945 static void 2946 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac) 2947 { 2948 u32 tmp; 2949 2950 tmp = *(u32 *)mac; 2951 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp); 2952 2953 tmp = (mac[5] << 8) | mac[4]; 2954 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp); 2955 } 2956 2957 2958 static int 2959 vmxnet3_set_mac_addr(struct net_device *netdev, void *p) 2960 { 2961 struct sockaddr *addr = p; 2962 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2963 2964 dev_addr_set(netdev, addr->sa_data); 2965 vmxnet3_write_mac_addr(adapter, addr->sa_data); 2966 2967 return 0; 2968 } 2969 2970 2971 /* ==================== initialization and cleanup routines ============ */ 2972 2973 static int 2974 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter) 2975 { 2976 int err; 2977 unsigned long mmio_start, mmio_len; 2978 struct pci_dev *pdev = adapter->pdev; 2979 2980 err = pci_enable_device(pdev); 2981 if (err) { 2982 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err); 2983 return err; 2984 } 2985 2986 err = pci_request_selected_regions(pdev, (1 << 2) - 1, 2987 vmxnet3_driver_name); 2988 if (err) { 2989 dev_err(&pdev->dev, 2990 "Failed to request region for adapter: error %d\n", err); 2991 goto err_enable_device; 2992 } 2993 2994 pci_set_master(pdev); 2995 2996 mmio_start = pci_resource_start(pdev, 0); 2997 mmio_len = pci_resource_len(pdev, 0); 2998 adapter->hw_addr0 = ioremap(mmio_start, mmio_len); 2999 if (!adapter->hw_addr0) { 3000 dev_err(&pdev->dev, "Failed to map bar0\n"); 3001 err = -EIO; 3002 goto err_ioremap; 3003 } 3004 3005 mmio_start = pci_resource_start(pdev, 1); 3006 mmio_len = pci_resource_len(pdev, 1); 3007 adapter->hw_addr1 = ioremap(mmio_start, mmio_len); 3008 if (!adapter->hw_addr1) { 3009 dev_err(&pdev->dev, "Failed to map bar1\n"); 3010 err = -EIO; 3011 goto err_bar1; 3012 } 3013 return 0; 3014 3015 err_bar1: 3016 iounmap(adapter->hw_addr0); 3017 err_ioremap: 3018 pci_release_selected_regions(pdev, (1 << 2) - 1); 3019 err_enable_device: 3020 pci_disable_device(pdev); 3021 return err; 3022 } 3023 3024 3025 static void 3026 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter) 3027 { 3028 BUG_ON(!adapter->pdev); 3029 3030 iounmap(adapter->hw_addr0); 3031 iounmap(adapter->hw_addr1); 3032 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1); 3033 pci_disable_device(adapter->pdev); 3034 } 3035 3036 3037 static void 3038 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter) 3039 { 3040 size_t sz, i, ring0_size, ring1_size, comp_size; 3041 /* With version7 ring1 will have only T0 buffers */ 3042 if (!VMXNET3_VERSION_GE_7(adapter)) { 3043 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE - 3044 VMXNET3_MAX_ETH_HDR_SIZE) { 3045 adapter->skb_buf_size = adapter->netdev->mtu + 3046 VMXNET3_MAX_ETH_HDR_SIZE; 3047 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE) 3048 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE; 3049 3050 adapter->rx_buf_per_pkt = 1; 3051 } else { 3052 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE; 3053 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE + 3054 VMXNET3_MAX_ETH_HDR_SIZE; 3055 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE; 3056 } 3057 } else { 3058 adapter->skb_buf_size = min((int)adapter->netdev->mtu + VMXNET3_MAX_ETH_HDR_SIZE, 3059 VMXNET3_MAX_SKB_BUF_SIZE); 3060 adapter->rx_buf_per_pkt = 1; 3061 adapter->ringBufSize.ring1BufSizeType0 = cpu_to_le16(adapter->skb_buf_size); 3062 adapter->ringBufSize.ring1BufSizeType1 = 0; 3063 adapter->ringBufSize.ring2BufSizeType1 = cpu_to_le16(PAGE_SIZE); 3064 } 3065 3066 /* 3067 * for simplicity, force the ring0 size to be a multiple of 3068 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN 3069 */ 3070 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN; 3071 ring0_size = adapter->rx_queue[0].rx_ring[0].size; 3072 ring0_size = (ring0_size + sz - 1) / sz * sz; 3073 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE / 3074 sz * sz); 3075 ring1_size = adapter->rx_queue[0].rx_ring[1].size; 3076 ring1_size = (ring1_size + sz - 1) / sz * sz; 3077 ring1_size = min_t(u32, ring1_size, VMXNET3_RX_RING2_MAX_SIZE / 3078 sz * sz); 3079 /* For v7 and later, keep ring size power of 2 for UPT */ 3080 if (VMXNET3_VERSION_GE_7(adapter)) { 3081 ring0_size = rounddown_pow_of_two(ring0_size); 3082 ring1_size = rounddown_pow_of_two(ring1_size); 3083 } 3084 comp_size = ring0_size + ring1_size; 3085 3086 for (i = 0; i < adapter->num_rx_queues; i++) { 3087 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3088 3089 rq->rx_ring[0].size = ring0_size; 3090 rq->rx_ring[1].size = ring1_size; 3091 rq->comp_ring.size = comp_size; 3092 } 3093 } 3094 3095 3096 int 3097 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size, 3098 u32 rx_ring_size, u32 rx_ring2_size, 3099 u16 txdata_desc_size, u16 rxdata_desc_size) 3100 { 3101 int err = 0, i; 3102 3103 for (i = 0; i < adapter->num_tx_queues; i++) { 3104 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 3105 tq->tx_ring.size = tx_ring_size; 3106 tq->data_ring.size = tx_ring_size; 3107 tq->comp_ring.size = tx_ring_size; 3108 tq->txdata_desc_size = txdata_desc_size; 3109 tq->shared = &adapter->tqd_start[i].ctrl; 3110 tq->stopped = true; 3111 tq->adapter = adapter; 3112 tq->qid = i; 3113 err = vmxnet3_tq_create(tq, adapter); 3114 /* 3115 * Too late to change num_tx_queues. We cannot do away with 3116 * lesser number of queues than what we asked for 3117 */ 3118 if (err) 3119 goto queue_err; 3120 } 3121 3122 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size; 3123 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size; 3124 vmxnet3_adjust_rx_ring_size(adapter); 3125 3126 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 3127 for (i = 0; i < adapter->num_rx_queues; i++) { 3128 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3129 /* qid and qid2 for rx queues will be assigned later when num 3130 * of rx queues is finalized after allocating intrs */ 3131 rq->shared = &adapter->rqd_start[i].ctrl; 3132 rq->adapter = adapter; 3133 rq->data_ring.desc_size = rxdata_desc_size; 3134 err = vmxnet3_rq_create(rq, adapter); 3135 if (err) { 3136 if (i == 0) { 3137 netdev_err(adapter->netdev, 3138 "Could not allocate any rx queues. " 3139 "Aborting.\n"); 3140 goto queue_err; 3141 } else { 3142 netdev_info(adapter->netdev, 3143 "Number of rx queues changed " 3144 "to : %d.\n", i); 3145 adapter->num_rx_queues = i; 3146 err = 0; 3147 break; 3148 } 3149 } 3150 } 3151 3152 if (!adapter->rxdataring_enabled) 3153 vmxnet3_rq_destroy_all_rxdataring(adapter); 3154 3155 return err; 3156 queue_err: 3157 vmxnet3_tq_destroy_all(adapter); 3158 return err; 3159 } 3160 3161 static int 3162 vmxnet3_open(struct net_device *netdev) 3163 { 3164 struct vmxnet3_adapter *adapter; 3165 int err, i; 3166 3167 adapter = netdev_priv(netdev); 3168 3169 for (i = 0; i < adapter->num_tx_queues; i++) 3170 spin_lock_init(&adapter->tx_queue[i].tx_lock); 3171 3172 if (VMXNET3_VERSION_GE_3(adapter)) { 3173 unsigned long flags; 3174 u16 txdata_desc_size; 3175 3176 spin_lock_irqsave(&adapter->cmd_lock, flags); 3177 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3178 VMXNET3_CMD_GET_TXDATA_DESC_SIZE); 3179 txdata_desc_size = VMXNET3_READ_BAR1_REG(adapter, 3180 VMXNET3_REG_CMD); 3181 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3182 3183 if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) || 3184 (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) || 3185 (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) { 3186 adapter->txdata_desc_size = 3187 sizeof(struct Vmxnet3_TxDataDesc); 3188 } else { 3189 adapter->txdata_desc_size = txdata_desc_size; 3190 } 3191 } else { 3192 adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc); 3193 } 3194 3195 err = vmxnet3_create_queues(adapter, 3196 adapter->tx_ring_size, 3197 adapter->rx_ring_size, 3198 adapter->rx_ring2_size, 3199 adapter->txdata_desc_size, 3200 adapter->rxdata_desc_size); 3201 if (err) 3202 goto queue_err; 3203 3204 err = vmxnet3_activate_dev(adapter); 3205 if (err) 3206 goto activate_err; 3207 3208 return 0; 3209 3210 activate_err: 3211 vmxnet3_rq_destroy_all(adapter); 3212 vmxnet3_tq_destroy_all(adapter); 3213 queue_err: 3214 return err; 3215 } 3216 3217 3218 static int 3219 vmxnet3_close(struct net_device *netdev) 3220 { 3221 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3222 3223 /* 3224 * Reset_work may be in the middle of resetting the device, wait for its 3225 * completion. 3226 */ 3227 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3228 usleep_range(1000, 2000); 3229 3230 vmxnet3_quiesce_dev(adapter); 3231 3232 vmxnet3_rq_destroy_all(adapter); 3233 vmxnet3_tq_destroy_all(adapter); 3234 3235 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3236 3237 3238 return 0; 3239 } 3240 3241 3242 void 3243 vmxnet3_force_close(struct vmxnet3_adapter *adapter) 3244 { 3245 int i; 3246 3247 /* 3248 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise 3249 * vmxnet3_close() will deadlock. 3250 */ 3251 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)); 3252 3253 /* we need to enable NAPI, otherwise dev_close will deadlock */ 3254 for (i = 0; i < adapter->num_rx_queues; i++) 3255 napi_enable(&adapter->rx_queue[i].napi); 3256 /* 3257 * Need to clear the quiesce bit to ensure that vmxnet3_close 3258 * can quiesce the device properly 3259 */ 3260 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3261 dev_close(adapter->netdev); 3262 } 3263 3264 3265 static int 3266 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu) 3267 { 3268 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3269 int err = 0; 3270 3271 netdev->mtu = new_mtu; 3272 3273 /* 3274 * Reset_work may be in the middle of resetting the device, wait for its 3275 * completion. 3276 */ 3277 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3278 usleep_range(1000, 2000); 3279 3280 if (netif_running(netdev)) { 3281 vmxnet3_quiesce_dev(adapter); 3282 vmxnet3_reset_dev(adapter); 3283 3284 /* we need to re-create the rx queue based on the new mtu */ 3285 vmxnet3_rq_destroy_all(adapter); 3286 vmxnet3_adjust_rx_ring_size(adapter); 3287 err = vmxnet3_rq_create_all(adapter); 3288 if (err) { 3289 netdev_err(netdev, 3290 "failed to re-create rx queues, " 3291 " error %d. Closing it.\n", err); 3292 goto out; 3293 } 3294 3295 err = vmxnet3_activate_dev(adapter); 3296 if (err) { 3297 netdev_err(netdev, 3298 "failed to re-activate, error %d. " 3299 "Closing it\n", err); 3300 goto out; 3301 } 3302 } 3303 3304 out: 3305 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3306 if (err) 3307 vmxnet3_force_close(adapter); 3308 3309 return err; 3310 } 3311 3312 3313 static void 3314 vmxnet3_declare_features(struct vmxnet3_adapter *adapter) 3315 { 3316 struct net_device *netdev = adapter->netdev; 3317 3318 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | 3319 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3320 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3321 NETIF_F_LRO | NETIF_F_HIGHDMA; 3322 3323 if (VMXNET3_VERSION_GE_4(adapter)) { 3324 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 3325 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3326 3327 netdev->hw_enc_features = NETIF_F_SG | NETIF_F_RXCSUM | 3328 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3329 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3330 NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL | 3331 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3332 } 3333 3334 if (VMXNET3_VERSION_GE_7(adapter)) { 3335 unsigned long flags; 3336 3337 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3338 VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) { 3339 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD; 3340 } 3341 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3342 VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) { 3343 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD; 3344 } 3345 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3346 VMXNET3_CAP_GENEVE_TSO)) { 3347 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO; 3348 } 3349 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3350 VMXNET3_CAP_VXLAN_TSO)) { 3351 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO; 3352 } 3353 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3354 VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) { 3355 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD; 3356 } 3357 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3358 VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) { 3359 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD; 3360 } 3361 3362 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3363 spin_lock_irqsave(&adapter->cmd_lock, flags); 3364 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3365 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3366 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3367 3368 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) && 3369 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) { 3370 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3371 netdev->features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3372 } 3373 } 3374 3375 netdev->vlan_features = netdev->hw_features & 3376 ~(NETIF_F_HW_VLAN_CTAG_TX | 3377 NETIF_F_HW_VLAN_CTAG_RX); 3378 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 3379 } 3380 3381 3382 static void 3383 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac) 3384 { 3385 u32 tmp; 3386 3387 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL); 3388 *(u32 *)mac = tmp; 3389 3390 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH); 3391 mac[4] = tmp & 0xff; 3392 mac[5] = (tmp >> 8) & 0xff; 3393 } 3394 3395 #ifdef CONFIG_PCI_MSI 3396 3397 /* 3398 * Enable MSIx vectors. 3399 * Returns : 3400 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required 3401 * were enabled. 3402 * number of vectors which were enabled otherwise (this number is greater 3403 * than VMXNET3_LINUX_MIN_MSIX_VECT) 3404 */ 3405 3406 static int 3407 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec) 3408 { 3409 int ret = pci_enable_msix_range(adapter->pdev, 3410 adapter->intr.msix_entries, nvec, nvec); 3411 3412 if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) { 3413 dev_err(&adapter->netdev->dev, 3414 "Failed to enable %d MSI-X, trying %d\n", 3415 nvec, VMXNET3_LINUX_MIN_MSIX_VECT); 3416 3417 ret = pci_enable_msix_range(adapter->pdev, 3418 adapter->intr.msix_entries, 3419 VMXNET3_LINUX_MIN_MSIX_VECT, 3420 VMXNET3_LINUX_MIN_MSIX_VECT); 3421 } 3422 3423 if (ret < 0) { 3424 dev_err(&adapter->netdev->dev, 3425 "Failed to enable MSI-X, error: %d\n", ret); 3426 } 3427 3428 return ret; 3429 } 3430 3431 3432 #endif /* CONFIG_PCI_MSI */ 3433 3434 static void 3435 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter) 3436 { 3437 u32 cfg; 3438 unsigned long flags; 3439 3440 /* intr settings */ 3441 spin_lock_irqsave(&adapter->cmd_lock, flags); 3442 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3443 VMXNET3_CMD_GET_CONF_INTR); 3444 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3445 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3446 adapter->intr.type = cfg & 0x3; 3447 adapter->intr.mask_mode = (cfg >> 2) & 0x3; 3448 3449 if (adapter->intr.type == VMXNET3_IT_AUTO) { 3450 adapter->intr.type = VMXNET3_IT_MSIX; 3451 } 3452 3453 #ifdef CONFIG_PCI_MSI 3454 if (adapter->intr.type == VMXNET3_IT_MSIX) { 3455 int i, nvec, nvec_allocated; 3456 3457 nvec = adapter->share_intr == VMXNET3_INTR_TXSHARE ? 3458 1 : adapter->num_tx_queues; 3459 nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ? 3460 0 : adapter->num_rx_queues; 3461 nvec += 1; /* for link event */ 3462 nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ? 3463 nvec : VMXNET3_LINUX_MIN_MSIX_VECT; 3464 3465 for (i = 0; i < nvec; i++) 3466 adapter->intr.msix_entries[i].entry = i; 3467 3468 nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec); 3469 if (nvec_allocated < 0) 3470 goto msix_err; 3471 3472 /* If we cannot allocate one MSIx vector per queue 3473 * then limit the number of rx queues to 1 3474 */ 3475 if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT && 3476 nvec != VMXNET3_LINUX_MIN_MSIX_VECT) { 3477 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE 3478 || adapter->num_rx_queues != 1) { 3479 adapter->share_intr = VMXNET3_INTR_TXSHARE; 3480 netdev_err(adapter->netdev, 3481 "Number of rx queues : 1\n"); 3482 adapter->num_rx_queues = 1; 3483 } 3484 } 3485 3486 adapter->intr.num_intrs = nvec_allocated; 3487 return; 3488 3489 msix_err: 3490 /* If we cannot allocate MSIx vectors use only one rx queue */ 3491 dev_info(&adapter->pdev->dev, 3492 "Failed to enable MSI-X, error %d. " 3493 "Limiting #rx queues to 1, try MSI.\n", nvec_allocated); 3494 3495 adapter->intr.type = VMXNET3_IT_MSI; 3496 } 3497 3498 if (adapter->intr.type == VMXNET3_IT_MSI) { 3499 if (!pci_enable_msi(adapter->pdev)) { 3500 adapter->num_rx_queues = 1; 3501 adapter->intr.num_intrs = 1; 3502 return; 3503 } 3504 } 3505 #endif /* CONFIG_PCI_MSI */ 3506 3507 adapter->num_rx_queues = 1; 3508 dev_info(&adapter->netdev->dev, 3509 "Using INTx interrupt, #Rx queues: 1.\n"); 3510 adapter->intr.type = VMXNET3_IT_INTX; 3511 3512 /* INT-X related setting */ 3513 adapter->intr.num_intrs = 1; 3514 } 3515 3516 3517 static void 3518 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter) 3519 { 3520 if (adapter->intr.type == VMXNET3_IT_MSIX) 3521 pci_disable_msix(adapter->pdev); 3522 else if (adapter->intr.type == VMXNET3_IT_MSI) 3523 pci_disable_msi(adapter->pdev); 3524 else 3525 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX); 3526 } 3527 3528 3529 static void 3530 vmxnet3_tx_timeout(struct net_device *netdev, unsigned int txqueue) 3531 { 3532 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3533 adapter->tx_timeout_count++; 3534 3535 netdev_err(adapter->netdev, "tx hang\n"); 3536 schedule_work(&adapter->work); 3537 } 3538 3539 3540 static void 3541 vmxnet3_reset_work(struct work_struct *data) 3542 { 3543 struct vmxnet3_adapter *adapter; 3544 3545 adapter = container_of(data, struct vmxnet3_adapter, work); 3546 3547 /* if another thread is resetting the device, no need to proceed */ 3548 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3549 return; 3550 3551 /* if the device is closed, we must leave it alone */ 3552 rtnl_lock(); 3553 if (netif_running(adapter->netdev)) { 3554 netdev_notice(adapter->netdev, "resetting\n"); 3555 vmxnet3_quiesce_dev(adapter); 3556 vmxnet3_reset_dev(adapter); 3557 vmxnet3_activate_dev(adapter); 3558 } else { 3559 netdev_info(adapter->netdev, "already closed\n"); 3560 } 3561 rtnl_unlock(); 3562 3563 netif_wake_queue(adapter->netdev); 3564 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3565 } 3566 3567 3568 static int 3569 vmxnet3_probe_device(struct pci_dev *pdev, 3570 const struct pci_device_id *id) 3571 { 3572 static const struct net_device_ops vmxnet3_netdev_ops = { 3573 .ndo_open = vmxnet3_open, 3574 .ndo_stop = vmxnet3_close, 3575 .ndo_start_xmit = vmxnet3_xmit_frame, 3576 .ndo_set_mac_address = vmxnet3_set_mac_addr, 3577 .ndo_change_mtu = vmxnet3_change_mtu, 3578 .ndo_fix_features = vmxnet3_fix_features, 3579 .ndo_set_features = vmxnet3_set_features, 3580 .ndo_features_check = vmxnet3_features_check, 3581 .ndo_get_stats64 = vmxnet3_get_stats64, 3582 .ndo_tx_timeout = vmxnet3_tx_timeout, 3583 .ndo_set_rx_mode = vmxnet3_set_mc, 3584 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid, 3585 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid, 3586 #ifdef CONFIG_NET_POLL_CONTROLLER 3587 .ndo_poll_controller = vmxnet3_netpoll, 3588 #endif 3589 }; 3590 int err; 3591 u32 ver; 3592 struct net_device *netdev; 3593 struct vmxnet3_adapter *adapter; 3594 u8 mac[ETH_ALEN]; 3595 int size; 3596 int num_tx_queues; 3597 int num_rx_queues; 3598 int queues; 3599 unsigned long flags; 3600 3601 if (!pci_msi_enabled()) 3602 enable_mq = 0; 3603 3604 #ifdef VMXNET3_RSS 3605 if (enable_mq) 3606 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 3607 (int)num_online_cpus()); 3608 else 3609 #endif 3610 num_rx_queues = 1; 3611 3612 if (enable_mq) 3613 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES, 3614 (int)num_online_cpus()); 3615 else 3616 num_tx_queues = 1; 3617 3618 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter), 3619 max(num_tx_queues, num_rx_queues)); 3620 if (!netdev) 3621 return -ENOMEM; 3622 3623 pci_set_drvdata(pdev, netdev); 3624 adapter = netdev_priv(netdev); 3625 adapter->netdev = netdev; 3626 adapter->pdev = pdev; 3627 3628 adapter->tx_ring_size = VMXNET3_DEF_TX_RING_SIZE; 3629 adapter->rx_ring_size = VMXNET3_DEF_RX_RING_SIZE; 3630 adapter->rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE; 3631 3632 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3633 if (err) { 3634 dev_err(&pdev->dev, "dma_set_mask failed\n"); 3635 goto err_set_mask; 3636 } 3637 3638 spin_lock_init(&adapter->cmd_lock); 3639 adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter, 3640 sizeof(struct vmxnet3_adapter), 3641 DMA_TO_DEVICE); 3642 if (dma_mapping_error(&adapter->pdev->dev, adapter->adapter_pa)) { 3643 dev_err(&pdev->dev, "Failed to map dma\n"); 3644 err = -EFAULT; 3645 goto err_set_mask; 3646 } 3647 adapter->shared = dma_alloc_coherent( 3648 &adapter->pdev->dev, 3649 sizeof(struct Vmxnet3_DriverShared), 3650 &adapter->shared_pa, GFP_KERNEL); 3651 if (!adapter->shared) { 3652 dev_err(&pdev->dev, "Failed to allocate memory\n"); 3653 err = -ENOMEM; 3654 goto err_alloc_shared; 3655 } 3656 3657 err = vmxnet3_alloc_pci_resources(adapter); 3658 if (err < 0) 3659 goto err_alloc_pci; 3660 3661 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS); 3662 if (ver & (1 << VMXNET3_REV_7)) { 3663 VMXNET3_WRITE_BAR1_REG(adapter, 3664 VMXNET3_REG_VRRS, 3665 1 << VMXNET3_REV_7); 3666 adapter->version = VMXNET3_REV_7 + 1; 3667 } else if (ver & (1 << VMXNET3_REV_6)) { 3668 VMXNET3_WRITE_BAR1_REG(adapter, 3669 VMXNET3_REG_VRRS, 3670 1 << VMXNET3_REV_6); 3671 adapter->version = VMXNET3_REV_6 + 1; 3672 } else if (ver & (1 << VMXNET3_REV_5)) { 3673 VMXNET3_WRITE_BAR1_REG(adapter, 3674 VMXNET3_REG_VRRS, 3675 1 << VMXNET3_REV_5); 3676 adapter->version = VMXNET3_REV_5 + 1; 3677 } else if (ver & (1 << VMXNET3_REV_4)) { 3678 VMXNET3_WRITE_BAR1_REG(adapter, 3679 VMXNET3_REG_VRRS, 3680 1 << VMXNET3_REV_4); 3681 adapter->version = VMXNET3_REV_4 + 1; 3682 } else if (ver & (1 << VMXNET3_REV_3)) { 3683 VMXNET3_WRITE_BAR1_REG(adapter, 3684 VMXNET3_REG_VRRS, 3685 1 << VMXNET3_REV_3); 3686 adapter->version = VMXNET3_REV_3 + 1; 3687 } else if (ver & (1 << VMXNET3_REV_2)) { 3688 VMXNET3_WRITE_BAR1_REG(adapter, 3689 VMXNET3_REG_VRRS, 3690 1 << VMXNET3_REV_2); 3691 adapter->version = VMXNET3_REV_2 + 1; 3692 } else if (ver & (1 << VMXNET3_REV_1)) { 3693 VMXNET3_WRITE_BAR1_REG(adapter, 3694 VMXNET3_REG_VRRS, 3695 1 << VMXNET3_REV_1); 3696 adapter->version = VMXNET3_REV_1 + 1; 3697 } else { 3698 dev_err(&pdev->dev, 3699 "Incompatible h/w version (0x%x) for adapter\n", ver); 3700 err = -EBUSY; 3701 goto err_ver; 3702 } 3703 dev_dbg(&pdev->dev, "Using device version %d\n", adapter->version); 3704 3705 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS); 3706 if (ver & 1) { 3707 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1); 3708 } else { 3709 dev_err(&pdev->dev, 3710 "Incompatible upt version (0x%x) for adapter\n", ver); 3711 err = -EBUSY; 3712 goto err_ver; 3713 } 3714 3715 if (VMXNET3_VERSION_GE_7(adapter)) { 3716 adapter->devcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DCR); 3717 adapter->ptcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_PTCR); 3718 if (adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 3719 adapter->dev_caps[0] = adapter->devcap_supported[0] & 3720 (1UL << VMXNET3_CAP_LARGE_BAR); 3721 } 3722 if (!(adapter->ptcap_supported[0] & (1UL << VMXNET3_DCR_ERROR)) && 3723 adapter->ptcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP) && 3724 adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) { 3725 adapter->dev_caps[0] |= adapter->devcap_supported[0] & 3726 (1UL << VMXNET3_CAP_OOORX_COMP); 3727 } 3728 if (adapter->dev_caps[0]) 3729 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3730 3731 spin_lock_irqsave(&adapter->cmd_lock, flags); 3732 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3733 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3734 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3735 } 3736 3737 if (VMXNET3_VERSION_GE_7(adapter) && 3738 adapter->dev_caps[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 3739 adapter->tx_prod_offset = VMXNET3_REG_LB_TXPROD; 3740 adapter->rx_prod_offset = VMXNET3_REG_LB_RXPROD; 3741 adapter->rx_prod2_offset = VMXNET3_REG_LB_RXPROD2; 3742 } else { 3743 adapter->tx_prod_offset = VMXNET3_REG_TXPROD; 3744 adapter->rx_prod_offset = VMXNET3_REG_RXPROD; 3745 adapter->rx_prod2_offset = VMXNET3_REG_RXPROD2; 3746 } 3747 3748 if (VMXNET3_VERSION_GE_6(adapter)) { 3749 spin_lock_irqsave(&adapter->cmd_lock, flags); 3750 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3751 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 3752 queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3753 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3754 if (queues > 0) { 3755 adapter->num_rx_queues = min(num_rx_queues, ((queues >> 8) & 0xff)); 3756 adapter->num_tx_queues = min(num_tx_queues, (queues & 0xff)); 3757 } else { 3758 adapter->num_rx_queues = min(num_rx_queues, 3759 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3760 adapter->num_tx_queues = min(num_tx_queues, 3761 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 3762 } 3763 if (adapter->num_rx_queues > VMXNET3_MAX_RX_QUEUES || 3764 adapter->num_tx_queues > VMXNET3_MAX_TX_QUEUES) { 3765 adapter->queuesExtEnabled = true; 3766 } else { 3767 adapter->queuesExtEnabled = false; 3768 } 3769 } else { 3770 adapter->queuesExtEnabled = false; 3771 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 3772 num_tx_queues = rounddown_pow_of_two(num_tx_queues); 3773 adapter->num_rx_queues = min(num_rx_queues, 3774 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3775 adapter->num_tx_queues = min(num_tx_queues, 3776 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 3777 } 3778 dev_info(&pdev->dev, 3779 "# of Tx queues : %d, # of Rx queues : %d\n", 3780 adapter->num_tx_queues, adapter->num_rx_queues); 3781 3782 adapter->rx_buf_per_pkt = 1; 3783 3784 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 3785 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues; 3786 adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size, 3787 &adapter->queue_desc_pa, 3788 GFP_KERNEL); 3789 3790 if (!adapter->tqd_start) { 3791 dev_err(&pdev->dev, "Failed to allocate memory\n"); 3792 err = -ENOMEM; 3793 goto err_ver; 3794 } 3795 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start + 3796 adapter->num_tx_queues); 3797 3798 adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev, 3799 sizeof(struct Vmxnet3_PMConf), 3800 &adapter->pm_conf_pa, 3801 GFP_KERNEL); 3802 if (adapter->pm_conf == NULL) { 3803 err = -ENOMEM; 3804 goto err_alloc_pm; 3805 } 3806 3807 #ifdef VMXNET3_RSS 3808 3809 adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev, 3810 sizeof(struct UPT1_RSSConf), 3811 &adapter->rss_conf_pa, 3812 GFP_KERNEL); 3813 if (adapter->rss_conf == NULL) { 3814 err = -ENOMEM; 3815 goto err_alloc_rss; 3816 } 3817 #endif /* VMXNET3_RSS */ 3818 3819 if (VMXNET3_VERSION_GE_3(adapter)) { 3820 adapter->coal_conf = 3821 dma_alloc_coherent(&adapter->pdev->dev, 3822 sizeof(struct Vmxnet3_CoalesceScheme) 3823 , 3824 &adapter->coal_conf_pa, 3825 GFP_KERNEL); 3826 if (!adapter->coal_conf) { 3827 err = -ENOMEM; 3828 goto err_coal_conf; 3829 } 3830 adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED; 3831 adapter->default_coal_mode = true; 3832 } 3833 3834 if (VMXNET3_VERSION_GE_4(adapter)) { 3835 adapter->default_rss_fields = true; 3836 adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT; 3837 } 3838 3839 SET_NETDEV_DEV(netdev, &pdev->dev); 3840 vmxnet3_declare_features(adapter); 3841 3842 adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ? 3843 VMXNET3_DEF_RXDATA_DESC_SIZE : 0; 3844 3845 if (adapter->num_tx_queues == adapter->num_rx_queues) 3846 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE; 3847 else 3848 adapter->share_intr = VMXNET3_INTR_DONTSHARE; 3849 3850 vmxnet3_alloc_intr_resources(adapter); 3851 3852 #ifdef VMXNET3_RSS 3853 if (adapter->num_rx_queues > 1 && 3854 adapter->intr.type == VMXNET3_IT_MSIX) { 3855 adapter->rss = true; 3856 netdev->hw_features |= NETIF_F_RXHASH; 3857 netdev->features |= NETIF_F_RXHASH; 3858 dev_dbg(&pdev->dev, "RSS is enabled.\n"); 3859 } else { 3860 adapter->rss = false; 3861 } 3862 #endif 3863 3864 vmxnet3_read_mac_addr(adapter, mac); 3865 dev_addr_set(netdev, mac); 3866 3867 netdev->netdev_ops = &vmxnet3_netdev_ops; 3868 vmxnet3_set_ethtool_ops(netdev); 3869 netdev->watchdog_timeo = 5 * HZ; 3870 3871 /* MTU range: 60 - 9190 */ 3872 netdev->min_mtu = VMXNET3_MIN_MTU; 3873 if (VMXNET3_VERSION_GE_6(adapter)) 3874 netdev->max_mtu = VMXNET3_V6_MAX_MTU; 3875 else 3876 netdev->max_mtu = VMXNET3_MAX_MTU; 3877 3878 INIT_WORK(&adapter->work, vmxnet3_reset_work); 3879 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3880 3881 if (adapter->intr.type == VMXNET3_IT_MSIX) { 3882 int i; 3883 for (i = 0; i < adapter->num_rx_queues; i++) { 3884 netif_napi_add(adapter->netdev, 3885 &adapter->rx_queue[i].napi, 3886 vmxnet3_poll_rx_only, 64); 3887 } 3888 } else { 3889 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi, 3890 vmxnet3_poll, 64); 3891 } 3892 3893 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues); 3894 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues); 3895 3896 netif_carrier_off(netdev); 3897 err = register_netdev(netdev); 3898 3899 if (err) { 3900 dev_err(&pdev->dev, "Failed to register adapter\n"); 3901 goto err_register; 3902 } 3903 3904 vmxnet3_check_link(adapter, false); 3905 return 0; 3906 3907 err_register: 3908 if (VMXNET3_VERSION_GE_3(adapter)) { 3909 dma_free_coherent(&adapter->pdev->dev, 3910 sizeof(struct Vmxnet3_CoalesceScheme), 3911 adapter->coal_conf, adapter->coal_conf_pa); 3912 } 3913 vmxnet3_free_intr_resources(adapter); 3914 err_coal_conf: 3915 #ifdef VMXNET3_RSS 3916 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 3917 adapter->rss_conf, adapter->rss_conf_pa); 3918 err_alloc_rss: 3919 #endif 3920 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 3921 adapter->pm_conf, adapter->pm_conf_pa); 3922 err_alloc_pm: 3923 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 3924 adapter->queue_desc_pa); 3925 err_ver: 3926 vmxnet3_free_pci_resources(adapter); 3927 err_alloc_pci: 3928 dma_free_coherent(&adapter->pdev->dev, 3929 sizeof(struct Vmxnet3_DriverShared), 3930 adapter->shared, adapter->shared_pa); 3931 err_alloc_shared: 3932 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 3933 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 3934 err_set_mask: 3935 free_netdev(netdev); 3936 return err; 3937 } 3938 3939 3940 static void 3941 vmxnet3_remove_device(struct pci_dev *pdev) 3942 { 3943 struct net_device *netdev = pci_get_drvdata(pdev); 3944 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3945 int size = 0; 3946 int num_rx_queues, rx_queues; 3947 unsigned long flags; 3948 3949 #ifdef VMXNET3_RSS 3950 if (enable_mq) 3951 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 3952 (int)num_online_cpus()); 3953 else 3954 #endif 3955 num_rx_queues = 1; 3956 if (!VMXNET3_VERSION_GE_6(adapter)) { 3957 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 3958 } 3959 if (VMXNET3_VERSION_GE_6(adapter)) { 3960 spin_lock_irqsave(&adapter->cmd_lock, flags); 3961 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3962 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 3963 rx_queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3964 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3965 if (rx_queues > 0) 3966 rx_queues = (rx_queues >> 8) & 0xff; 3967 else 3968 rx_queues = min(num_rx_queues, VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3969 num_rx_queues = min(num_rx_queues, rx_queues); 3970 } else { 3971 num_rx_queues = min(num_rx_queues, 3972 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3973 } 3974 3975 cancel_work_sync(&adapter->work); 3976 3977 unregister_netdev(netdev); 3978 3979 vmxnet3_free_intr_resources(adapter); 3980 vmxnet3_free_pci_resources(adapter); 3981 if (VMXNET3_VERSION_GE_3(adapter)) { 3982 dma_free_coherent(&adapter->pdev->dev, 3983 sizeof(struct Vmxnet3_CoalesceScheme), 3984 adapter->coal_conf, adapter->coal_conf_pa); 3985 } 3986 #ifdef VMXNET3_RSS 3987 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 3988 adapter->rss_conf, adapter->rss_conf_pa); 3989 #endif 3990 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 3991 adapter->pm_conf, adapter->pm_conf_pa); 3992 3993 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 3994 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues; 3995 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 3996 adapter->queue_desc_pa); 3997 dma_free_coherent(&adapter->pdev->dev, 3998 sizeof(struct Vmxnet3_DriverShared), 3999 adapter->shared, adapter->shared_pa); 4000 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 4001 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 4002 free_netdev(netdev); 4003 } 4004 4005 static void vmxnet3_shutdown_device(struct pci_dev *pdev) 4006 { 4007 struct net_device *netdev = pci_get_drvdata(pdev); 4008 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4009 unsigned long flags; 4010 4011 /* Reset_work may be in the middle of resetting the device, wait for its 4012 * completion. 4013 */ 4014 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 4015 usleep_range(1000, 2000); 4016 4017 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, 4018 &adapter->state)) { 4019 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4020 return; 4021 } 4022 spin_lock_irqsave(&adapter->cmd_lock, flags); 4023 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4024 VMXNET3_CMD_QUIESCE_DEV); 4025 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4026 vmxnet3_disable_all_intrs(adapter); 4027 4028 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4029 } 4030 4031 4032 #ifdef CONFIG_PM 4033 4034 static int 4035 vmxnet3_suspend(struct device *device) 4036 { 4037 struct pci_dev *pdev = to_pci_dev(device); 4038 struct net_device *netdev = pci_get_drvdata(pdev); 4039 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4040 struct Vmxnet3_PMConf *pmConf; 4041 struct ethhdr *ehdr; 4042 struct arphdr *ahdr; 4043 u8 *arpreq; 4044 struct in_device *in_dev; 4045 struct in_ifaddr *ifa; 4046 unsigned long flags; 4047 int i = 0; 4048 4049 if (!netif_running(netdev)) 4050 return 0; 4051 4052 for (i = 0; i < adapter->num_rx_queues; i++) 4053 napi_disable(&adapter->rx_queue[i].napi); 4054 4055 vmxnet3_disable_all_intrs(adapter); 4056 vmxnet3_free_irqs(adapter); 4057 vmxnet3_free_intr_resources(adapter); 4058 4059 netif_device_detach(netdev); 4060 4061 /* Create wake-up filters. */ 4062 pmConf = adapter->pm_conf; 4063 memset(pmConf, 0, sizeof(*pmConf)); 4064 4065 if (adapter->wol & WAKE_UCAST) { 4066 pmConf->filters[i].patternSize = ETH_ALEN; 4067 pmConf->filters[i].maskSize = 1; 4068 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN); 4069 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */ 4070 4071 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4072 i++; 4073 } 4074 4075 if (adapter->wol & WAKE_ARP) { 4076 rcu_read_lock(); 4077 4078 in_dev = __in_dev_get_rcu(netdev); 4079 if (!in_dev) { 4080 rcu_read_unlock(); 4081 goto skip_arp; 4082 } 4083 4084 ifa = rcu_dereference(in_dev->ifa_list); 4085 if (!ifa) { 4086 rcu_read_unlock(); 4087 goto skip_arp; 4088 } 4089 4090 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/ 4091 sizeof(struct arphdr) + /* ARP header */ 4092 2 * ETH_ALEN + /* 2 Ethernet addresses*/ 4093 2 * sizeof(u32); /*2 IPv4 addresses */ 4094 pmConf->filters[i].maskSize = 4095 (pmConf->filters[i].patternSize - 1) / 8 + 1; 4096 4097 /* ETH_P_ARP in Ethernet header. */ 4098 ehdr = (struct ethhdr *)pmConf->filters[i].pattern; 4099 ehdr->h_proto = htons(ETH_P_ARP); 4100 4101 /* ARPOP_REQUEST in ARP header. */ 4102 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN]; 4103 ahdr->ar_op = htons(ARPOP_REQUEST); 4104 arpreq = (u8 *)(ahdr + 1); 4105 4106 /* The Unicast IPv4 address in 'tip' field. */ 4107 arpreq += 2 * ETH_ALEN + sizeof(u32); 4108 *(__be32 *)arpreq = ifa->ifa_address; 4109 4110 rcu_read_unlock(); 4111 4112 /* The mask for the relevant bits. */ 4113 pmConf->filters[i].mask[0] = 0x00; 4114 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */ 4115 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */ 4116 pmConf->filters[i].mask[3] = 0x00; 4117 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */ 4118 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */ 4119 4120 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4121 i++; 4122 } 4123 4124 skip_arp: 4125 if (adapter->wol & WAKE_MAGIC) 4126 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC; 4127 4128 pmConf->numFilters = i; 4129 4130 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1); 4131 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof( 4132 *pmConf)); 4133 adapter->shared->devRead.pmConfDesc.confPA = 4134 cpu_to_le64(adapter->pm_conf_pa); 4135 4136 spin_lock_irqsave(&adapter->cmd_lock, flags); 4137 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4138 VMXNET3_CMD_UPDATE_PMCFG); 4139 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4140 4141 pci_save_state(pdev); 4142 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND), 4143 adapter->wol); 4144 pci_disable_device(pdev); 4145 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND)); 4146 4147 return 0; 4148 } 4149 4150 4151 static int 4152 vmxnet3_resume(struct device *device) 4153 { 4154 int err; 4155 unsigned long flags; 4156 struct pci_dev *pdev = to_pci_dev(device); 4157 struct net_device *netdev = pci_get_drvdata(pdev); 4158 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4159 4160 if (!netif_running(netdev)) 4161 return 0; 4162 4163 pci_set_power_state(pdev, PCI_D0); 4164 pci_restore_state(pdev); 4165 err = pci_enable_device_mem(pdev); 4166 if (err != 0) 4167 return err; 4168 4169 pci_enable_wake(pdev, PCI_D0, 0); 4170 4171 vmxnet3_alloc_intr_resources(adapter); 4172 4173 /* During hibernate and suspend, device has to be reinitialized as the 4174 * device state need not be preserved. 4175 */ 4176 4177 /* Need not check adapter state as other reset tasks cannot run during 4178 * device resume. 4179 */ 4180 spin_lock_irqsave(&adapter->cmd_lock, flags); 4181 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4182 VMXNET3_CMD_QUIESCE_DEV); 4183 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4184 vmxnet3_tq_cleanup_all(adapter); 4185 vmxnet3_rq_cleanup_all(adapter); 4186 4187 vmxnet3_reset_dev(adapter); 4188 err = vmxnet3_activate_dev(adapter); 4189 if (err != 0) { 4190 netdev_err(netdev, 4191 "failed to re-activate on resume, error: %d", err); 4192 vmxnet3_force_close(adapter); 4193 return err; 4194 } 4195 netif_device_attach(netdev); 4196 4197 return 0; 4198 } 4199 4200 static const struct dev_pm_ops vmxnet3_pm_ops = { 4201 .suspend = vmxnet3_suspend, 4202 .resume = vmxnet3_resume, 4203 .freeze = vmxnet3_suspend, 4204 .restore = vmxnet3_resume, 4205 }; 4206 #endif 4207 4208 static struct pci_driver vmxnet3_driver = { 4209 .name = vmxnet3_driver_name, 4210 .id_table = vmxnet3_pciid_table, 4211 .probe = vmxnet3_probe_device, 4212 .remove = vmxnet3_remove_device, 4213 .shutdown = vmxnet3_shutdown_device, 4214 #ifdef CONFIG_PM 4215 .driver.pm = &vmxnet3_pm_ops, 4216 #endif 4217 }; 4218 4219 4220 static int __init 4221 vmxnet3_init_module(void) 4222 { 4223 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC, 4224 VMXNET3_DRIVER_VERSION_REPORT); 4225 return pci_register_driver(&vmxnet3_driver); 4226 } 4227 4228 module_init(vmxnet3_init_module); 4229 4230 4231 static void 4232 vmxnet3_exit_module(void) 4233 { 4234 pci_unregister_driver(&vmxnet3_driver); 4235 } 4236 4237 module_exit(vmxnet3_exit_module); 4238 4239 MODULE_AUTHOR("VMware, Inc."); 4240 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC); 4241 MODULE_LICENSE("GPL v2"); 4242 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING); 4243