1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/aer.h> 8 9 #include "igc.h" 10 #include "igc_hw.h" 11 12 #define DRV_VERSION "0.0.1-k" 13 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 14 15 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 16 17 static int debug = -1; 18 19 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 20 MODULE_DESCRIPTION(DRV_SUMMARY); 21 MODULE_LICENSE("GPL v2"); 22 MODULE_VERSION(DRV_VERSION); 23 module_param(debug, int, 0); 24 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 25 26 char igc_driver_name[] = "igc"; 27 char igc_driver_version[] = DRV_VERSION; 28 static const char igc_driver_string[] = DRV_SUMMARY; 29 static const char igc_copyright[] = 30 "Copyright(c) 2018 Intel Corporation."; 31 32 static const struct igc_info *igc_info_tbl[] = { 33 [board_base] = &igc_base_info, 34 }; 35 36 static const struct pci_device_id igc_pci_tbl[] = { 37 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base }, 38 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base }, 39 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, 40 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, 41 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, 42 /* required last entry */ 43 {0, } 44 }; 45 46 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 47 48 /* forward declaration */ 49 static void igc_clean_tx_ring(struct igc_ring *tx_ring); 50 static int igc_sw_init(struct igc_adapter *); 51 static void igc_configure(struct igc_adapter *adapter); 52 static void igc_power_down_link(struct igc_adapter *adapter); 53 static void igc_set_default_mac_filter(struct igc_adapter *adapter); 54 static void igc_set_rx_mode(struct net_device *netdev); 55 static void igc_write_itr(struct igc_q_vector *q_vector); 56 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector); 57 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx); 58 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 59 bool msix); 60 static void igc_free_q_vectors(struct igc_adapter *adapter); 61 static void igc_irq_disable(struct igc_adapter *adapter); 62 static void igc_irq_enable(struct igc_adapter *adapter); 63 static void igc_configure_msix(struct igc_adapter *adapter); 64 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 65 struct igc_rx_buffer *bi); 66 67 enum latency_range { 68 lowest_latency = 0, 69 low_latency = 1, 70 bulk_latency = 2, 71 latency_invalid = 255 72 }; 73 74 void igc_reset(struct igc_adapter *adapter) 75 { 76 struct pci_dev *pdev = adapter->pdev; 77 struct igc_hw *hw = &adapter->hw; 78 struct igc_fc_info *fc = &hw->fc; 79 u32 pba, hwm; 80 81 /* Repartition PBA for greater than 9k MTU if required */ 82 pba = IGC_PBA_34K; 83 84 /* flow control settings 85 * The high water mark must be low enough to fit one full frame 86 * after transmitting the pause frame. As such we must have enough 87 * space to allow for us to complete our current transmit and then 88 * receive the frame that is in progress from the link partner. 89 * Set it to: 90 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 91 */ 92 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 93 94 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 95 fc->low_water = fc->high_water - 16; 96 fc->pause_time = 0xFFFF; 97 fc->send_xon = 1; 98 fc->current_mode = fc->requested_mode; 99 100 hw->mac.ops.reset_hw(hw); 101 102 if (hw->mac.ops.init_hw(hw)) 103 dev_err(&pdev->dev, "Hardware Error\n"); 104 105 if (!netif_running(adapter->netdev)) 106 igc_power_down_link(adapter); 107 108 igc_get_phy_info(hw); 109 } 110 111 /** 112 * igc_power_up_link - Power up the phy/serdes link 113 * @adapter: address of board private structure 114 */ 115 static void igc_power_up_link(struct igc_adapter *adapter) 116 { 117 igc_reset_phy(&adapter->hw); 118 119 if (adapter->hw.phy.media_type == igc_media_type_copper) 120 igc_power_up_phy_copper(&adapter->hw); 121 122 igc_setup_link(&adapter->hw); 123 } 124 125 /** 126 * igc_power_down_link - Power down the phy/serdes link 127 * @adapter: address of board private structure 128 */ 129 static void igc_power_down_link(struct igc_adapter *adapter) 130 { 131 if (adapter->hw.phy.media_type == igc_media_type_copper) 132 igc_power_down_phy_copper_base(&adapter->hw); 133 } 134 135 /** 136 * igc_release_hw_control - release control of the h/w to f/w 137 * @adapter: address of board private structure 138 * 139 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 140 * For ASF and Pass Through versions of f/w this means that the 141 * driver is no longer loaded. 142 */ 143 static void igc_release_hw_control(struct igc_adapter *adapter) 144 { 145 struct igc_hw *hw = &adapter->hw; 146 u32 ctrl_ext; 147 148 /* Let firmware take over control of h/w */ 149 ctrl_ext = rd32(IGC_CTRL_EXT); 150 wr32(IGC_CTRL_EXT, 151 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 152 } 153 154 /** 155 * igc_get_hw_control - get control of the h/w from f/w 156 * @adapter: address of board private structure 157 * 158 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 159 * For ASF and Pass Through versions of f/w this means that 160 * the driver is loaded. 161 */ 162 static void igc_get_hw_control(struct igc_adapter *adapter) 163 { 164 struct igc_hw *hw = &adapter->hw; 165 u32 ctrl_ext; 166 167 /* Let firmware know the driver has taken over */ 168 ctrl_ext = rd32(IGC_CTRL_EXT); 169 wr32(IGC_CTRL_EXT, 170 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 171 } 172 173 /** 174 * igc_free_tx_resources - Free Tx Resources per Queue 175 * @tx_ring: Tx descriptor ring for a specific queue 176 * 177 * Free all transmit software resources 178 */ 179 void igc_free_tx_resources(struct igc_ring *tx_ring) 180 { 181 igc_clean_tx_ring(tx_ring); 182 183 vfree(tx_ring->tx_buffer_info); 184 tx_ring->tx_buffer_info = NULL; 185 186 /* if not set, then don't free */ 187 if (!tx_ring->desc) 188 return; 189 190 dma_free_coherent(tx_ring->dev, tx_ring->size, 191 tx_ring->desc, tx_ring->dma); 192 193 tx_ring->desc = NULL; 194 } 195 196 /** 197 * igc_free_all_tx_resources - Free Tx Resources for All Queues 198 * @adapter: board private structure 199 * 200 * Free all transmit software resources 201 */ 202 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 203 { 204 int i; 205 206 for (i = 0; i < adapter->num_tx_queues; i++) 207 igc_free_tx_resources(adapter->tx_ring[i]); 208 } 209 210 /** 211 * igc_clean_tx_ring - Free Tx Buffers 212 * @tx_ring: ring to be cleaned 213 */ 214 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 215 { 216 u16 i = tx_ring->next_to_clean; 217 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 218 219 while (i != tx_ring->next_to_use) { 220 union igc_adv_tx_desc *eop_desc, *tx_desc; 221 222 /* Free all the Tx ring sk_buffs */ 223 dev_kfree_skb_any(tx_buffer->skb); 224 225 /* unmap skb header data */ 226 dma_unmap_single(tx_ring->dev, 227 dma_unmap_addr(tx_buffer, dma), 228 dma_unmap_len(tx_buffer, len), 229 DMA_TO_DEVICE); 230 231 /* check for eop_desc to determine the end of the packet */ 232 eop_desc = tx_buffer->next_to_watch; 233 tx_desc = IGC_TX_DESC(tx_ring, i); 234 235 /* unmap remaining buffers */ 236 while (tx_desc != eop_desc) { 237 tx_buffer++; 238 tx_desc++; 239 i++; 240 if (unlikely(i == tx_ring->count)) { 241 i = 0; 242 tx_buffer = tx_ring->tx_buffer_info; 243 tx_desc = IGC_TX_DESC(tx_ring, 0); 244 } 245 246 /* unmap any remaining paged data */ 247 if (dma_unmap_len(tx_buffer, len)) 248 dma_unmap_page(tx_ring->dev, 249 dma_unmap_addr(tx_buffer, dma), 250 dma_unmap_len(tx_buffer, len), 251 DMA_TO_DEVICE); 252 } 253 254 /* move us one more past the eop_desc for start of next pkt */ 255 tx_buffer++; 256 i++; 257 if (unlikely(i == tx_ring->count)) { 258 i = 0; 259 tx_buffer = tx_ring->tx_buffer_info; 260 } 261 } 262 263 /* reset BQL for queue */ 264 netdev_tx_reset_queue(txring_txq(tx_ring)); 265 266 /* reset next_to_use and next_to_clean */ 267 tx_ring->next_to_use = 0; 268 tx_ring->next_to_clean = 0; 269 } 270 271 /** 272 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 273 * @adapter: board private structure 274 */ 275 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 276 { 277 int i; 278 279 for (i = 0; i < adapter->num_tx_queues; i++) 280 if (adapter->tx_ring[i]) 281 igc_clean_tx_ring(adapter->tx_ring[i]); 282 } 283 284 /** 285 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 286 * @tx_ring: tx descriptor ring (for a specific queue) to setup 287 * 288 * Return 0 on success, negative on failure 289 */ 290 int igc_setup_tx_resources(struct igc_ring *tx_ring) 291 { 292 struct device *dev = tx_ring->dev; 293 int size = 0; 294 295 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 296 tx_ring->tx_buffer_info = vzalloc(size); 297 if (!tx_ring->tx_buffer_info) 298 goto err; 299 300 /* round up to nearest 4K */ 301 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 302 tx_ring->size = ALIGN(tx_ring->size, 4096); 303 304 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 305 &tx_ring->dma, GFP_KERNEL); 306 307 if (!tx_ring->desc) 308 goto err; 309 310 tx_ring->next_to_use = 0; 311 tx_ring->next_to_clean = 0; 312 313 return 0; 314 315 err: 316 vfree(tx_ring->tx_buffer_info); 317 dev_err(dev, 318 "Unable to allocate memory for the transmit descriptor ring\n"); 319 return -ENOMEM; 320 } 321 322 /** 323 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 324 * @adapter: board private structure 325 * 326 * Return 0 on success, negative on failure 327 */ 328 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 329 { 330 struct pci_dev *pdev = adapter->pdev; 331 int i, err = 0; 332 333 for (i = 0; i < adapter->num_tx_queues; i++) { 334 err = igc_setup_tx_resources(adapter->tx_ring[i]); 335 if (err) { 336 dev_err(&pdev->dev, 337 "Allocation for Tx Queue %u failed\n", i); 338 for (i--; i >= 0; i--) 339 igc_free_tx_resources(adapter->tx_ring[i]); 340 break; 341 } 342 } 343 344 return err; 345 } 346 347 /** 348 * igc_clean_rx_ring - Free Rx Buffers per Queue 349 * @rx_ring: ring to free buffers from 350 */ 351 static void igc_clean_rx_ring(struct igc_ring *rx_ring) 352 { 353 u16 i = rx_ring->next_to_clean; 354 355 if (rx_ring->skb) 356 dev_kfree_skb(rx_ring->skb); 357 rx_ring->skb = NULL; 358 359 /* Free all the Rx ring sk_buffs */ 360 while (i != rx_ring->next_to_alloc) { 361 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 362 363 /* Invalidate cache lines that may have been written to by 364 * device so that we avoid corrupting memory. 365 */ 366 dma_sync_single_range_for_cpu(rx_ring->dev, 367 buffer_info->dma, 368 buffer_info->page_offset, 369 igc_rx_bufsz(rx_ring), 370 DMA_FROM_DEVICE); 371 372 /* free resources associated with mapping */ 373 dma_unmap_page_attrs(rx_ring->dev, 374 buffer_info->dma, 375 igc_rx_pg_size(rx_ring), 376 DMA_FROM_DEVICE, 377 IGC_RX_DMA_ATTR); 378 __page_frag_cache_drain(buffer_info->page, 379 buffer_info->pagecnt_bias); 380 381 i++; 382 if (i == rx_ring->count) 383 i = 0; 384 } 385 386 rx_ring->next_to_alloc = 0; 387 rx_ring->next_to_clean = 0; 388 rx_ring->next_to_use = 0; 389 } 390 391 /** 392 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 393 * @adapter: board private structure 394 */ 395 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 396 { 397 int i; 398 399 for (i = 0; i < adapter->num_rx_queues; i++) 400 if (adapter->rx_ring[i]) 401 igc_clean_rx_ring(adapter->rx_ring[i]); 402 } 403 404 /** 405 * igc_free_rx_resources - Free Rx Resources 406 * @rx_ring: ring to clean the resources from 407 * 408 * Free all receive software resources 409 */ 410 void igc_free_rx_resources(struct igc_ring *rx_ring) 411 { 412 igc_clean_rx_ring(rx_ring); 413 414 vfree(rx_ring->rx_buffer_info); 415 rx_ring->rx_buffer_info = NULL; 416 417 /* if not set, then don't free */ 418 if (!rx_ring->desc) 419 return; 420 421 dma_free_coherent(rx_ring->dev, rx_ring->size, 422 rx_ring->desc, rx_ring->dma); 423 424 rx_ring->desc = NULL; 425 } 426 427 /** 428 * igc_free_all_rx_resources - Free Rx Resources for All Queues 429 * @adapter: board private structure 430 * 431 * Free all receive software resources 432 */ 433 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 434 { 435 int i; 436 437 for (i = 0; i < adapter->num_rx_queues; i++) 438 igc_free_rx_resources(adapter->rx_ring[i]); 439 } 440 441 /** 442 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 443 * @rx_ring: rx descriptor ring (for a specific queue) to setup 444 * 445 * Returns 0 on success, negative on failure 446 */ 447 int igc_setup_rx_resources(struct igc_ring *rx_ring) 448 { 449 struct device *dev = rx_ring->dev; 450 int size, desc_len; 451 452 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 453 rx_ring->rx_buffer_info = vzalloc(size); 454 if (!rx_ring->rx_buffer_info) 455 goto err; 456 457 desc_len = sizeof(union igc_adv_rx_desc); 458 459 /* Round up to nearest 4K */ 460 rx_ring->size = rx_ring->count * desc_len; 461 rx_ring->size = ALIGN(rx_ring->size, 4096); 462 463 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 464 &rx_ring->dma, GFP_KERNEL); 465 466 if (!rx_ring->desc) 467 goto err; 468 469 rx_ring->next_to_alloc = 0; 470 rx_ring->next_to_clean = 0; 471 rx_ring->next_to_use = 0; 472 473 return 0; 474 475 err: 476 vfree(rx_ring->rx_buffer_info); 477 rx_ring->rx_buffer_info = NULL; 478 dev_err(dev, 479 "Unable to allocate memory for the receive descriptor ring\n"); 480 return -ENOMEM; 481 } 482 483 /** 484 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 485 * (Descriptors) for all queues 486 * @adapter: board private structure 487 * 488 * Return 0 on success, negative on failure 489 */ 490 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 491 { 492 struct pci_dev *pdev = adapter->pdev; 493 int i, err = 0; 494 495 for (i = 0; i < adapter->num_rx_queues; i++) { 496 err = igc_setup_rx_resources(adapter->rx_ring[i]); 497 if (err) { 498 dev_err(&pdev->dev, 499 "Allocation for Rx Queue %u failed\n", i); 500 for (i--; i >= 0; i--) 501 igc_free_rx_resources(adapter->rx_ring[i]); 502 break; 503 } 504 } 505 506 return err; 507 } 508 509 /** 510 * igc_configure_rx_ring - Configure a receive ring after Reset 511 * @adapter: board private structure 512 * @ring: receive ring to be configured 513 * 514 * Configure the Rx unit of the MAC after a reset. 515 */ 516 static void igc_configure_rx_ring(struct igc_adapter *adapter, 517 struct igc_ring *ring) 518 { 519 struct igc_hw *hw = &adapter->hw; 520 union igc_adv_rx_desc *rx_desc; 521 int reg_idx = ring->reg_idx; 522 u32 srrctl = 0, rxdctl = 0; 523 u64 rdba = ring->dma; 524 525 /* disable the queue */ 526 wr32(IGC_RXDCTL(reg_idx), 0); 527 528 /* Set DMA base address registers */ 529 wr32(IGC_RDBAL(reg_idx), 530 rdba & 0x00000000ffffffffULL); 531 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 532 wr32(IGC_RDLEN(reg_idx), 533 ring->count * sizeof(union igc_adv_rx_desc)); 534 535 /* initialize head and tail */ 536 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 537 wr32(IGC_RDH(reg_idx), 0); 538 writel(0, ring->tail); 539 540 /* reset next-to- use/clean to place SW in sync with hardware */ 541 ring->next_to_clean = 0; 542 ring->next_to_use = 0; 543 544 /* set descriptor configuration */ 545 srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; 546 if (ring_uses_large_buffer(ring)) 547 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 548 else 549 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 550 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 551 552 wr32(IGC_SRRCTL(reg_idx), srrctl); 553 554 rxdctl |= IGC_RX_PTHRESH; 555 rxdctl |= IGC_RX_HTHRESH << 8; 556 rxdctl |= IGC_RX_WTHRESH << 16; 557 558 /* initialize rx_buffer_info */ 559 memset(ring->rx_buffer_info, 0, 560 sizeof(struct igc_rx_buffer) * ring->count); 561 562 /* initialize Rx descriptor 0 */ 563 rx_desc = IGC_RX_DESC(ring, 0); 564 rx_desc->wb.upper.length = 0; 565 566 /* enable receive descriptor fetching */ 567 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 568 569 wr32(IGC_RXDCTL(reg_idx), rxdctl); 570 } 571 572 /** 573 * igc_configure_rx - Configure receive Unit after Reset 574 * @adapter: board private structure 575 * 576 * Configure the Rx unit of the MAC after a reset. 577 */ 578 static void igc_configure_rx(struct igc_adapter *adapter) 579 { 580 int i; 581 582 /* Setup the HW Rx Head and Tail Descriptor Pointers and 583 * the Base and Length of the Rx Descriptor Ring 584 */ 585 for (i = 0; i < adapter->num_rx_queues; i++) 586 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 587 } 588 589 /** 590 * igc_configure_tx_ring - Configure transmit ring after Reset 591 * @adapter: board private structure 592 * @ring: tx ring to configure 593 * 594 * Configure a transmit ring after a reset. 595 */ 596 static void igc_configure_tx_ring(struct igc_adapter *adapter, 597 struct igc_ring *ring) 598 { 599 struct igc_hw *hw = &adapter->hw; 600 int reg_idx = ring->reg_idx; 601 u64 tdba = ring->dma; 602 u32 txdctl = 0; 603 604 /* disable the queue */ 605 wr32(IGC_TXDCTL(reg_idx), 0); 606 wrfl(); 607 mdelay(10); 608 609 wr32(IGC_TDLEN(reg_idx), 610 ring->count * sizeof(union igc_adv_tx_desc)); 611 wr32(IGC_TDBAL(reg_idx), 612 tdba & 0x00000000ffffffffULL); 613 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 614 615 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 616 wr32(IGC_TDH(reg_idx), 0); 617 writel(0, ring->tail); 618 619 txdctl |= IGC_TX_PTHRESH; 620 txdctl |= IGC_TX_HTHRESH << 8; 621 txdctl |= IGC_TX_WTHRESH << 16; 622 623 txdctl |= IGC_TXDCTL_QUEUE_ENABLE; 624 wr32(IGC_TXDCTL(reg_idx), txdctl); 625 } 626 627 /** 628 * igc_configure_tx - Configure transmit Unit after Reset 629 * @adapter: board private structure 630 * 631 * Configure the Tx unit of the MAC after a reset. 632 */ 633 static void igc_configure_tx(struct igc_adapter *adapter) 634 { 635 int i; 636 637 for (i = 0; i < adapter->num_tx_queues; i++) 638 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 639 } 640 641 /** 642 * igc_setup_mrqc - configure the multiple receive queue control registers 643 * @adapter: Board private structure 644 */ 645 static void igc_setup_mrqc(struct igc_adapter *adapter) 646 { 647 struct igc_hw *hw = &adapter->hw; 648 u32 j, num_rx_queues; 649 u32 mrqc, rxcsum; 650 u32 rss_key[10]; 651 652 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 653 for (j = 0; j < 10; j++) 654 wr32(IGC_RSSRK(j), rss_key[j]); 655 656 num_rx_queues = adapter->rss_queues; 657 658 if (adapter->rss_indir_tbl_init != num_rx_queues) { 659 for (j = 0; j < IGC_RETA_SIZE; j++) 660 adapter->rss_indir_tbl[j] = 661 (j * num_rx_queues) / IGC_RETA_SIZE; 662 adapter->rss_indir_tbl_init = num_rx_queues; 663 } 664 igc_write_rss_indir_tbl(adapter); 665 666 /* Disable raw packet checksumming so that RSS hash is placed in 667 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 668 * offloads as they are enabled by default 669 */ 670 rxcsum = rd32(IGC_RXCSUM); 671 rxcsum |= IGC_RXCSUM_PCSD; 672 673 /* Enable Receive Checksum Offload for SCTP */ 674 rxcsum |= IGC_RXCSUM_CRCOFL; 675 676 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 677 wr32(IGC_RXCSUM, rxcsum); 678 679 /* Generate RSS hash based on packet types, TCP/UDP 680 * port numbers and/or IPv4/v6 src and dst addresses 681 */ 682 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 683 IGC_MRQC_RSS_FIELD_IPV4_TCP | 684 IGC_MRQC_RSS_FIELD_IPV6 | 685 IGC_MRQC_RSS_FIELD_IPV6_TCP | 686 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 687 688 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 689 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 690 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 691 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 692 693 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 694 695 wr32(IGC_MRQC, mrqc); 696 } 697 698 /** 699 * igc_setup_rctl - configure the receive control registers 700 * @adapter: Board private structure 701 */ 702 static void igc_setup_rctl(struct igc_adapter *adapter) 703 { 704 struct igc_hw *hw = &adapter->hw; 705 u32 rctl; 706 707 rctl = rd32(IGC_RCTL); 708 709 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 710 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 711 712 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 713 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 714 715 /* enable stripping of CRC. Newer features require 716 * that the HW strips the CRC. 717 */ 718 rctl |= IGC_RCTL_SECRC; 719 720 /* disable store bad packets and clear size bits. */ 721 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 722 723 /* enable LPE to allow for reception of jumbo frames */ 724 rctl |= IGC_RCTL_LPE; 725 726 /* disable queue 0 to prevent tail write w/o re-config */ 727 wr32(IGC_RXDCTL(0), 0); 728 729 /* This is useful for sniffing bad packets. */ 730 if (adapter->netdev->features & NETIF_F_RXALL) { 731 /* UPE and MPE will be handled by normal PROMISC logic 732 * in set_rx_mode 733 */ 734 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 735 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 736 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 737 738 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 739 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 740 } 741 742 wr32(IGC_RCTL, rctl); 743 } 744 745 /** 746 * igc_setup_tctl - configure the transmit control registers 747 * @adapter: Board private structure 748 */ 749 static void igc_setup_tctl(struct igc_adapter *adapter) 750 { 751 struct igc_hw *hw = &adapter->hw; 752 u32 tctl; 753 754 /* disable queue 0 which icould be enabled by default */ 755 wr32(IGC_TXDCTL(0), 0); 756 757 /* Program the Transmit Control Register */ 758 tctl = rd32(IGC_TCTL); 759 tctl &= ~IGC_TCTL_CT; 760 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 761 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 762 763 /* Enable transmits */ 764 tctl |= IGC_TCTL_EN; 765 766 wr32(IGC_TCTL, tctl); 767 } 768 769 /** 770 * igc_set_mac - Change the Ethernet Address of the NIC 771 * @netdev: network interface device structure 772 * @p: pointer to an address structure 773 * 774 * Returns 0 on success, negative on failure 775 */ 776 static int igc_set_mac(struct net_device *netdev, void *p) 777 { 778 struct igc_adapter *adapter = netdev_priv(netdev); 779 struct igc_hw *hw = &adapter->hw; 780 struct sockaddr *addr = p; 781 782 if (!is_valid_ether_addr(addr->sa_data)) 783 return -EADDRNOTAVAIL; 784 785 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 786 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 787 788 /* set the correct pool for the new PF MAC address in entry 0 */ 789 igc_set_default_mac_filter(adapter); 790 791 return 0; 792 } 793 794 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first) 795 { 796 } 797 798 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 799 { 800 struct net_device *netdev = tx_ring->netdev; 801 802 netif_stop_subqueue(netdev, tx_ring->queue_index); 803 804 /* memory barriier comment */ 805 smp_mb(); 806 807 /* We need to check again in a case another CPU has just 808 * made room available. 809 */ 810 if (igc_desc_unused(tx_ring) < size) 811 return -EBUSY; 812 813 /* A reprieve! */ 814 netif_wake_subqueue(netdev, tx_ring->queue_index); 815 816 u64_stats_update_begin(&tx_ring->tx_syncp2); 817 tx_ring->tx_stats.restart_queue2++; 818 u64_stats_update_end(&tx_ring->tx_syncp2); 819 820 return 0; 821 } 822 823 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 824 { 825 if (igc_desc_unused(tx_ring) >= size) 826 return 0; 827 return __igc_maybe_stop_tx(tx_ring, size); 828 } 829 830 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 831 { 832 /* set type for advanced descriptor with frame checksum insertion */ 833 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 834 IGC_ADVTXD_DCMD_DEXT | 835 IGC_ADVTXD_DCMD_IFCS; 836 837 return cmd_type; 838 } 839 840 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 841 union igc_adv_tx_desc *tx_desc, 842 u32 tx_flags, unsigned int paylen) 843 { 844 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 845 846 /* insert L4 checksum */ 847 olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) * 848 ((IGC_TXD_POPTS_TXSM << 8) / 849 IGC_TX_FLAGS_CSUM); 850 851 /* insert IPv4 checksum */ 852 olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) * 853 (((IGC_TXD_POPTS_IXSM << 8)) / 854 IGC_TX_FLAGS_IPV4); 855 856 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 857 } 858 859 static int igc_tx_map(struct igc_ring *tx_ring, 860 struct igc_tx_buffer *first, 861 const u8 hdr_len) 862 { 863 struct sk_buff *skb = first->skb; 864 struct igc_tx_buffer *tx_buffer; 865 union igc_adv_tx_desc *tx_desc; 866 u32 tx_flags = first->tx_flags; 867 skb_frag_t *frag; 868 u16 i = tx_ring->next_to_use; 869 unsigned int data_len, size; 870 dma_addr_t dma; 871 u32 cmd_type = igc_tx_cmd_type(skb, tx_flags); 872 873 tx_desc = IGC_TX_DESC(tx_ring, i); 874 875 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 876 877 size = skb_headlen(skb); 878 data_len = skb->data_len; 879 880 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 881 882 tx_buffer = first; 883 884 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 885 if (dma_mapping_error(tx_ring->dev, dma)) 886 goto dma_error; 887 888 /* record length, and DMA address */ 889 dma_unmap_len_set(tx_buffer, len, size); 890 dma_unmap_addr_set(tx_buffer, dma, dma); 891 892 tx_desc->read.buffer_addr = cpu_to_le64(dma); 893 894 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 895 tx_desc->read.cmd_type_len = 896 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 897 898 i++; 899 tx_desc++; 900 if (i == tx_ring->count) { 901 tx_desc = IGC_TX_DESC(tx_ring, 0); 902 i = 0; 903 } 904 tx_desc->read.olinfo_status = 0; 905 906 dma += IGC_MAX_DATA_PER_TXD; 907 size -= IGC_MAX_DATA_PER_TXD; 908 909 tx_desc->read.buffer_addr = cpu_to_le64(dma); 910 } 911 912 if (likely(!data_len)) 913 break; 914 915 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 916 917 i++; 918 tx_desc++; 919 if (i == tx_ring->count) { 920 tx_desc = IGC_TX_DESC(tx_ring, 0); 921 i = 0; 922 } 923 tx_desc->read.olinfo_status = 0; 924 925 size = skb_frag_size(frag); 926 data_len -= size; 927 928 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 929 size, DMA_TO_DEVICE); 930 931 tx_buffer = &tx_ring->tx_buffer_info[i]; 932 } 933 934 /* write last descriptor with RS and EOP bits */ 935 cmd_type |= size | IGC_TXD_DCMD; 936 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 937 938 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 939 940 /* set the timestamp */ 941 first->time_stamp = jiffies; 942 943 skb_tx_timestamp(skb); 944 945 /* Force memory writes to complete before letting h/w know there 946 * are new descriptors to fetch. (Only applicable for weak-ordered 947 * memory model archs, such as IA-64). 948 * 949 * We also need this memory barrier to make certain all of the 950 * status bits have been updated before next_to_watch is written. 951 */ 952 wmb(); 953 954 /* set next_to_watch value indicating a packet is present */ 955 first->next_to_watch = tx_desc; 956 957 i++; 958 if (i == tx_ring->count) 959 i = 0; 960 961 tx_ring->next_to_use = i; 962 963 /* Make sure there is space in the ring for the next send. */ 964 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 965 966 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 967 writel(i, tx_ring->tail); 968 } 969 970 return 0; 971 dma_error: 972 dev_err(tx_ring->dev, "TX DMA map failed\n"); 973 tx_buffer = &tx_ring->tx_buffer_info[i]; 974 975 /* clear dma mappings for failed tx_buffer_info map */ 976 while (tx_buffer != first) { 977 if (dma_unmap_len(tx_buffer, len)) 978 dma_unmap_page(tx_ring->dev, 979 dma_unmap_addr(tx_buffer, dma), 980 dma_unmap_len(tx_buffer, len), 981 DMA_TO_DEVICE); 982 dma_unmap_len_set(tx_buffer, len, 0); 983 984 if (i-- == 0) 985 i += tx_ring->count; 986 tx_buffer = &tx_ring->tx_buffer_info[i]; 987 } 988 989 if (dma_unmap_len(tx_buffer, len)) 990 dma_unmap_single(tx_ring->dev, 991 dma_unmap_addr(tx_buffer, dma), 992 dma_unmap_len(tx_buffer, len), 993 DMA_TO_DEVICE); 994 dma_unmap_len_set(tx_buffer, len, 0); 995 996 dev_kfree_skb_any(tx_buffer->skb); 997 tx_buffer->skb = NULL; 998 999 tx_ring->next_to_use = i; 1000 1001 return -1; 1002 } 1003 1004 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1005 struct igc_ring *tx_ring) 1006 { 1007 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1008 __be16 protocol = vlan_get_protocol(skb); 1009 struct igc_tx_buffer *first; 1010 u32 tx_flags = 0; 1011 unsigned short f; 1012 u8 hdr_len = 0; 1013 1014 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1015 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1016 * + 2 desc gap to keep tail from touching head, 1017 * + 1 desc for context descriptor, 1018 * otherwise try next time 1019 */ 1020 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1021 count += TXD_USE_COUNT(skb_frag_size( 1022 &skb_shinfo(skb)->frags[f])); 1023 1024 if (igc_maybe_stop_tx(tx_ring, count + 3)) { 1025 /* this is a hard error */ 1026 return NETDEV_TX_BUSY; 1027 } 1028 1029 /* record the location of the first descriptor for this packet */ 1030 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1031 first->skb = skb; 1032 first->bytecount = skb->len; 1033 first->gso_segs = 1; 1034 1035 /* record initial flags and protocol */ 1036 first->tx_flags = tx_flags; 1037 first->protocol = protocol; 1038 1039 igc_tx_csum(tx_ring, first); 1040 1041 igc_tx_map(tx_ring, first, hdr_len); 1042 1043 return NETDEV_TX_OK; 1044 } 1045 1046 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1047 struct sk_buff *skb) 1048 { 1049 unsigned int r_idx = skb->queue_mapping; 1050 1051 if (r_idx >= adapter->num_tx_queues) 1052 r_idx = r_idx % adapter->num_tx_queues; 1053 1054 return adapter->tx_ring[r_idx]; 1055 } 1056 1057 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1058 struct net_device *netdev) 1059 { 1060 struct igc_adapter *adapter = netdev_priv(netdev); 1061 1062 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1063 * in order to meet this minimum size requirement. 1064 */ 1065 if (skb->len < 17) { 1066 if (skb_padto(skb, 17)) 1067 return NETDEV_TX_OK; 1068 skb->len = 17; 1069 } 1070 1071 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1072 } 1073 1074 static inline void igc_rx_hash(struct igc_ring *ring, 1075 union igc_adv_rx_desc *rx_desc, 1076 struct sk_buff *skb) 1077 { 1078 if (ring->netdev->features & NETIF_F_RXHASH) 1079 skb_set_hash(skb, 1080 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 1081 PKT_HASH_TYPE_L3); 1082 } 1083 1084 /** 1085 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1086 * @rx_ring: rx descriptor ring packet is being transacted on 1087 * @rx_desc: pointer to the EOP Rx descriptor 1088 * @skb: pointer to current skb being populated 1089 * 1090 * This function checks the ring, descriptor, and packet information in 1091 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 1092 * other fields within the skb. 1093 */ 1094 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1095 union igc_adv_rx_desc *rx_desc, 1096 struct sk_buff *skb) 1097 { 1098 igc_rx_hash(rx_ring, rx_desc, skb); 1099 1100 skb_record_rx_queue(skb, rx_ring->queue_index); 1101 1102 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1103 } 1104 1105 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1106 const unsigned int size) 1107 { 1108 struct igc_rx_buffer *rx_buffer; 1109 1110 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1111 prefetchw(rx_buffer->page); 1112 1113 /* we are reusing so sync this buffer for CPU use */ 1114 dma_sync_single_range_for_cpu(rx_ring->dev, 1115 rx_buffer->dma, 1116 rx_buffer->page_offset, 1117 size, 1118 DMA_FROM_DEVICE); 1119 1120 rx_buffer->pagecnt_bias--; 1121 1122 return rx_buffer; 1123 } 1124 1125 /** 1126 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1127 * @rx_ring: rx descriptor ring to transact packets on 1128 * @rx_buffer: buffer containing page to add 1129 * @skb: sk_buff to place the data into 1130 * @size: size of buffer to be added 1131 * 1132 * This function will add the data contained in rx_buffer->page to the skb. 1133 */ 1134 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1135 struct igc_rx_buffer *rx_buffer, 1136 struct sk_buff *skb, 1137 unsigned int size) 1138 { 1139 #if (PAGE_SIZE < 8192) 1140 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1141 1142 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1143 rx_buffer->page_offset, size, truesize); 1144 rx_buffer->page_offset ^= truesize; 1145 #else 1146 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1147 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1148 SKB_DATA_ALIGN(size); 1149 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1150 rx_buffer->page_offset, size, truesize); 1151 rx_buffer->page_offset += truesize; 1152 #endif 1153 } 1154 1155 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1156 struct igc_rx_buffer *rx_buffer, 1157 union igc_adv_rx_desc *rx_desc, 1158 unsigned int size) 1159 { 1160 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1161 #if (PAGE_SIZE < 8192) 1162 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1163 #else 1164 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1165 SKB_DATA_ALIGN(IGC_SKB_PAD + size); 1166 #endif 1167 struct sk_buff *skb; 1168 1169 /* prefetch first cache line of first page */ 1170 prefetch(va); 1171 #if L1_CACHE_BYTES < 128 1172 prefetch(va + L1_CACHE_BYTES); 1173 #endif 1174 1175 /* build an skb around the page buffer */ 1176 skb = build_skb(va - IGC_SKB_PAD, truesize); 1177 if (unlikely(!skb)) 1178 return NULL; 1179 1180 /* update pointers within the skb to store the data */ 1181 skb_reserve(skb, IGC_SKB_PAD); 1182 __skb_put(skb, size); 1183 1184 /* update buffer offset */ 1185 #if (PAGE_SIZE < 8192) 1186 rx_buffer->page_offset ^= truesize; 1187 #else 1188 rx_buffer->page_offset += truesize; 1189 #endif 1190 1191 return skb; 1192 } 1193 1194 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1195 struct igc_rx_buffer *rx_buffer, 1196 union igc_adv_rx_desc *rx_desc, 1197 unsigned int size) 1198 { 1199 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1200 #if (PAGE_SIZE < 8192) 1201 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1202 #else 1203 unsigned int truesize = SKB_DATA_ALIGN(size); 1204 #endif 1205 unsigned int headlen; 1206 struct sk_buff *skb; 1207 1208 /* prefetch first cache line of first page */ 1209 prefetch(va); 1210 #if L1_CACHE_BYTES < 128 1211 prefetch(va + L1_CACHE_BYTES); 1212 #endif 1213 1214 /* allocate a skb to store the frags */ 1215 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); 1216 if (unlikely(!skb)) 1217 return NULL; 1218 1219 /* Determine available headroom for copy */ 1220 headlen = size; 1221 if (headlen > IGC_RX_HDR_LEN) 1222 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 1223 1224 /* align pull length to size of long to optimize memcpy performance */ 1225 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 1226 1227 /* update all of the pointers */ 1228 size -= headlen; 1229 if (size) { 1230 skb_add_rx_frag(skb, 0, rx_buffer->page, 1231 (va + headlen) - page_address(rx_buffer->page), 1232 size, truesize); 1233 #if (PAGE_SIZE < 8192) 1234 rx_buffer->page_offset ^= truesize; 1235 #else 1236 rx_buffer->page_offset += truesize; 1237 #endif 1238 } else { 1239 rx_buffer->pagecnt_bias++; 1240 } 1241 1242 return skb; 1243 } 1244 1245 /** 1246 * igc_reuse_rx_page - page flip buffer and store it back on the ring 1247 * @rx_ring: rx descriptor ring to store buffers on 1248 * @old_buff: donor buffer to have page reused 1249 * 1250 * Synchronizes page for reuse by the adapter 1251 */ 1252 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 1253 struct igc_rx_buffer *old_buff) 1254 { 1255 u16 nta = rx_ring->next_to_alloc; 1256 struct igc_rx_buffer *new_buff; 1257 1258 new_buff = &rx_ring->rx_buffer_info[nta]; 1259 1260 /* update, and store next to alloc */ 1261 nta++; 1262 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1263 1264 /* Transfer page from old buffer to new buffer. 1265 * Move each member individually to avoid possible store 1266 * forwarding stalls. 1267 */ 1268 new_buff->dma = old_buff->dma; 1269 new_buff->page = old_buff->page; 1270 new_buff->page_offset = old_buff->page_offset; 1271 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1272 } 1273 1274 static inline bool igc_page_is_reserved(struct page *page) 1275 { 1276 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 1277 } 1278 1279 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) 1280 { 1281 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1282 struct page *page = rx_buffer->page; 1283 1284 /* avoid re-using remote pages */ 1285 if (unlikely(igc_page_is_reserved(page))) 1286 return false; 1287 1288 #if (PAGE_SIZE < 8192) 1289 /* if we are only owner of page we can reuse it */ 1290 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 1291 return false; 1292 #else 1293 #define IGC_LAST_OFFSET \ 1294 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 1295 1296 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 1297 return false; 1298 #endif 1299 1300 /* If we have drained the page fragment pool we need to update 1301 * the pagecnt_bias and page count so that we fully restock the 1302 * number of references the driver holds. 1303 */ 1304 if (unlikely(!pagecnt_bias)) { 1305 page_ref_add(page, USHRT_MAX); 1306 rx_buffer->pagecnt_bias = USHRT_MAX; 1307 } 1308 1309 return true; 1310 } 1311 1312 /** 1313 * igc_is_non_eop - process handling of non-EOP buffers 1314 * @rx_ring: Rx ring being processed 1315 * @rx_desc: Rx descriptor for current buffer 1316 * @skb: current socket buffer containing buffer in progress 1317 * 1318 * This function updates next to clean. If the buffer is an EOP buffer 1319 * this function exits returning false, otherwise it will place the 1320 * sk_buff in the next buffer to be chained and return true indicating 1321 * that this is in fact a non-EOP buffer. 1322 */ 1323 static bool igc_is_non_eop(struct igc_ring *rx_ring, 1324 union igc_adv_rx_desc *rx_desc) 1325 { 1326 u32 ntc = rx_ring->next_to_clean + 1; 1327 1328 /* fetch, update, and store next to clean */ 1329 ntc = (ntc < rx_ring->count) ? ntc : 0; 1330 rx_ring->next_to_clean = ntc; 1331 1332 prefetch(IGC_RX_DESC(rx_ring, ntc)); 1333 1334 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 1335 return false; 1336 1337 return true; 1338 } 1339 1340 /** 1341 * igc_cleanup_headers - Correct corrupted or empty headers 1342 * @rx_ring: rx descriptor ring packet is being transacted on 1343 * @rx_desc: pointer to the EOP Rx descriptor 1344 * @skb: pointer to current skb being fixed 1345 * 1346 * Address the case where we are pulling data in on pages only 1347 * and as such no data is present in the skb header. 1348 * 1349 * In addition if skb is not at least 60 bytes we need to pad it so that 1350 * it is large enough to qualify as a valid Ethernet frame. 1351 * 1352 * Returns true if an error was encountered and skb was freed. 1353 */ 1354 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 1355 union igc_adv_rx_desc *rx_desc, 1356 struct sk_buff *skb) 1357 { 1358 if (unlikely((igc_test_staterr(rx_desc, 1359 IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) { 1360 struct net_device *netdev = rx_ring->netdev; 1361 1362 if (!(netdev->features & NETIF_F_RXALL)) { 1363 dev_kfree_skb_any(skb); 1364 return true; 1365 } 1366 } 1367 1368 /* if eth_skb_pad returns an error the skb was freed */ 1369 if (eth_skb_pad(skb)) 1370 return true; 1371 1372 return false; 1373 } 1374 1375 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 1376 struct igc_rx_buffer *rx_buffer) 1377 { 1378 if (igc_can_reuse_rx_page(rx_buffer)) { 1379 /* hand second half of page back to the ring */ 1380 igc_reuse_rx_page(rx_ring, rx_buffer); 1381 } else { 1382 /* We are not reusing the buffer so unmap it and free 1383 * any references we are holding to it 1384 */ 1385 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1386 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 1387 IGC_RX_DMA_ATTR); 1388 __page_frag_cache_drain(rx_buffer->page, 1389 rx_buffer->pagecnt_bias); 1390 } 1391 1392 /* clear contents of rx_buffer */ 1393 rx_buffer->page = NULL; 1394 } 1395 1396 /** 1397 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 1398 * @adapter: address of board private structure 1399 */ 1400 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 1401 { 1402 union igc_adv_rx_desc *rx_desc; 1403 u16 i = rx_ring->next_to_use; 1404 struct igc_rx_buffer *bi; 1405 u16 bufsz; 1406 1407 /* nothing to do */ 1408 if (!cleaned_count) 1409 return; 1410 1411 rx_desc = IGC_RX_DESC(rx_ring, i); 1412 bi = &rx_ring->rx_buffer_info[i]; 1413 i -= rx_ring->count; 1414 1415 bufsz = igc_rx_bufsz(rx_ring); 1416 1417 do { 1418 if (!igc_alloc_mapped_page(rx_ring, bi)) 1419 break; 1420 1421 /* sync the buffer for use by the device */ 1422 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1423 bi->page_offset, bufsz, 1424 DMA_FROM_DEVICE); 1425 1426 /* Refresh the desc even if buffer_addrs didn't change 1427 * because each write-back erases this info. 1428 */ 1429 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1430 1431 rx_desc++; 1432 bi++; 1433 i++; 1434 if (unlikely(!i)) { 1435 rx_desc = IGC_RX_DESC(rx_ring, 0); 1436 bi = rx_ring->rx_buffer_info; 1437 i -= rx_ring->count; 1438 } 1439 1440 /* clear the length for the next_to_use descriptor */ 1441 rx_desc->wb.upper.length = 0; 1442 1443 cleaned_count--; 1444 } while (cleaned_count); 1445 1446 i += rx_ring->count; 1447 1448 if (rx_ring->next_to_use != i) { 1449 /* record the next descriptor to use */ 1450 rx_ring->next_to_use = i; 1451 1452 /* update next to alloc since we have filled the ring */ 1453 rx_ring->next_to_alloc = i; 1454 1455 /* Force memory writes to complete before letting h/w 1456 * know there are new descriptors to fetch. (Only 1457 * applicable for weak-ordered memory model archs, 1458 * such as IA-64). 1459 */ 1460 wmb(); 1461 writel(i, rx_ring->tail); 1462 } 1463 } 1464 1465 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 1466 { 1467 unsigned int total_bytes = 0, total_packets = 0; 1468 struct igc_ring *rx_ring = q_vector->rx.ring; 1469 struct sk_buff *skb = rx_ring->skb; 1470 u16 cleaned_count = igc_desc_unused(rx_ring); 1471 1472 while (likely(total_packets < budget)) { 1473 union igc_adv_rx_desc *rx_desc; 1474 struct igc_rx_buffer *rx_buffer; 1475 unsigned int size; 1476 1477 /* return some buffers to hardware, one at a time is too slow */ 1478 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 1479 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1480 cleaned_count = 0; 1481 } 1482 1483 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 1484 size = le16_to_cpu(rx_desc->wb.upper.length); 1485 if (!size) 1486 break; 1487 1488 /* This memory barrier is needed to keep us from reading 1489 * any other fields out of the rx_desc until we know the 1490 * descriptor has been written back 1491 */ 1492 dma_rmb(); 1493 1494 rx_buffer = igc_get_rx_buffer(rx_ring, size); 1495 1496 /* retrieve a buffer from the ring */ 1497 if (skb) 1498 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 1499 else if (ring_uses_build_skb(rx_ring)) 1500 skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); 1501 else 1502 skb = igc_construct_skb(rx_ring, rx_buffer, 1503 rx_desc, size); 1504 1505 /* exit if we failed to retrieve a buffer */ 1506 if (!skb) { 1507 rx_ring->rx_stats.alloc_failed++; 1508 rx_buffer->pagecnt_bias++; 1509 break; 1510 } 1511 1512 igc_put_rx_buffer(rx_ring, rx_buffer); 1513 cleaned_count++; 1514 1515 /* fetch next buffer in frame if non-eop */ 1516 if (igc_is_non_eop(rx_ring, rx_desc)) 1517 continue; 1518 1519 /* verify the packet layout is correct */ 1520 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) { 1521 skb = NULL; 1522 continue; 1523 } 1524 1525 /* probably a little skewed due to removing CRC */ 1526 total_bytes += skb->len; 1527 1528 /* populate checksum, timestamp, VLAN, and protocol */ 1529 igc_process_skb_fields(rx_ring, rx_desc, skb); 1530 1531 napi_gro_receive(&q_vector->napi, skb); 1532 1533 /* reset skb pointer */ 1534 skb = NULL; 1535 1536 /* update budget accounting */ 1537 total_packets++; 1538 } 1539 1540 /* place incomplete frames back on ring for completion */ 1541 rx_ring->skb = skb; 1542 1543 u64_stats_update_begin(&rx_ring->rx_syncp); 1544 rx_ring->rx_stats.packets += total_packets; 1545 rx_ring->rx_stats.bytes += total_bytes; 1546 u64_stats_update_end(&rx_ring->rx_syncp); 1547 q_vector->rx.total_packets += total_packets; 1548 q_vector->rx.total_bytes += total_bytes; 1549 1550 if (cleaned_count) 1551 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1552 1553 return total_packets; 1554 } 1555 1556 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 1557 { 1558 return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; 1559 } 1560 1561 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 1562 struct igc_rx_buffer *bi) 1563 { 1564 struct page *page = bi->page; 1565 dma_addr_t dma; 1566 1567 /* since we are recycling buffers we should seldom need to alloc */ 1568 if (likely(page)) 1569 return true; 1570 1571 /* alloc new page for storage */ 1572 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 1573 if (unlikely(!page)) { 1574 rx_ring->rx_stats.alloc_failed++; 1575 return false; 1576 } 1577 1578 /* map page for use */ 1579 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1580 igc_rx_pg_size(rx_ring), 1581 DMA_FROM_DEVICE, 1582 IGC_RX_DMA_ATTR); 1583 1584 /* if mapping failed free memory back to system since 1585 * there isn't much point in holding memory we can't use 1586 */ 1587 if (dma_mapping_error(rx_ring->dev, dma)) { 1588 __free_page(page); 1589 1590 rx_ring->rx_stats.alloc_failed++; 1591 return false; 1592 } 1593 1594 bi->dma = dma; 1595 bi->page = page; 1596 bi->page_offset = igc_rx_offset(rx_ring); 1597 bi->pagecnt_bias = 1; 1598 1599 return true; 1600 } 1601 1602 /** 1603 * igc_clean_tx_irq - Reclaim resources after transmit completes 1604 * @q_vector: pointer to q_vector containing needed info 1605 * @napi_budget: Used to determine if we are in netpoll 1606 * 1607 * returns true if ring is completely cleaned 1608 */ 1609 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 1610 { 1611 struct igc_adapter *adapter = q_vector->adapter; 1612 unsigned int total_bytes = 0, total_packets = 0; 1613 unsigned int budget = q_vector->tx.work_limit; 1614 struct igc_ring *tx_ring = q_vector->tx.ring; 1615 unsigned int i = tx_ring->next_to_clean; 1616 struct igc_tx_buffer *tx_buffer; 1617 union igc_adv_tx_desc *tx_desc; 1618 1619 if (test_bit(__IGC_DOWN, &adapter->state)) 1620 return true; 1621 1622 tx_buffer = &tx_ring->tx_buffer_info[i]; 1623 tx_desc = IGC_TX_DESC(tx_ring, i); 1624 i -= tx_ring->count; 1625 1626 do { 1627 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 1628 1629 /* if next_to_watch is not set then there is no work pending */ 1630 if (!eop_desc) 1631 break; 1632 1633 /* prevent any other reads prior to eop_desc */ 1634 smp_rmb(); 1635 1636 /* if DD is not set pending work has not been completed */ 1637 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 1638 break; 1639 1640 /* clear next_to_watch to prevent false hangs */ 1641 tx_buffer->next_to_watch = NULL; 1642 1643 /* update the statistics for this packet */ 1644 total_bytes += tx_buffer->bytecount; 1645 total_packets += tx_buffer->gso_segs; 1646 1647 /* free the skb */ 1648 napi_consume_skb(tx_buffer->skb, napi_budget); 1649 1650 /* unmap skb header data */ 1651 dma_unmap_single(tx_ring->dev, 1652 dma_unmap_addr(tx_buffer, dma), 1653 dma_unmap_len(tx_buffer, len), 1654 DMA_TO_DEVICE); 1655 1656 /* clear tx_buffer data */ 1657 dma_unmap_len_set(tx_buffer, len, 0); 1658 1659 /* clear last DMA location and unmap remaining buffers */ 1660 while (tx_desc != eop_desc) { 1661 tx_buffer++; 1662 tx_desc++; 1663 i++; 1664 if (unlikely(!i)) { 1665 i -= tx_ring->count; 1666 tx_buffer = tx_ring->tx_buffer_info; 1667 tx_desc = IGC_TX_DESC(tx_ring, 0); 1668 } 1669 1670 /* unmap any remaining paged data */ 1671 if (dma_unmap_len(tx_buffer, len)) { 1672 dma_unmap_page(tx_ring->dev, 1673 dma_unmap_addr(tx_buffer, dma), 1674 dma_unmap_len(tx_buffer, len), 1675 DMA_TO_DEVICE); 1676 dma_unmap_len_set(tx_buffer, len, 0); 1677 } 1678 } 1679 1680 /* move us one more past the eop_desc for start of next pkt */ 1681 tx_buffer++; 1682 tx_desc++; 1683 i++; 1684 if (unlikely(!i)) { 1685 i -= tx_ring->count; 1686 tx_buffer = tx_ring->tx_buffer_info; 1687 tx_desc = IGC_TX_DESC(tx_ring, 0); 1688 } 1689 1690 /* issue prefetch for next Tx descriptor */ 1691 prefetch(tx_desc); 1692 1693 /* update budget accounting */ 1694 budget--; 1695 } while (likely(budget)); 1696 1697 netdev_tx_completed_queue(txring_txq(tx_ring), 1698 total_packets, total_bytes); 1699 1700 i += tx_ring->count; 1701 tx_ring->next_to_clean = i; 1702 u64_stats_update_begin(&tx_ring->tx_syncp); 1703 tx_ring->tx_stats.bytes += total_bytes; 1704 tx_ring->tx_stats.packets += total_packets; 1705 u64_stats_update_end(&tx_ring->tx_syncp); 1706 q_vector->tx.total_bytes += total_bytes; 1707 q_vector->tx.total_packets += total_packets; 1708 1709 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 1710 struct igc_hw *hw = &adapter->hw; 1711 1712 /* Detect a transmit hang in hardware, this serializes the 1713 * check with the clearing of time_stamp and movement of i 1714 */ 1715 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 1716 if (tx_buffer->next_to_watch && 1717 time_after(jiffies, tx_buffer->time_stamp + 1718 (adapter->tx_timeout_factor * HZ)) && 1719 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { 1720 /* detected Tx unit hang */ 1721 dev_err(tx_ring->dev, 1722 "Detected Tx Unit Hang\n" 1723 " Tx Queue <%d>\n" 1724 " TDH <%x>\n" 1725 " TDT <%x>\n" 1726 " next_to_use <%x>\n" 1727 " next_to_clean <%x>\n" 1728 "buffer_info[next_to_clean]\n" 1729 " time_stamp <%lx>\n" 1730 " next_to_watch <%p>\n" 1731 " jiffies <%lx>\n" 1732 " desc.status <%x>\n", 1733 tx_ring->queue_index, 1734 rd32(IGC_TDH(tx_ring->reg_idx)), 1735 readl(tx_ring->tail), 1736 tx_ring->next_to_use, 1737 tx_ring->next_to_clean, 1738 tx_buffer->time_stamp, 1739 tx_buffer->next_to_watch, 1740 jiffies, 1741 tx_buffer->next_to_watch->wb.status); 1742 netif_stop_subqueue(tx_ring->netdev, 1743 tx_ring->queue_index); 1744 1745 /* we are about to reset, no point in enabling stuff */ 1746 return true; 1747 } 1748 } 1749 1750 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 1751 if (unlikely(total_packets && 1752 netif_carrier_ok(tx_ring->netdev) && 1753 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 1754 /* Make sure that anybody stopping the queue after this 1755 * sees the new next_to_clean. 1756 */ 1757 smp_mb(); 1758 if (__netif_subqueue_stopped(tx_ring->netdev, 1759 tx_ring->queue_index) && 1760 !(test_bit(__IGC_DOWN, &adapter->state))) { 1761 netif_wake_subqueue(tx_ring->netdev, 1762 tx_ring->queue_index); 1763 1764 u64_stats_update_begin(&tx_ring->tx_syncp); 1765 tx_ring->tx_stats.restart_queue++; 1766 u64_stats_update_end(&tx_ring->tx_syncp); 1767 } 1768 } 1769 1770 return !!budget; 1771 } 1772 1773 /** 1774 * igc_up - Open the interface and prepare it to handle traffic 1775 * @adapter: board private structure 1776 */ 1777 void igc_up(struct igc_adapter *adapter) 1778 { 1779 struct igc_hw *hw = &adapter->hw; 1780 int i = 0; 1781 1782 /* hardware has been reset, we need to reload some things */ 1783 igc_configure(adapter); 1784 1785 clear_bit(__IGC_DOWN, &adapter->state); 1786 1787 for (i = 0; i < adapter->num_q_vectors; i++) 1788 napi_enable(&adapter->q_vector[i]->napi); 1789 1790 if (adapter->msix_entries) 1791 igc_configure_msix(adapter); 1792 else 1793 igc_assign_vector(adapter->q_vector[0], 0); 1794 1795 /* Clear any pending interrupts. */ 1796 rd32(IGC_ICR); 1797 igc_irq_enable(adapter); 1798 1799 netif_tx_start_all_queues(adapter->netdev); 1800 1801 /* start the watchdog. */ 1802 hw->mac.get_link_status = 1; 1803 schedule_work(&adapter->watchdog_task); 1804 } 1805 1806 /** 1807 * igc_update_stats - Update the board statistics counters 1808 * @adapter: board private structure 1809 */ 1810 void igc_update_stats(struct igc_adapter *adapter) 1811 { 1812 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 1813 struct pci_dev *pdev = adapter->pdev; 1814 struct igc_hw *hw = &adapter->hw; 1815 u64 _bytes, _packets; 1816 u64 bytes, packets; 1817 unsigned int start; 1818 u32 mpc; 1819 int i; 1820 1821 /* Prevent stats update while adapter is being reset, or if the pci 1822 * connection is down. 1823 */ 1824 if (adapter->link_speed == 0) 1825 return; 1826 if (pci_channel_offline(pdev)) 1827 return; 1828 1829 packets = 0; 1830 bytes = 0; 1831 1832 rcu_read_lock(); 1833 for (i = 0; i < adapter->num_rx_queues; i++) { 1834 struct igc_ring *ring = adapter->rx_ring[i]; 1835 u32 rqdpc = rd32(IGC_RQDPC(i)); 1836 1837 if (hw->mac.type >= igc_i225) 1838 wr32(IGC_RQDPC(i), 0); 1839 1840 if (rqdpc) { 1841 ring->rx_stats.drops += rqdpc; 1842 net_stats->rx_fifo_errors += rqdpc; 1843 } 1844 1845 do { 1846 start = u64_stats_fetch_begin_irq(&ring->rx_syncp); 1847 _bytes = ring->rx_stats.bytes; 1848 _packets = ring->rx_stats.packets; 1849 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); 1850 bytes += _bytes; 1851 packets += _packets; 1852 } 1853 1854 net_stats->rx_bytes = bytes; 1855 net_stats->rx_packets = packets; 1856 1857 packets = 0; 1858 bytes = 0; 1859 for (i = 0; i < adapter->num_tx_queues; i++) { 1860 struct igc_ring *ring = adapter->tx_ring[i]; 1861 1862 do { 1863 start = u64_stats_fetch_begin_irq(&ring->tx_syncp); 1864 _bytes = ring->tx_stats.bytes; 1865 _packets = ring->tx_stats.packets; 1866 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); 1867 bytes += _bytes; 1868 packets += _packets; 1869 } 1870 net_stats->tx_bytes = bytes; 1871 net_stats->tx_packets = packets; 1872 rcu_read_unlock(); 1873 1874 /* read stats registers */ 1875 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 1876 adapter->stats.gprc += rd32(IGC_GPRC); 1877 adapter->stats.gorc += rd32(IGC_GORCL); 1878 rd32(IGC_GORCH); /* clear GORCL */ 1879 adapter->stats.bprc += rd32(IGC_BPRC); 1880 adapter->stats.mprc += rd32(IGC_MPRC); 1881 adapter->stats.roc += rd32(IGC_ROC); 1882 1883 adapter->stats.prc64 += rd32(IGC_PRC64); 1884 adapter->stats.prc127 += rd32(IGC_PRC127); 1885 adapter->stats.prc255 += rd32(IGC_PRC255); 1886 adapter->stats.prc511 += rd32(IGC_PRC511); 1887 adapter->stats.prc1023 += rd32(IGC_PRC1023); 1888 adapter->stats.prc1522 += rd32(IGC_PRC1522); 1889 adapter->stats.symerrs += rd32(IGC_SYMERRS); 1890 adapter->stats.sec += rd32(IGC_SEC); 1891 1892 mpc = rd32(IGC_MPC); 1893 adapter->stats.mpc += mpc; 1894 net_stats->rx_fifo_errors += mpc; 1895 adapter->stats.scc += rd32(IGC_SCC); 1896 adapter->stats.ecol += rd32(IGC_ECOL); 1897 adapter->stats.mcc += rd32(IGC_MCC); 1898 adapter->stats.latecol += rd32(IGC_LATECOL); 1899 adapter->stats.dc += rd32(IGC_DC); 1900 adapter->stats.rlec += rd32(IGC_RLEC); 1901 adapter->stats.xonrxc += rd32(IGC_XONRXC); 1902 adapter->stats.xontxc += rd32(IGC_XONTXC); 1903 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 1904 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 1905 adapter->stats.fcruc += rd32(IGC_FCRUC); 1906 adapter->stats.gptc += rd32(IGC_GPTC); 1907 adapter->stats.gotc += rd32(IGC_GOTCL); 1908 rd32(IGC_GOTCH); /* clear GOTCL */ 1909 adapter->stats.rnbc += rd32(IGC_RNBC); 1910 adapter->stats.ruc += rd32(IGC_RUC); 1911 adapter->stats.rfc += rd32(IGC_RFC); 1912 adapter->stats.rjc += rd32(IGC_RJC); 1913 adapter->stats.tor += rd32(IGC_TORH); 1914 adapter->stats.tot += rd32(IGC_TOTH); 1915 adapter->stats.tpr += rd32(IGC_TPR); 1916 1917 adapter->stats.ptc64 += rd32(IGC_PTC64); 1918 adapter->stats.ptc127 += rd32(IGC_PTC127); 1919 adapter->stats.ptc255 += rd32(IGC_PTC255); 1920 adapter->stats.ptc511 += rd32(IGC_PTC511); 1921 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 1922 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 1923 1924 adapter->stats.mptc += rd32(IGC_MPTC); 1925 adapter->stats.bptc += rd32(IGC_BPTC); 1926 1927 adapter->stats.tpt += rd32(IGC_TPT); 1928 adapter->stats.colc += rd32(IGC_COLC); 1929 1930 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 1931 1932 adapter->stats.tsctc += rd32(IGC_TSCTC); 1933 adapter->stats.tsctfc += rd32(IGC_TSCTFC); 1934 1935 adapter->stats.iac += rd32(IGC_IAC); 1936 adapter->stats.icrxoc += rd32(IGC_ICRXOC); 1937 adapter->stats.icrxptc += rd32(IGC_ICRXPTC); 1938 adapter->stats.icrxatc += rd32(IGC_ICRXATC); 1939 adapter->stats.ictxptc += rd32(IGC_ICTXPTC); 1940 adapter->stats.ictxatc += rd32(IGC_ICTXATC); 1941 adapter->stats.ictxqec += rd32(IGC_ICTXQEC); 1942 adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC); 1943 adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC); 1944 1945 /* Fill out the OS statistics structure */ 1946 net_stats->multicast = adapter->stats.mprc; 1947 net_stats->collisions = adapter->stats.colc; 1948 1949 /* Rx Errors */ 1950 1951 /* RLEC on some newer hardware can be incorrect so build 1952 * our own version based on RUC and ROC 1953 */ 1954 net_stats->rx_errors = adapter->stats.rxerrc + 1955 adapter->stats.crcerrs + adapter->stats.algnerrc + 1956 adapter->stats.ruc + adapter->stats.roc + 1957 adapter->stats.cexterr; 1958 net_stats->rx_length_errors = adapter->stats.ruc + 1959 adapter->stats.roc; 1960 net_stats->rx_crc_errors = adapter->stats.crcerrs; 1961 net_stats->rx_frame_errors = adapter->stats.algnerrc; 1962 net_stats->rx_missed_errors = adapter->stats.mpc; 1963 1964 /* Tx Errors */ 1965 net_stats->tx_errors = adapter->stats.ecol + 1966 adapter->stats.latecol; 1967 net_stats->tx_aborted_errors = adapter->stats.ecol; 1968 net_stats->tx_window_errors = adapter->stats.latecol; 1969 net_stats->tx_carrier_errors = adapter->stats.tncrs; 1970 1971 /* Tx Dropped needs to be maintained elsewhere */ 1972 1973 /* Management Stats */ 1974 adapter->stats.mgptc += rd32(IGC_MGTPTC); 1975 adapter->stats.mgprc += rd32(IGC_MGTPRC); 1976 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 1977 } 1978 1979 static void igc_nfc_filter_exit(struct igc_adapter *adapter) 1980 { 1981 struct igc_nfc_filter *rule; 1982 1983 spin_lock(&adapter->nfc_lock); 1984 1985 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) 1986 igc_erase_filter(adapter, rule); 1987 1988 hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node) 1989 igc_erase_filter(adapter, rule); 1990 1991 spin_unlock(&adapter->nfc_lock); 1992 } 1993 1994 static void igc_nfc_filter_restore(struct igc_adapter *adapter) 1995 { 1996 struct igc_nfc_filter *rule; 1997 1998 spin_lock(&adapter->nfc_lock); 1999 2000 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) 2001 igc_add_filter(adapter, rule); 2002 2003 spin_unlock(&adapter->nfc_lock); 2004 } 2005 2006 /** 2007 * igc_down - Close the interface 2008 * @adapter: board private structure 2009 */ 2010 void igc_down(struct igc_adapter *adapter) 2011 { 2012 struct net_device *netdev = adapter->netdev; 2013 struct igc_hw *hw = &adapter->hw; 2014 u32 tctl, rctl; 2015 int i = 0; 2016 2017 set_bit(__IGC_DOWN, &adapter->state); 2018 2019 /* disable receives in the hardware */ 2020 rctl = rd32(IGC_RCTL); 2021 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 2022 /* flush and sleep below */ 2023 2024 igc_nfc_filter_exit(adapter); 2025 2026 /* set trans_start so we don't get spurious watchdogs during reset */ 2027 netif_trans_update(netdev); 2028 2029 netif_carrier_off(netdev); 2030 netif_tx_stop_all_queues(netdev); 2031 2032 /* disable transmits in the hardware */ 2033 tctl = rd32(IGC_TCTL); 2034 tctl &= ~IGC_TCTL_EN; 2035 wr32(IGC_TCTL, tctl); 2036 /* flush both disables and wait for them to finish */ 2037 wrfl(); 2038 usleep_range(10000, 20000); 2039 2040 igc_irq_disable(adapter); 2041 2042 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 2043 2044 for (i = 0; i < adapter->num_q_vectors; i++) { 2045 if (adapter->q_vector[i]) { 2046 napi_synchronize(&adapter->q_vector[i]->napi); 2047 napi_disable(&adapter->q_vector[i]->napi); 2048 } 2049 } 2050 2051 del_timer_sync(&adapter->watchdog_timer); 2052 del_timer_sync(&adapter->phy_info_timer); 2053 2054 /* record the stats before reset*/ 2055 spin_lock(&adapter->stats64_lock); 2056 igc_update_stats(adapter); 2057 spin_unlock(&adapter->stats64_lock); 2058 2059 adapter->link_speed = 0; 2060 adapter->link_duplex = 0; 2061 2062 if (!pci_channel_offline(adapter->pdev)) 2063 igc_reset(adapter); 2064 2065 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 2066 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 2067 2068 igc_clean_all_tx_rings(adapter); 2069 igc_clean_all_rx_rings(adapter); 2070 } 2071 2072 void igc_reinit_locked(struct igc_adapter *adapter) 2073 { 2074 WARN_ON(in_interrupt()); 2075 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 2076 usleep_range(1000, 2000); 2077 igc_down(adapter); 2078 igc_up(adapter); 2079 clear_bit(__IGC_RESETTING, &adapter->state); 2080 } 2081 2082 static void igc_reset_task(struct work_struct *work) 2083 { 2084 struct igc_adapter *adapter; 2085 2086 adapter = container_of(work, struct igc_adapter, reset_task); 2087 2088 netdev_err(adapter->netdev, "Reset adapter\n"); 2089 igc_reinit_locked(adapter); 2090 } 2091 2092 /** 2093 * igc_change_mtu - Change the Maximum Transfer Unit 2094 * @netdev: network interface device structure 2095 * @new_mtu: new value for maximum frame size 2096 * 2097 * Returns 0 on success, negative on failure 2098 */ 2099 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 2100 { 2101 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 2102 struct igc_adapter *adapter = netdev_priv(netdev); 2103 struct pci_dev *pdev = adapter->pdev; 2104 2105 /* adjust max frame to be at least the size of a standard frame */ 2106 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 2107 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 2108 2109 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 2110 usleep_range(1000, 2000); 2111 2112 /* igc_down has a dependency on max_frame_size */ 2113 adapter->max_frame_size = max_frame; 2114 2115 if (netif_running(netdev)) 2116 igc_down(adapter); 2117 2118 dev_info(&pdev->dev, "changing MTU from %d to %d\n", 2119 netdev->mtu, new_mtu); 2120 netdev->mtu = new_mtu; 2121 2122 if (netif_running(netdev)) 2123 igc_up(adapter); 2124 else 2125 igc_reset(adapter); 2126 2127 clear_bit(__IGC_RESETTING, &adapter->state); 2128 2129 return 0; 2130 } 2131 2132 /** 2133 * igc_get_stats - Get System Network Statistics 2134 * @netdev: network interface device structure 2135 * 2136 * Returns the address of the device statistics structure. 2137 * The statistics are updated here and also from the timer callback. 2138 */ 2139 static struct net_device_stats *igc_get_stats(struct net_device *netdev) 2140 { 2141 struct igc_adapter *adapter = netdev_priv(netdev); 2142 2143 if (!test_bit(__IGC_RESETTING, &adapter->state)) 2144 igc_update_stats(adapter); 2145 2146 /* only return the current stats */ 2147 return &netdev->stats; 2148 } 2149 2150 static netdev_features_t igc_fix_features(struct net_device *netdev, 2151 netdev_features_t features) 2152 { 2153 /* Since there is no support for separate Rx/Tx vlan accel 2154 * enable/disable make sure Tx flag is always in same state as Rx. 2155 */ 2156 if (features & NETIF_F_HW_VLAN_CTAG_RX) 2157 features |= NETIF_F_HW_VLAN_CTAG_TX; 2158 else 2159 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 2160 2161 return features; 2162 } 2163 2164 static int igc_set_features(struct net_device *netdev, 2165 netdev_features_t features) 2166 { 2167 netdev_features_t changed = netdev->features ^ features; 2168 struct igc_adapter *adapter = netdev_priv(netdev); 2169 2170 /* Add VLAN support */ 2171 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 2172 return 0; 2173 2174 if (!(features & NETIF_F_NTUPLE)) { 2175 struct hlist_node *node2; 2176 struct igc_nfc_filter *rule; 2177 2178 spin_lock(&adapter->nfc_lock); 2179 hlist_for_each_entry_safe(rule, node2, 2180 &adapter->nfc_filter_list, nfc_node) { 2181 igc_erase_filter(adapter, rule); 2182 hlist_del(&rule->nfc_node); 2183 kfree(rule); 2184 } 2185 spin_unlock(&adapter->nfc_lock); 2186 adapter->nfc_filter_count = 0; 2187 } 2188 2189 netdev->features = features; 2190 2191 if (netif_running(netdev)) 2192 igc_reinit_locked(adapter); 2193 else 2194 igc_reset(adapter); 2195 2196 return 1; 2197 } 2198 2199 static netdev_features_t 2200 igc_features_check(struct sk_buff *skb, struct net_device *dev, 2201 netdev_features_t features) 2202 { 2203 unsigned int network_hdr_len, mac_hdr_len; 2204 2205 /* Make certain the headers can be described by a context descriptor */ 2206 mac_hdr_len = skb_network_header(skb) - skb->data; 2207 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 2208 return features & ~(NETIF_F_HW_CSUM | 2209 NETIF_F_SCTP_CRC | 2210 NETIF_F_HW_VLAN_CTAG_TX | 2211 NETIF_F_TSO | 2212 NETIF_F_TSO6); 2213 2214 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 2215 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 2216 return features & ~(NETIF_F_HW_CSUM | 2217 NETIF_F_SCTP_CRC | 2218 NETIF_F_TSO | 2219 NETIF_F_TSO6); 2220 2221 /* We can only support IPv4 TSO in tunnels if we can mangle the 2222 * inner IP ID field, so strip TSO if MANGLEID is not supported. 2223 */ 2224 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 2225 features &= ~NETIF_F_TSO; 2226 2227 return features; 2228 } 2229 2230 /** 2231 * igc_configure - configure the hardware for RX and TX 2232 * @adapter: private board structure 2233 */ 2234 static void igc_configure(struct igc_adapter *adapter) 2235 { 2236 struct net_device *netdev = adapter->netdev; 2237 int i = 0; 2238 2239 igc_get_hw_control(adapter); 2240 igc_set_rx_mode(netdev); 2241 2242 igc_setup_tctl(adapter); 2243 igc_setup_mrqc(adapter); 2244 igc_setup_rctl(adapter); 2245 2246 igc_nfc_filter_restore(adapter); 2247 igc_configure_tx(adapter); 2248 igc_configure_rx(adapter); 2249 2250 igc_rx_fifo_flush_base(&adapter->hw); 2251 2252 /* call igc_desc_unused which always leaves 2253 * at least 1 descriptor unused to make sure 2254 * next_to_use != next_to_clean 2255 */ 2256 for (i = 0; i < adapter->num_rx_queues; i++) { 2257 struct igc_ring *ring = adapter->rx_ring[i]; 2258 2259 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 2260 } 2261 } 2262 2263 /** 2264 * igc_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table 2265 * @adapter: address of board private structure 2266 * @index: Index of the RAR entry which need to be synced with MAC table 2267 */ 2268 static void igc_rar_set_index(struct igc_adapter *adapter, u32 index) 2269 { 2270 u8 *addr = adapter->mac_table[index].addr; 2271 struct igc_hw *hw = &adapter->hw; 2272 u32 rar_low, rar_high; 2273 2274 /* HW expects these to be in network order when they are plugged 2275 * into the registers which are little endian. In order to guarantee 2276 * that ordering we need to do an leXX_to_cpup here in order to be 2277 * ready for the byteswap that occurs with writel 2278 */ 2279 rar_low = le32_to_cpup((__le32 *)(addr)); 2280 rar_high = le16_to_cpup((__le16 *)(addr + 4)); 2281 2282 /* Indicate to hardware the Address is Valid. */ 2283 if (adapter->mac_table[index].state & IGC_MAC_STATE_IN_USE) { 2284 if (is_valid_ether_addr(addr)) 2285 rar_high |= IGC_RAH_AV; 2286 2287 rar_high |= IGC_RAH_POOL_1 << 2288 adapter->mac_table[index].queue; 2289 } 2290 2291 wr32(IGC_RAL(index), rar_low); 2292 wrfl(); 2293 wr32(IGC_RAH(index), rar_high); 2294 wrfl(); 2295 } 2296 2297 /* Set default MAC address for the PF in the first RAR entry */ 2298 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 2299 { 2300 struct igc_mac_addr *mac_table = &adapter->mac_table[0]; 2301 2302 ether_addr_copy(mac_table->addr, adapter->hw.mac.addr); 2303 mac_table->state = IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; 2304 2305 igc_rar_set_index(adapter, 0); 2306 } 2307 2308 /* If the filter to be added and an already existing filter express 2309 * the same address and address type, it should be possible to only 2310 * override the other configurations, for example the queue to steer 2311 * traffic. 2312 */ 2313 static bool igc_mac_entry_can_be_used(const struct igc_mac_addr *entry, 2314 const u8 *addr, const u8 flags) 2315 { 2316 if (!(entry->state & IGC_MAC_STATE_IN_USE)) 2317 return true; 2318 2319 if ((entry->state & IGC_MAC_STATE_SRC_ADDR) != 2320 (flags & IGC_MAC_STATE_SRC_ADDR)) 2321 return false; 2322 2323 if (!ether_addr_equal(addr, entry->addr)) 2324 return false; 2325 2326 return true; 2327 } 2328 2329 /* Add a MAC filter for 'addr' directing matching traffic to 'queue', 2330 * 'flags' is used to indicate what kind of match is made, match is by 2331 * default for the destination address, if matching by source address 2332 * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. 2333 */ 2334 static int igc_add_mac_filter_flags(struct igc_adapter *adapter, 2335 const u8 *addr, const u8 queue, 2336 const u8 flags) 2337 { 2338 struct igc_hw *hw = &adapter->hw; 2339 int rar_entries = hw->mac.rar_entry_count; 2340 int i; 2341 2342 if (is_zero_ether_addr(addr)) 2343 return -EINVAL; 2344 2345 /* Search for the first empty entry in the MAC table. 2346 * Do not touch entries at the end of the table reserved for the VF MAC 2347 * addresses. 2348 */ 2349 for (i = 0; i < rar_entries; i++) { 2350 if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], 2351 addr, flags)) 2352 continue; 2353 2354 ether_addr_copy(adapter->mac_table[i].addr, addr); 2355 adapter->mac_table[i].queue = queue; 2356 adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE | flags; 2357 2358 igc_rar_set_index(adapter, i); 2359 return i; 2360 } 2361 2362 return -ENOSPC; 2363 } 2364 2365 int igc_add_mac_steering_filter(struct igc_adapter *adapter, 2366 const u8 *addr, u8 queue, u8 flags) 2367 { 2368 return igc_add_mac_filter_flags(adapter, addr, queue, 2369 IGC_MAC_STATE_QUEUE_STEERING | flags); 2370 } 2371 2372 /* Remove a MAC filter for 'addr' directing matching traffic to 2373 * 'queue', 'flags' is used to indicate what kind of match need to be 2374 * removed, match is by default for the destination address, if 2375 * matching by source address is to be removed the flag 2376 * IGC_MAC_STATE_SRC_ADDR can be used. 2377 */ 2378 static int igc_del_mac_filter_flags(struct igc_adapter *adapter, 2379 const u8 *addr, const u8 queue, 2380 const u8 flags) 2381 { 2382 struct igc_hw *hw = &adapter->hw; 2383 int rar_entries = hw->mac.rar_entry_count; 2384 int i; 2385 2386 if (is_zero_ether_addr(addr)) 2387 return -EINVAL; 2388 2389 /* Search for matching entry in the MAC table based on given address 2390 * and queue. Do not touch entries at the end of the table reserved 2391 * for the VF MAC addresses. 2392 */ 2393 for (i = 0; i < rar_entries; i++) { 2394 if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) 2395 continue; 2396 if ((adapter->mac_table[i].state & flags) != flags) 2397 continue; 2398 if (adapter->mac_table[i].queue != queue) 2399 continue; 2400 if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) 2401 continue; 2402 2403 /* When a filter for the default address is "deleted", 2404 * we return it to its initial configuration 2405 */ 2406 if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { 2407 adapter->mac_table[i].state = 2408 IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; 2409 } else { 2410 adapter->mac_table[i].state = 0; 2411 adapter->mac_table[i].queue = 0; 2412 memset(adapter->mac_table[i].addr, 0, ETH_ALEN); 2413 } 2414 2415 igc_rar_set_index(adapter, i); 2416 return 0; 2417 } 2418 2419 return -ENOENT; 2420 } 2421 2422 int igc_del_mac_steering_filter(struct igc_adapter *adapter, 2423 const u8 *addr, u8 queue, u8 flags) 2424 { 2425 return igc_del_mac_filter_flags(adapter, addr, queue, 2426 IGC_MAC_STATE_QUEUE_STEERING | flags); 2427 } 2428 2429 /** 2430 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 2431 * @netdev: network interface device structure 2432 * 2433 * The set_rx_mode entry point is called whenever the unicast or multicast 2434 * address lists or the network interface flags are updated. This routine is 2435 * responsible for configuring the hardware for proper unicast, multicast, 2436 * promiscuous mode, and all-multi behavior. 2437 */ 2438 static void igc_set_rx_mode(struct net_device *netdev) 2439 { 2440 } 2441 2442 /** 2443 * igc_msix_other - msix other interrupt handler 2444 * @irq: interrupt number 2445 * @data: pointer to a q_vector 2446 */ 2447 static irqreturn_t igc_msix_other(int irq, void *data) 2448 { 2449 struct igc_adapter *adapter = data; 2450 struct igc_hw *hw = &adapter->hw; 2451 u32 icr = rd32(IGC_ICR); 2452 2453 /* reading ICR causes bit 31 of EICR to be cleared */ 2454 if (icr & IGC_ICR_DRSTA) 2455 schedule_work(&adapter->reset_task); 2456 2457 if (icr & IGC_ICR_DOUTSYNC) { 2458 /* HW is reporting DMA is out of sync */ 2459 adapter->stats.doosync++; 2460 } 2461 2462 if (icr & IGC_ICR_LSC) { 2463 hw->mac.get_link_status = 1; 2464 /* guard against interrupt when we're going down */ 2465 if (!test_bit(__IGC_DOWN, &adapter->state)) 2466 mod_timer(&adapter->watchdog_timer, jiffies + 1); 2467 } 2468 2469 wr32(IGC_EIMS, adapter->eims_other); 2470 2471 return IRQ_HANDLED; 2472 } 2473 2474 /** 2475 * igc_write_ivar - configure ivar for given MSI-X vector 2476 * @hw: pointer to the HW structure 2477 * @msix_vector: vector number we are allocating to a given ring 2478 * @index: row index of IVAR register to write within IVAR table 2479 * @offset: column offset of in IVAR, should be multiple of 8 2480 * 2481 * The IVAR table consists of 2 columns, 2482 * each containing an cause allocation for an Rx and Tx ring, and a 2483 * variable number of rows depending on the number of queues supported. 2484 */ 2485 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 2486 int index, int offset) 2487 { 2488 u32 ivar = array_rd32(IGC_IVAR0, index); 2489 2490 /* clear any bits that are currently set */ 2491 ivar &= ~((u32)0xFF << offset); 2492 2493 /* write vector and valid bit */ 2494 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 2495 2496 array_wr32(IGC_IVAR0, index, ivar); 2497 } 2498 2499 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 2500 { 2501 struct igc_adapter *adapter = q_vector->adapter; 2502 struct igc_hw *hw = &adapter->hw; 2503 int rx_queue = IGC_N0_QUEUE; 2504 int tx_queue = IGC_N0_QUEUE; 2505 2506 if (q_vector->rx.ring) 2507 rx_queue = q_vector->rx.ring->reg_idx; 2508 if (q_vector->tx.ring) 2509 tx_queue = q_vector->tx.ring->reg_idx; 2510 2511 switch (hw->mac.type) { 2512 case igc_i225: 2513 if (rx_queue > IGC_N0_QUEUE) 2514 igc_write_ivar(hw, msix_vector, 2515 rx_queue >> 1, 2516 (rx_queue & 0x1) << 4); 2517 if (tx_queue > IGC_N0_QUEUE) 2518 igc_write_ivar(hw, msix_vector, 2519 tx_queue >> 1, 2520 ((tx_queue & 0x1) << 4) + 8); 2521 q_vector->eims_value = BIT(msix_vector); 2522 break; 2523 default: 2524 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 2525 break; 2526 } 2527 2528 /* add q_vector eims value to global eims_enable_mask */ 2529 adapter->eims_enable_mask |= q_vector->eims_value; 2530 2531 /* configure q_vector to set itr on first interrupt */ 2532 q_vector->set_itr = 1; 2533 } 2534 2535 /** 2536 * igc_configure_msix - Configure MSI-X hardware 2537 * @adapter: Pointer to adapter structure 2538 * 2539 * igc_configure_msix sets up the hardware to properly 2540 * generate MSI-X interrupts. 2541 */ 2542 static void igc_configure_msix(struct igc_adapter *adapter) 2543 { 2544 struct igc_hw *hw = &adapter->hw; 2545 int i, vector = 0; 2546 u32 tmp; 2547 2548 adapter->eims_enable_mask = 0; 2549 2550 /* set vector for other causes, i.e. link changes */ 2551 switch (hw->mac.type) { 2552 case igc_i225: 2553 /* Turn on MSI-X capability first, or our settings 2554 * won't stick. And it will take days to debug. 2555 */ 2556 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 2557 IGC_GPIE_PBA | IGC_GPIE_EIAME | 2558 IGC_GPIE_NSICR); 2559 2560 /* enable msix_other interrupt */ 2561 adapter->eims_other = BIT(vector); 2562 tmp = (vector++ | IGC_IVAR_VALID) << 8; 2563 2564 wr32(IGC_IVAR_MISC, tmp); 2565 break; 2566 default: 2567 /* do nothing, since nothing else supports MSI-X */ 2568 break; 2569 } /* switch (hw->mac.type) */ 2570 2571 adapter->eims_enable_mask |= adapter->eims_other; 2572 2573 for (i = 0; i < adapter->num_q_vectors; i++) 2574 igc_assign_vector(adapter->q_vector[i], vector++); 2575 2576 wrfl(); 2577 } 2578 2579 static irqreturn_t igc_msix_ring(int irq, void *data) 2580 { 2581 struct igc_q_vector *q_vector = data; 2582 2583 /* Write the ITR value calculated from the previous interrupt. */ 2584 igc_write_itr(q_vector); 2585 2586 napi_schedule(&q_vector->napi); 2587 2588 return IRQ_HANDLED; 2589 } 2590 2591 /** 2592 * igc_request_msix - Initialize MSI-X interrupts 2593 * @adapter: Pointer to adapter structure 2594 * 2595 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 2596 * kernel. 2597 */ 2598 static int igc_request_msix(struct igc_adapter *adapter) 2599 { 2600 int i = 0, err = 0, vector = 0, free_vector = 0; 2601 struct net_device *netdev = adapter->netdev; 2602 2603 err = request_irq(adapter->msix_entries[vector].vector, 2604 &igc_msix_other, 0, netdev->name, adapter); 2605 if (err) 2606 goto err_out; 2607 2608 for (i = 0; i < adapter->num_q_vectors; i++) { 2609 struct igc_q_vector *q_vector = adapter->q_vector[i]; 2610 2611 vector++; 2612 2613 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 2614 2615 if (q_vector->rx.ring && q_vector->tx.ring) 2616 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 2617 q_vector->rx.ring->queue_index); 2618 else if (q_vector->tx.ring) 2619 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 2620 q_vector->tx.ring->queue_index); 2621 else if (q_vector->rx.ring) 2622 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 2623 q_vector->rx.ring->queue_index); 2624 else 2625 sprintf(q_vector->name, "%s-unused", netdev->name); 2626 2627 err = request_irq(adapter->msix_entries[vector].vector, 2628 igc_msix_ring, 0, q_vector->name, 2629 q_vector); 2630 if (err) 2631 goto err_free; 2632 } 2633 2634 igc_configure_msix(adapter); 2635 return 0; 2636 2637 err_free: 2638 /* free already assigned IRQs */ 2639 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 2640 2641 vector--; 2642 for (i = 0; i < vector; i++) { 2643 free_irq(adapter->msix_entries[free_vector++].vector, 2644 adapter->q_vector[i]); 2645 } 2646 err_out: 2647 return err; 2648 } 2649 2650 /** 2651 * igc_reset_q_vector - Reset config for interrupt vector 2652 * @adapter: board private structure to initialize 2653 * @v_idx: Index of vector to be reset 2654 * 2655 * If NAPI is enabled it will delete any references to the 2656 * NAPI struct. This is preparation for igc_free_q_vector. 2657 */ 2658 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 2659 { 2660 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2661 2662 /* if we're coming from igc_set_interrupt_capability, the vectors are 2663 * not yet allocated 2664 */ 2665 if (!q_vector) 2666 return; 2667 2668 if (q_vector->tx.ring) 2669 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 2670 2671 if (q_vector->rx.ring) 2672 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 2673 2674 netif_napi_del(&q_vector->napi); 2675 } 2676 2677 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 2678 { 2679 int v_idx = adapter->num_q_vectors; 2680 2681 if (adapter->msix_entries) { 2682 pci_disable_msix(adapter->pdev); 2683 kfree(adapter->msix_entries); 2684 adapter->msix_entries = NULL; 2685 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 2686 pci_disable_msi(adapter->pdev); 2687 } 2688 2689 while (v_idx--) 2690 igc_reset_q_vector(adapter, v_idx); 2691 } 2692 2693 /** 2694 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 2695 * @adapter: Pointer to adapter structure 2696 * 2697 * This function resets the device so that it has 0 rx queues, tx queues, and 2698 * MSI-X interrupts allocated. 2699 */ 2700 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 2701 { 2702 igc_free_q_vectors(adapter); 2703 igc_reset_interrupt_capability(adapter); 2704 } 2705 2706 /** 2707 * igc_free_q_vectors - Free memory allocated for interrupt vectors 2708 * @adapter: board private structure to initialize 2709 * 2710 * This function frees the memory allocated to the q_vectors. In addition if 2711 * NAPI is enabled it will delete any references to the NAPI struct prior 2712 * to freeing the q_vector. 2713 */ 2714 static void igc_free_q_vectors(struct igc_adapter *adapter) 2715 { 2716 int v_idx = adapter->num_q_vectors; 2717 2718 adapter->num_tx_queues = 0; 2719 adapter->num_rx_queues = 0; 2720 adapter->num_q_vectors = 0; 2721 2722 while (v_idx--) { 2723 igc_reset_q_vector(adapter, v_idx); 2724 igc_free_q_vector(adapter, v_idx); 2725 } 2726 } 2727 2728 /** 2729 * igc_free_q_vector - Free memory allocated for specific interrupt vector 2730 * @adapter: board private structure to initialize 2731 * @v_idx: Index of vector to be freed 2732 * 2733 * This function frees the memory allocated to the q_vector. 2734 */ 2735 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 2736 { 2737 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2738 2739 adapter->q_vector[v_idx] = NULL; 2740 2741 /* igc_get_stats64() might access the rings on this vector, 2742 * we must wait a grace period before freeing it. 2743 */ 2744 if (q_vector) 2745 kfree_rcu(q_vector, rcu); 2746 } 2747 2748 /* Need to wait a few seconds after link up to get diagnostic information from 2749 * the phy 2750 */ 2751 static void igc_update_phy_info(struct timer_list *t) 2752 { 2753 struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); 2754 2755 igc_get_phy_info(&adapter->hw); 2756 } 2757 2758 /** 2759 * igc_has_link - check shared code for link and determine up/down 2760 * @adapter: pointer to driver private info 2761 */ 2762 bool igc_has_link(struct igc_adapter *adapter) 2763 { 2764 struct igc_hw *hw = &adapter->hw; 2765 bool link_active = false; 2766 2767 /* get_link_status is set on LSC (link status) interrupt or 2768 * rx sequence error interrupt. get_link_status will stay 2769 * false until the igc_check_for_link establishes link 2770 * for copper adapters ONLY 2771 */ 2772 switch (hw->phy.media_type) { 2773 case igc_media_type_copper: 2774 if (!hw->mac.get_link_status) 2775 return true; 2776 hw->mac.ops.check_for_link(hw); 2777 link_active = !hw->mac.get_link_status; 2778 break; 2779 default: 2780 case igc_media_type_unknown: 2781 break; 2782 } 2783 2784 if (hw->mac.type == igc_i225 && 2785 hw->phy.id == I225_I_PHY_ID) { 2786 if (!netif_carrier_ok(adapter->netdev)) { 2787 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 2788 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 2789 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 2790 adapter->link_check_timeout = jiffies; 2791 } 2792 } 2793 2794 return link_active; 2795 } 2796 2797 /** 2798 * igc_watchdog - Timer Call-back 2799 * @data: pointer to adapter cast into an unsigned long 2800 */ 2801 static void igc_watchdog(struct timer_list *t) 2802 { 2803 struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); 2804 /* Do the rest outside of interrupt context */ 2805 schedule_work(&adapter->watchdog_task); 2806 } 2807 2808 static void igc_watchdog_task(struct work_struct *work) 2809 { 2810 struct igc_adapter *adapter = container_of(work, 2811 struct igc_adapter, 2812 watchdog_task); 2813 struct net_device *netdev = adapter->netdev; 2814 struct igc_hw *hw = &adapter->hw; 2815 struct igc_phy_info *phy = &hw->phy; 2816 u16 phy_data, retry_count = 20; 2817 u32 connsw; 2818 u32 link; 2819 int i; 2820 2821 link = igc_has_link(adapter); 2822 2823 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 2824 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 2825 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 2826 else 2827 link = false; 2828 } 2829 2830 /* Force link down if we have fiber to swap to */ 2831 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 2832 if (hw->phy.media_type == igc_media_type_copper) { 2833 connsw = rd32(IGC_CONNSW); 2834 if (!(connsw & IGC_CONNSW_AUTOSENSE_EN)) 2835 link = 0; 2836 } 2837 } 2838 if (link) { 2839 if (!netif_carrier_ok(netdev)) { 2840 u32 ctrl; 2841 2842 hw->mac.ops.get_speed_and_duplex(hw, 2843 &adapter->link_speed, 2844 &adapter->link_duplex); 2845 2846 ctrl = rd32(IGC_CTRL); 2847 /* Link status message must follow this format */ 2848 netdev_info(netdev, 2849 "igc: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 2850 netdev->name, 2851 adapter->link_speed, 2852 adapter->link_duplex == FULL_DUPLEX ? 2853 "Full" : "Half", 2854 (ctrl & IGC_CTRL_TFCE) && 2855 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 2856 (ctrl & IGC_CTRL_RFCE) ? "RX" : 2857 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 2858 2859 /* check if SmartSpeed worked */ 2860 igc_check_downshift(hw); 2861 if (phy->speed_downgraded) 2862 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 2863 2864 /* adjust timeout factor according to speed/duplex */ 2865 adapter->tx_timeout_factor = 1; 2866 switch (adapter->link_speed) { 2867 case SPEED_10: 2868 adapter->tx_timeout_factor = 14; 2869 break; 2870 case SPEED_100: 2871 /* maybe add some timeout factor ? */ 2872 break; 2873 } 2874 2875 if (adapter->link_speed != SPEED_1000) 2876 goto no_wait; 2877 2878 /* wait for Remote receiver status OK */ 2879 retry_read_status: 2880 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 2881 &phy_data)) { 2882 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 2883 retry_count) { 2884 msleep(100); 2885 retry_count--; 2886 goto retry_read_status; 2887 } else if (!retry_count) { 2888 dev_err(&adapter->pdev->dev, "exceed max 2 second\n"); 2889 } 2890 } else { 2891 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n"); 2892 } 2893 no_wait: 2894 netif_carrier_on(netdev); 2895 2896 /* link state has changed, schedule phy info update */ 2897 if (!test_bit(__IGC_DOWN, &adapter->state)) 2898 mod_timer(&adapter->phy_info_timer, 2899 round_jiffies(jiffies + 2 * HZ)); 2900 } 2901 } else { 2902 if (netif_carrier_ok(netdev)) { 2903 adapter->link_speed = 0; 2904 adapter->link_duplex = 0; 2905 2906 /* Links status message must follow this format */ 2907 netdev_info(netdev, "igc: %s NIC Link is Down\n", 2908 netdev->name); 2909 netif_carrier_off(netdev); 2910 2911 /* link state has changed, schedule phy info update */ 2912 if (!test_bit(__IGC_DOWN, &adapter->state)) 2913 mod_timer(&adapter->phy_info_timer, 2914 round_jiffies(jiffies + 2 * HZ)); 2915 2916 /* link is down, time to check for alternate media */ 2917 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 2918 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 2919 schedule_work(&adapter->reset_task); 2920 /* return immediately */ 2921 return; 2922 } 2923 } 2924 2925 /* also check for alternate media here */ 2926 } else if (!netif_carrier_ok(netdev) && 2927 (adapter->flags & IGC_FLAG_MAS_ENABLE)) { 2928 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 2929 schedule_work(&adapter->reset_task); 2930 /* return immediately */ 2931 return; 2932 } 2933 } 2934 } 2935 2936 spin_lock(&adapter->stats64_lock); 2937 igc_update_stats(adapter); 2938 spin_unlock(&adapter->stats64_lock); 2939 2940 for (i = 0; i < adapter->num_tx_queues; i++) { 2941 struct igc_ring *tx_ring = adapter->tx_ring[i]; 2942 2943 if (!netif_carrier_ok(netdev)) { 2944 /* We've lost link, so the controller stops DMA, 2945 * but we've got queued Tx work that's never going 2946 * to get done, so reset controller to flush Tx. 2947 * (Do the reset outside of interrupt context). 2948 */ 2949 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 2950 adapter->tx_timeout_count++; 2951 schedule_work(&adapter->reset_task); 2952 /* return immediately since reset is imminent */ 2953 return; 2954 } 2955 } 2956 2957 /* Force detection of hung controller every watchdog period */ 2958 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 2959 } 2960 2961 /* Cause software interrupt to ensure Rx ring is cleaned */ 2962 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 2963 u32 eics = 0; 2964 2965 for (i = 0; i < adapter->num_q_vectors; i++) 2966 eics |= adapter->q_vector[i]->eims_value; 2967 wr32(IGC_EICS, eics); 2968 } else { 2969 wr32(IGC_ICS, IGC_ICS_RXDMT0); 2970 } 2971 2972 /* Reset the timer */ 2973 if (!test_bit(__IGC_DOWN, &adapter->state)) { 2974 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 2975 mod_timer(&adapter->watchdog_timer, 2976 round_jiffies(jiffies + HZ)); 2977 else 2978 mod_timer(&adapter->watchdog_timer, 2979 round_jiffies(jiffies + 2 * HZ)); 2980 } 2981 } 2982 2983 /** 2984 * igc_update_ring_itr - update the dynamic ITR value based on packet size 2985 * @q_vector: pointer to q_vector 2986 * 2987 * Stores a new ITR value based on strictly on packet size. This 2988 * algorithm is less sophisticated than that used in igc_update_itr, 2989 * due to the difficulty of synchronizing statistics across multiple 2990 * receive rings. The divisors and thresholds used by this function 2991 * were determined based on theoretical maximum wire speed and testing 2992 * data, in order to minimize response time while increasing bulk 2993 * throughput. 2994 * NOTE: This function is called only when operating in a multiqueue 2995 * receive environment. 2996 */ 2997 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 2998 { 2999 struct igc_adapter *adapter = q_vector->adapter; 3000 int new_val = q_vector->itr_val; 3001 int avg_wire_size = 0; 3002 unsigned int packets; 3003 3004 /* For non-gigabit speeds, just fix the interrupt rate at 4000 3005 * ints/sec - ITR timer value of 120 ticks. 3006 */ 3007 switch (adapter->link_speed) { 3008 case SPEED_10: 3009 case SPEED_100: 3010 new_val = IGC_4K_ITR; 3011 goto set_itr_val; 3012 default: 3013 break; 3014 } 3015 3016 packets = q_vector->rx.total_packets; 3017 if (packets) 3018 avg_wire_size = q_vector->rx.total_bytes / packets; 3019 3020 packets = q_vector->tx.total_packets; 3021 if (packets) 3022 avg_wire_size = max_t(u32, avg_wire_size, 3023 q_vector->tx.total_bytes / packets); 3024 3025 /* if avg_wire_size isn't set no work was done */ 3026 if (!avg_wire_size) 3027 goto clear_counts; 3028 3029 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 3030 avg_wire_size += 24; 3031 3032 /* Don't starve jumbo frames */ 3033 avg_wire_size = min(avg_wire_size, 3000); 3034 3035 /* Give a little boost to mid-size frames */ 3036 if (avg_wire_size > 300 && avg_wire_size < 1200) 3037 new_val = avg_wire_size / 3; 3038 else 3039 new_val = avg_wire_size / 2; 3040 3041 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3042 if (new_val < IGC_20K_ITR && 3043 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3044 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3045 new_val = IGC_20K_ITR; 3046 3047 set_itr_val: 3048 if (new_val != q_vector->itr_val) { 3049 q_vector->itr_val = new_val; 3050 q_vector->set_itr = 1; 3051 } 3052 clear_counts: 3053 q_vector->rx.total_bytes = 0; 3054 q_vector->rx.total_packets = 0; 3055 q_vector->tx.total_bytes = 0; 3056 q_vector->tx.total_packets = 0; 3057 } 3058 3059 /** 3060 * igc_update_itr - update the dynamic ITR value based on statistics 3061 * @q_vector: pointer to q_vector 3062 * @ring_container: ring info to update the itr for 3063 * 3064 * Stores a new ITR value based on packets and byte 3065 * counts during the last interrupt. The advantage of per interrupt 3066 * computation is faster updates and more accurate ITR for the current 3067 * traffic pattern. Constants in this function were computed 3068 * based on theoretical maximum wire speed and thresholds were set based 3069 * on testing data as well as attempting to minimize response time 3070 * while increasing bulk throughput. 3071 * NOTE: These calculations are only valid when operating in a single- 3072 * queue environment. 3073 */ 3074 static void igc_update_itr(struct igc_q_vector *q_vector, 3075 struct igc_ring_container *ring_container) 3076 { 3077 unsigned int packets = ring_container->total_packets; 3078 unsigned int bytes = ring_container->total_bytes; 3079 u8 itrval = ring_container->itr; 3080 3081 /* no packets, exit with status unchanged */ 3082 if (packets == 0) 3083 return; 3084 3085 switch (itrval) { 3086 case lowest_latency: 3087 /* handle TSO and jumbo frames */ 3088 if (bytes / packets > 8000) 3089 itrval = bulk_latency; 3090 else if ((packets < 5) && (bytes > 512)) 3091 itrval = low_latency; 3092 break; 3093 case low_latency: /* 50 usec aka 20000 ints/s */ 3094 if (bytes > 10000) { 3095 /* this if handles the TSO accounting */ 3096 if (bytes / packets > 8000) 3097 itrval = bulk_latency; 3098 else if ((packets < 10) || ((bytes / packets) > 1200)) 3099 itrval = bulk_latency; 3100 else if ((packets > 35)) 3101 itrval = lowest_latency; 3102 } else if (bytes / packets > 2000) { 3103 itrval = bulk_latency; 3104 } else if (packets <= 2 && bytes < 512) { 3105 itrval = lowest_latency; 3106 } 3107 break; 3108 case bulk_latency: /* 250 usec aka 4000 ints/s */ 3109 if (bytes > 25000) { 3110 if (packets > 35) 3111 itrval = low_latency; 3112 } else if (bytes < 1500) { 3113 itrval = low_latency; 3114 } 3115 break; 3116 } 3117 3118 /* clear work counters since we have the values we need */ 3119 ring_container->total_bytes = 0; 3120 ring_container->total_packets = 0; 3121 3122 /* write updated itr to ring container */ 3123 ring_container->itr = itrval; 3124 } 3125 3126 /** 3127 * igc_intr_msi - Interrupt Handler 3128 * @irq: interrupt number 3129 * @data: pointer to a network interface device structure 3130 */ 3131 static irqreturn_t igc_intr_msi(int irq, void *data) 3132 { 3133 struct igc_adapter *adapter = data; 3134 struct igc_q_vector *q_vector = adapter->q_vector[0]; 3135 struct igc_hw *hw = &adapter->hw; 3136 /* read ICR disables interrupts using IAM */ 3137 u32 icr = rd32(IGC_ICR); 3138 3139 igc_write_itr(q_vector); 3140 3141 if (icr & IGC_ICR_DRSTA) 3142 schedule_work(&adapter->reset_task); 3143 3144 if (icr & IGC_ICR_DOUTSYNC) { 3145 /* HW is reporting DMA is out of sync */ 3146 adapter->stats.doosync++; 3147 } 3148 3149 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 3150 hw->mac.get_link_status = 1; 3151 if (!test_bit(__IGC_DOWN, &adapter->state)) 3152 mod_timer(&adapter->watchdog_timer, jiffies + 1); 3153 } 3154 3155 napi_schedule(&q_vector->napi); 3156 3157 return IRQ_HANDLED; 3158 } 3159 3160 /** 3161 * igc_intr - Legacy Interrupt Handler 3162 * @irq: interrupt number 3163 * @data: pointer to a network interface device structure 3164 */ 3165 static irqreturn_t igc_intr(int irq, void *data) 3166 { 3167 struct igc_adapter *adapter = data; 3168 struct igc_q_vector *q_vector = adapter->q_vector[0]; 3169 struct igc_hw *hw = &adapter->hw; 3170 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 3171 * need for the IMC write 3172 */ 3173 u32 icr = rd32(IGC_ICR); 3174 3175 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 3176 * not set, then the adapter didn't send an interrupt 3177 */ 3178 if (!(icr & IGC_ICR_INT_ASSERTED)) 3179 return IRQ_NONE; 3180 3181 igc_write_itr(q_vector); 3182 3183 if (icr & IGC_ICR_DRSTA) 3184 schedule_work(&adapter->reset_task); 3185 3186 if (icr & IGC_ICR_DOUTSYNC) { 3187 /* HW is reporting DMA is out of sync */ 3188 adapter->stats.doosync++; 3189 } 3190 3191 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 3192 hw->mac.get_link_status = 1; 3193 /* guard against interrupt when we're going down */ 3194 if (!test_bit(__IGC_DOWN, &adapter->state)) 3195 mod_timer(&adapter->watchdog_timer, jiffies + 1); 3196 } 3197 3198 napi_schedule(&q_vector->napi); 3199 3200 return IRQ_HANDLED; 3201 } 3202 3203 static void igc_set_itr(struct igc_q_vector *q_vector) 3204 { 3205 struct igc_adapter *adapter = q_vector->adapter; 3206 u32 new_itr = q_vector->itr_val; 3207 u8 current_itr = 0; 3208 3209 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 3210 switch (adapter->link_speed) { 3211 case SPEED_10: 3212 case SPEED_100: 3213 current_itr = 0; 3214 new_itr = IGC_4K_ITR; 3215 goto set_itr_now; 3216 default: 3217 break; 3218 } 3219 3220 igc_update_itr(q_vector, &q_vector->tx); 3221 igc_update_itr(q_vector, &q_vector->rx); 3222 3223 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 3224 3225 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3226 if (current_itr == lowest_latency && 3227 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3228 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3229 current_itr = low_latency; 3230 3231 switch (current_itr) { 3232 /* counts and packets in update_itr are dependent on these numbers */ 3233 case lowest_latency: 3234 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 3235 break; 3236 case low_latency: 3237 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 3238 break; 3239 case bulk_latency: 3240 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 3241 break; 3242 default: 3243 break; 3244 } 3245 3246 set_itr_now: 3247 if (new_itr != q_vector->itr_val) { 3248 /* this attempts to bias the interrupt rate towards Bulk 3249 * by adding intermediate steps when interrupt rate is 3250 * increasing 3251 */ 3252 new_itr = new_itr > q_vector->itr_val ? 3253 max((new_itr * q_vector->itr_val) / 3254 (new_itr + (q_vector->itr_val >> 2)), 3255 new_itr) : new_itr; 3256 /* Don't write the value here; it resets the adapter's 3257 * internal timer, and causes us to delay far longer than 3258 * we should between interrupts. Instead, we write the ITR 3259 * value at the beginning of the next interrupt so the timing 3260 * ends up being correct. 3261 */ 3262 q_vector->itr_val = new_itr; 3263 q_vector->set_itr = 1; 3264 } 3265 } 3266 3267 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 3268 { 3269 struct igc_adapter *adapter = q_vector->adapter; 3270 struct igc_hw *hw = &adapter->hw; 3271 3272 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 3273 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 3274 if (adapter->num_q_vectors == 1) 3275 igc_set_itr(q_vector); 3276 else 3277 igc_update_ring_itr(q_vector); 3278 } 3279 3280 if (!test_bit(__IGC_DOWN, &adapter->state)) { 3281 if (adapter->msix_entries) 3282 wr32(IGC_EIMS, q_vector->eims_value); 3283 else 3284 igc_irq_enable(adapter); 3285 } 3286 } 3287 3288 /** 3289 * igc_poll - NAPI Rx polling callback 3290 * @napi: napi polling structure 3291 * @budget: count of how many packets we should handle 3292 */ 3293 static int igc_poll(struct napi_struct *napi, int budget) 3294 { 3295 struct igc_q_vector *q_vector = container_of(napi, 3296 struct igc_q_vector, 3297 napi); 3298 bool clean_complete = true; 3299 int work_done = 0; 3300 3301 if (q_vector->tx.ring) 3302 clean_complete = igc_clean_tx_irq(q_vector, budget); 3303 3304 if (q_vector->rx.ring) { 3305 int cleaned = igc_clean_rx_irq(q_vector, budget); 3306 3307 work_done += cleaned; 3308 if (cleaned >= budget) 3309 clean_complete = false; 3310 } 3311 3312 /* If all work not completed, return budget and keep polling */ 3313 if (!clean_complete) 3314 return budget; 3315 3316 /* Exit the polling mode, but don't re-enable interrupts if stack might 3317 * poll us due to busy-polling 3318 */ 3319 if (likely(napi_complete_done(napi, work_done))) 3320 igc_ring_irq_enable(q_vector); 3321 3322 return min(work_done, budget - 1); 3323 } 3324 3325 /** 3326 * igc_set_interrupt_capability - set MSI or MSI-X if supported 3327 * @adapter: Pointer to adapter structure 3328 * 3329 * Attempt to configure interrupts using the best available 3330 * capabilities of the hardware and kernel. 3331 */ 3332 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 3333 bool msix) 3334 { 3335 int numvecs, i; 3336 int err; 3337 3338 if (!msix) 3339 goto msi_only; 3340 adapter->flags |= IGC_FLAG_HAS_MSIX; 3341 3342 /* Number of supported queues. */ 3343 adapter->num_rx_queues = adapter->rss_queues; 3344 3345 adapter->num_tx_queues = adapter->rss_queues; 3346 3347 /* start with one vector for every Rx queue */ 3348 numvecs = adapter->num_rx_queues; 3349 3350 /* if Tx handler is separate add 1 for every Tx queue */ 3351 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 3352 numvecs += adapter->num_tx_queues; 3353 3354 /* store the number of vectors reserved for queues */ 3355 adapter->num_q_vectors = numvecs; 3356 3357 /* add 1 vector for link status interrupts */ 3358 numvecs++; 3359 3360 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), 3361 GFP_KERNEL); 3362 3363 if (!adapter->msix_entries) 3364 return; 3365 3366 /* populate entry values */ 3367 for (i = 0; i < numvecs; i++) 3368 adapter->msix_entries[i].entry = i; 3369 3370 err = pci_enable_msix_range(adapter->pdev, 3371 adapter->msix_entries, 3372 numvecs, 3373 numvecs); 3374 if (err > 0) 3375 return; 3376 3377 kfree(adapter->msix_entries); 3378 adapter->msix_entries = NULL; 3379 3380 igc_reset_interrupt_capability(adapter); 3381 3382 msi_only: 3383 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 3384 3385 adapter->rss_queues = 1; 3386 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 3387 adapter->num_rx_queues = 1; 3388 adapter->num_tx_queues = 1; 3389 adapter->num_q_vectors = 1; 3390 if (!pci_enable_msi(adapter->pdev)) 3391 adapter->flags |= IGC_FLAG_HAS_MSI; 3392 } 3393 3394 static void igc_add_ring(struct igc_ring *ring, 3395 struct igc_ring_container *head) 3396 { 3397 head->ring = ring; 3398 head->count++; 3399 } 3400 3401 /** 3402 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 3403 * @adapter: board private structure to initialize 3404 * @v_count: q_vectors allocated on adapter, used for ring interleaving 3405 * @v_idx: index of vector in adapter struct 3406 * @txr_count: total number of Tx rings to allocate 3407 * @txr_idx: index of first Tx ring to allocate 3408 * @rxr_count: total number of Rx rings to allocate 3409 * @rxr_idx: index of first Rx ring to allocate 3410 * 3411 * We allocate one q_vector. If allocation fails we return -ENOMEM. 3412 */ 3413 static int igc_alloc_q_vector(struct igc_adapter *adapter, 3414 unsigned int v_count, unsigned int v_idx, 3415 unsigned int txr_count, unsigned int txr_idx, 3416 unsigned int rxr_count, unsigned int rxr_idx) 3417 { 3418 struct igc_q_vector *q_vector; 3419 struct igc_ring *ring; 3420 int ring_count; 3421 3422 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 3423 if (txr_count > 1 || rxr_count > 1) 3424 return -ENOMEM; 3425 3426 ring_count = txr_count + rxr_count; 3427 3428 /* allocate q_vector and rings */ 3429 q_vector = adapter->q_vector[v_idx]; 3430 if (!q_vector) 3431 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), 3432 GFP_KERNEL); 3433 else 3434 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 3435 if (!q_vector) 3436 return -ENOMEM; 3437 3438 /* initialize NAPI */ 3439 netif_napi_add(adapter->netdev, &q_vector->napi, 3440 igc_poll, 64); 3441 3442 /* tie q_vector and adapter together */ 3443 adapter->q_vector[v_idx] = q_vector; 3444 q_vector->adapter = adapter; 3445 3446 /* initialize work limits */ 3447 q_vector->tx.work_limit = adapter->tx_work_limit; 3448 3449 /* initialize ITR configuration */ 3450 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 3451 q_vector->itr_val = IGC_START_ITR; 3452 3453 /* initialize pointer to rings */ 3454 ring = q_vector->ring; 3455 3456 /* initialize ITR */ 3457 if (rxr_count) { 3458 /* rx or rx/tx vector */ 3459 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 3460 q_vector->itr_val = adapter->rx_itr_setting; 3461 } else { 3462 /* tx only vector */ 3463 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 3464 q_vector->itr_val = adapter->tx_itr_setting; 3465 } 3466 3467 if (txr_count) { 3468 /* assign generic ring traits */ 3469 ring->dev = &adapter->pdev->dev; 3470 ring->netdev = adapter->netdev; 3471 3472 /* configure backlink on ring */ 3473 ring->q_vector = q_vector; 3474 3475 /* update q_vector Tx values */ 3476 igc_add_ring(ring, &q_vector->tx); 3477 3478 /* apply Tx specific ring traits */ 3479 ring->count = adapter->tx_ring_count; 3480 ring->queue_index = txr_idx; 3481 3482 /* assign ring to adapter */ 3483 adapter->tx_ring[txr_idx] = ring; 3484 3485 /* push pointer to next ring */ 3486 ring++; 3487 } 3488 3489 if (rxr_count) { 3490 /* assign generic ring traits */ 3491 ring->dev = &adapter->pdev->dev; 3492 ring->netdev = adapter->netdev; 3493 3494 /* configure backlink on ring */ 3495 ring->q_vector = q_vector; 3496 3497 /* update q_vector Rx values */ 3498 igc_add_ring(ring, &q_vector->rx); 3499 3500 /* apply Rx specific ring traits */ 3501 ring->count = adapter->rx_ring_count; 3502 ring->queue_index = rxr_idx; 3503 3504 /* assign ring to adapter */ 3505 adapter->rx_ring[rxr_idx] = ring; 3506 } 3507 3508 return 0; 3509 } 3510 3511 /** 3512 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 3513 * @adapter: board private structure to initialize 3514 * 3515 * We allocate one q_vector per queue interrupt. If allocation fails we 3516 * return -ENOMEM. 3517 */ 3518 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 3519 { 3520 int rxr_remaining = adapter->num_rx_queues; 3521 int txr_remaining = adapter->num_tx_queues; 3522 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 3523 int q_vectors = adapter->num_q_vectors; 3524 int err; 3525 3526 if (q_vectors >= (rxr_remaining + txr_remaining)) { 3527 for (; rxr_remaining; v_idx++) { 3528 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3529 0, 0, 1, rxr_idx); 3530 3531 if (err) 3532 goto err_out; 3533 3534 /* update counts and index */ 3535 rxr_remaining--; 3536 rxr_idx++; 3537 } 3538 } 3539 3540 for (; v_idx < q_vectors; v_idx++) { 3541 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 3542 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 3543 3544 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3545 tqpv, txr_idx, rqpv, rxr_idx); 3546 3547 if (err) 3548 goto err_out; 3549 3550 /* update counts and index */ 3551 rxr_remaining -= rqpv; 3552 txr_remaining -= tqpv; 3553 rxr_idx++; 3554 txr_idx++; 3555 } 3556 3557 return 0; 3558 3559 err_out: 3560 adapter->num_tx_queues = 0; 3561 adapter->num_rx_queues = 0; 3562 adapter->num_q_vectors = 0; 3563 3564 while (v_idx--) 3565 igc_free_q_vector(adapter, v_idx); 3566 3567 return -ENOMEM; 3568 } 3569 3570 /** 3571 * igc_cache_ring_register - Descriptor ring to register mapping 3572 * @adapter: board private structure to initialize 3573 * 3574 * Once we know the feature-set enabled for the device, we'll cache 3575 * the register offset the descriptor ring is assigned to. 3576 */ 3577 static void igc_cache_ring_register(struct igc_adapter *adapter) 3578 { 3579 int i = 0, j = 0; 3580 3581 switch (adapter->hw.mac.type) { 3582 case igc_i225: 3583 /* Fall through */ 3584 default: 3585 for (; i < adapter->num_rx_queues; i++) 3586 adapter->rx_ring[i]->reg_idx = i; 3587 for (; j < adapter->num_tx_queues; j++) 3588 adapter->tx_ring[j]->reg_idx = j; 3589 break; 3590 } 3591 } 3592 3593 /** 3594 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 3595 * @adapter: Pointer to adapter structure 3596 * 3597 * This function initializes the interrupts and allocates all of the queues. 3598 */ 3599 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 3600 { 3601 struct pci_dev *pdev = adapter->pdev; 3602 int err = 0; 3603 3604 igc_set_interrupt_capability(adapter, msix); 3605 3606 err = igc_alloc_q_vectors(adapter); 3607 if (err) { 3608 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); 3609 goto err_alloc_q_vectors; 3610 } 3611 3612 igc_cache_ring_register(adapter); 3613 3614 return 0; 3615 3616 err_alloc_q_vectors: 3617 igc_reset_interrupt_capability(adapter); 3618 return err; 3619 } 3620 3621 static void igc_free_irq(struct igc_adapter *adapter) 3622 { 3623 if (adapter->msix_entries) { 3624 int vector = 0, i; 3625 3626 free_irq(adapter->msix_entries[vector++].vector, adapter); 3627 3628 for (i = 0; i < adapter->num_q_vectors; i++) 3629 free_irq(adapter->msix_entries[vector++].vector, 3630 adapter->q_vector[i]); 3631 } else { 3632 free_irq(adapter->pdev->irq, adapter); 3633 } 3634 } 3635 3636 /** 3637 * igc_irq_disable - Mask off interrupt generation on the NIC 3638 * @adapter: board private structure 3639 */ 3640 static void igc_irq_disable(struct igc_adapter *adapter) 3641 { 3642 struct igc_hw *hw = &adapter->hw; 3643 3644 if (adapter->msix_entries) { 3645 u32 regval = rd32(IGC_EIAM); 3646 3647 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 3648 wr32(IGC_EIMC, adapter->eims_enable_mask); 3649 regval = rd32(IGC_EIAC); 3650 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 3651 } 3652 3653 wr32(IGC_IAM, 0); 3654 wr32(IGC_IMC, ~0); 3655 wrfl(); 3656 3657 if (adapter->msix_entries) { 3658 int vector = 0, i; 3659 3660 synchronize_irq(adapter->msix_entries[vector++].vector); 3661 3662 for (i = 0; i < adapter->num_q_vectors; i++) 3663 synchronize_irq(adapter->msix_entries[vector++].vector); 3664 } else { 3665 synchronize_irq(adapter->pdev->irq); 3666 } 3667 } 3668 3669 /** 3670 * igc_irq_enable - Enable default interrupt generation settings 3671 * @adapter: board private structure 3672 */ 3673 static void igc_irq_enable(struct igc_adapter *adapter) 3674 { 3675 struct igc_hw *hw = &adapter->hw; 3676 3677 if (adapter->msix_entries) { 3678 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 3679 u32 regval = rd32(IGC_EIAC); 3680 3681 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 3682 regval = rd32(IGC_EIAM); 3683 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 3684 wr32(IGC_EIMS, adapter->eims_enable_mask); 3685 wr32(IGC_IMS, ims); 3686 } else { 3687 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 3688 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 3689 } 3690 } 3691 3692 /** 3693 * igc_request_irq - initialize interrupts 3694 * @adapter: Pointer to adapter structure 3695 * 3696 * Attempts to configure interrupts using the best available 3697 * capabilities of the hardware and kernel. 3698 */ 3699 static int igc_request_irq(struct igc_adapter *adapter) 3700 { 3701 struct net_device *netdev = adapter->netdev; 3702 struct pci_dev *pdev = adapter->pdev; 3703 int err = 0; 3704 3705 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 3706 err = igc_request_msix(adapter); 3707 if (!err) 3708 goto request_done; 3709 /* fall back to MSI */ 3710 igc_free_all_tx_resources(adapter); 3711 igc_free_all_rx_resources(adapter); 3712 3713 igc_clear_interrupt_scheme(adapter); 3714 err = igc_init_interrupt_scheme(adapter, false); 3715 if (err) 3716 goto request_done; 3717 igc_setup_all_tx_resources(adapter); 3718 igc_setup_all_rx_resources(adapter); 3719 igc_configure(adapter); 3720 } 3721 3722 igc_assign_vector(adapter->q_vector[0], 0); 3723 3724 if (adapter->flags & IGC_FLAG_HAS_MSI) { 3725 err = request_irq(pdev->irq, &igc_intr_msi, 0, 3726 netdev->name, adapter); 3727 if (!err) 3728 goto request_done; 3729 3730 /* fall back to legacy interrupts */ 3731 igc_reset_interrupt_capability(adapter); 3732 adapter->flags &= ~IGC_FLAG_HAS_MSI; 3733 } 3734 3735 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 3736 netdev->name, adapter); 3737 3738 if (err) 3739 dev_err(&pdev->dev, "Error %d getting interrupt\n", 3740 err); 3741 3742 request_done: 3743 return err; 3744 } 3745 3746 static void igc_write_itr(struct igc_q_vector *q_vector) 3747 { 3748 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 3749 3750 if (!q_vector->set_itr) 3751 return; 3752 3753 if (!itr_val) 3754 itr_val = IGC_ITR_VAL_MASK; 3755 3756 itr_val |= IGC_EITR_CNT_IGNR; 3757 3758 writel(itr_val, q_vector->itr_register); 3759 q_vector->set_itr = 0; 3760 } 3761 3762 /** 3763 * igc_open - Called when a network interface is made active 3764 * @netdev: network interface device structure 3765 * 3766 * Returns 0 on success, negative value on failure 3767 * 3768 * The open entry point is called when a network interface is made 3769 * active by the system (IFF_UP). At this point all resources needed 3770 * for transmit and receive operations are allocated, the interrupt 3771 * handler is registered with the OS, the watchdog timer is started, 3772 * and the stack is notified that the interface is ready. 3773 */ 3774 static int __igc_open(struct net_device *netdev, bool resuming) 3775 { 3776 struct igc_adapter *adapter = netdev_priv(netdev); 3777 struct igc_hw *hw = &adapter->hw; 3778 int err = 0; 3779 int i = 0; 3780 3781 /* disallow open during test */ 3782 3783 if (test_bit(__IGC_TESTING, &adapter->state)) { 3784 WARN_ON(resuming); 3785 return -EBUSY; 3786 } 3787 3788 netif_carrier_off(netdev); 3789 3790 /* allocate transmit descriptors */ 3791 err = igc_setup_all_tx_resources(adapter); 3792 if (err) 3793 goto err_setup_tx; 3794 3795 /* allocate receive descriptors */ 3796 err = igc_setup_all_rx_resources(adapter); 3797 if (err) 3798 goto err_setup_rx; 3799 3800 igc_power_up_link(adapter); 3801 3802 igc_configure(adapter); 3803 3804 err = igc_request_irq(adapter); 3805 if (err) 3806 goto err_req_irq; 3807 3808 /* Notify the stack of the actual queue counts. */ 3809 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 3810 if (err) 3811 goto err_set_queues; 3812 3813 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 3814 if (err) 3815 goto err_set_queues; 3816 3817 clear_bit(__IGC_DOWN, &adapter->state); 3818 3819 for (i = 0; i < adapter->num_q_vectors; i++) 3820 napi_enable(&adapter->q_vector[i]->napi); 3821 3822 /* Clear any pending interrupts. */ 3823 rd32(IGC_ICR); 3824 igc_irq_enable(adapter); 3825 3826 netif_tx_start_all_queues(netdev); 3827 3828 /* start the watchdog. */ 3829 hw->mac.get_link_status = 1; 3830 schedule_work(&adapter->watchdog_task); 3831 3832 return IGC_SUCCESS; 3833 3834 err_set_queues: 3835 igc_free_irq(adapter); 3836 err_req_irq: 3837 igc_release_hw_control(adapter); 3838 igc_power_down_link(adapter); 3839 igc_free_all_rx_resources(adapter); 3840 err_setup_rx: 3841 igc_free_all_tx_resources(adapter); 3842 err_setup_tx: 3843 igc_reset(adapter); 3844 3845 return err; 3846 } 3847 3848 static int igc_open(struct net_device *netdev) 3849 { 3850 return __igc_open(netdev, false); 3851 } 3852 3853 /** 3854 * igc_close - Disables a network interface 3855 * @netdev: network interface device structure 3856 * 3857 * Returns 0, this is not allowed to fail 3858 * 3859 * The close entry point is called when an interface is de-activated 3860 * by the OS. The hardware is still under the driver's control, but 3861 * needs to be disabled. A global MAC reset is issued to stop the 3862 * hardware, and all transmit and receive resources are freed. 3863 */ 3864 static int __igc_close(struct net_device *netdev, bool suspending) 3865 { 3866 struct igc_adapter *adapter = netdev_priv(netdev); 3867 3868 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 3869 3870 igc_down(adapter); 3871 3872 igc_release_hw_control(adapter); 3873 3874 igc_free_irq(adapter); 3875 3876 igc_free_all_tx_resources(adapter); 3877 igc_free_all_rx_resources(adapter); 3878 3879 return 0; 3880 } 3881 3882 static int igc_close(struct net_device *netdev) 3883 { 3884 if (netif_device_present(netdev) || netdev->dismantle) 3885 return __igc_close(netdev, false); 3886 return 0; 3887 } 3888 3889 static const struct net_device_ops igc_netdev_ops = { 3890 .ndo_open = igc_open, 3891 .ndo_stop = igc_close, 3892 .ndo_start_xmit = igc_xmit_frame, 3893 .ndo_set_mac_address = igc_set_mac, 3894 .ndo_change_mtu = igc_change_mtu, 3895 .ndo_get_stats = igc_get_stats, 3896 .ndo_fix_features = igc_fix_features, 3897 .ndo_set_features = igc_set_features, 3898 .ndo_features_check = igc_features_check, 3899 }; 3900 3901 /* PCIe configuration access */ 3902 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 3903 { 3904 struct igc_adapter *adapter = hw->back; 3905 3906 pci_read_config_word(adapter->pdev, reg, value); 3907 } 3908 3909 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 3910 { 3911 struct igc_adapter *adapter = hw->back; 3912 3913 pci_write_config_word(adapter->pdev, reg, *value); 3914 } 3915 3916 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 3917 { 3918 struct igc_adapter *adapter = hw->back; 3919 3920 if (!pci_is_pcie(adapter->pdev)) 3921 return -IGC_ERR_CONFIG; 3922 3923 pcie_capability_read_word(adapter->pdev, reg, value); 3924 3925 return IGC_SUCCESS; 3926 } 3927 3928 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 3929 { 3930 struct igc_adapter *adapter = hw->back; 3931 3932 if (!pci_is_pcie(adapter->pdev)) 3933 return -IGC_ERR_CONFIG; 3934 3935 pcie_capability_write_word(adapter->pdev, reg, *value); 3936 3937 return IGC_SUCCESS; 3938 } 3939 3940 u32 igc_rd32(struct igc_hw *hw, u32 reg) 3941 { 3942 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 3943 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 3944 u32 value = 0; 3945 3946 if (IGC_REMOVED(hw_addr)) 3947 return ~value; 3948 3949 value = readl(&hw_addr[reg]); 3950 3951 /* reads should not return all F's */ 3952 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 3953 struct net_device *netdev = igc->netdev; 3954 3955 hw->hw_addr = NULL; 3956 netif_device_detach(netdev); 3957 netdev_err(netdev, "PCIe link lost, device now detached\n"); 3958 WARN(1, "igc: Failed to read reg 0x%x!\n", reg); 3959 } 3960 3961 return value; 3962 } 3963 3964 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) 3965 { 3966 struct pci_dev *pdev = adapter->pdev; 3967 struct igc_mac_info *mac = &adapter->hw.mac; 3968 3969 mac->autoneg = 0; 3970 3971 /* Make sure dplx is at most 1 bit and lsb of speed is not set 3972 * for the switch() below to work 3973 */ 3974 if ((spd & 1) || (dplx & ~1)) 3975 goto err_inval; 3976 3977 switch (spd + dplx) { 3978 case SPEED_10 + DUPLEX_HALF: 3979 mac->forced_speed_duplex = ADVERTISE_10_HALF; 3980 break; 3981 case SPEED_10 + DUPLEX_FULL: 3982 mac->forced_speed_duplex = ADVERTISE_10_FULL; 3983 break; 3984 case SPEED_100 + DUPLEX_HALF: 3985 mac->forced_speed_duplex = ADVERTISE_100_HALF; 3986 break; 3987 case SPEED_100 + DUPLEX_FULL: 3988 mac->forced_speed_duplex = ADVERTISE_100_FULL; 3989 break; 3990 case SPEED_1000 + DUPLEX_FULL: 3991 mac->autoneg = 1; 3992 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 3993 break; 3994 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 3995 goto err_inval; 3996 case SPEED_2500 + DUPLEX_FULL: 3997 mac->autoneg = 1; 3998 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 3999 break; 4000 case SPEED_2500 + DUPLEX_HALF: /* not supported */ 4001 default: 4002 goto err_inval; 4003 } 4004 4005 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 4006 adapter->hw.phy.mdix = AUTO_ALL_MODES; 4007 4008 return 0; 4009 4010 err_inval: 4011 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n"); 4012 return -EINVAL; 4013 } 4014 4015 /** 4016 * igc_probe - Device Initialization Routine 4017 * @pdev: PCI device information struct 4018 * @ent: entry in igc_pci_tbl 4019 * 4020 * Returns 0 on success, negative on failure 4021 * 4022 * igc_probe initializes an adapter identified by a pci_dev structure. 4023 * The OS initialization, configuring the adapter private structure, 4024 * and a hardware reset occur. 4025 */ 4026 static int igc_probe(struct pci_dev *pdev, 4027 const struct pci_device_id *ent) 4028 { 4029 struct igc_adapter *adapter; 4030 struct net_device *netdev; 4031 struct igc_hw *hw; 4032 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 4033 int err; 4034 4035 err = pci_enable_device_mem(pdev); 4036 if (err) 4037 return err; 4038 4039 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 4040 if (!err) { 4041 err = dma_set_coherent_mask(&pdev->dev, 4042 DMA_BIT_MASK(64)); 4043 } else { 4044 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 4045 if (err) { 4046 err = dma_set_coherent_mask(&pdev->dev, 4047 DMA_BIT_MASK(32)); 4048 if (err) { 4049 dev_err(&pdev->dev, "igc: Wrong DMA config\n"); 4050 goto err_dma; 4051 } 4052 } 4053 } 4054 4055 err = pci_request_selected_regions(pdev, 4056 pci_select_bars(pdev, 4057 IORESOURCE_MEM), 4058 igc_driver_name); 4059 if (err) 4060 goto err_pci_reg; 4061 4062 pci_enable_pcie_error_reporting(pdev); 4063 4064 pci_set_master(pdev); 4065 4066 err = -ENOMEM; 4067 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 4068 IGC_MAX_TX_QUEUES); 4069 4070 if (!netdev) 4071 goto err_alloc_etherdev; 4072 4073 SET_NETDEV_DEV(netdev, &pdev->dev); 4074 4075 pci_set_drvdata(pdev, netdev); 4076 adapter = netdev_priv(netdev); 4077 adapter->netdev = netdev; 4078 adapter->pdev = pdev; 4079 hw = &adapter->hw; 4080 hw->back = adapter; 4081 adapter->port_num = hw->bus.func; 4082 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 4083 4084 err = pci_save_state(pdev); 4085 if (err) 4086 goto err_ioremap; 4087 4088 err = -EIO; 4089 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 4090 pci_resource_len(pdev, 0)); 4091 if (!adapter->io_addr) 4092 goto err_ioremap; 4093 4094 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 4095 hw->hw_addr = adapter->io_addr; 4096 4097 netdev->netdev_ops = &igc_netdev_ops; 4098 igc_set_ethtool_ops(netdev); 4099 netdev->watchdog_timeo = 5 * HZ; 4100 4101 netdev->mem_start = pci_resource_start(pdev, 0); 4102 netdev->mem_end = pci_resource_end(pdev, 0); 4103 4104 /* PCI config space info */ 4105 hw->vendor_id = pdev->vendor; 4106 hw->device_id = pdev->device; 4107 hw->revision_id = pdev->revision; 4108 hw->subsystem_vendor_id = pdev->subsystem_vendor; 4109 hw->subsystem_device_id = pdev->subsystem_device; 4110 4111 /* Copy the default MAC and PHY function pointers */ 4112 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 4113 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 4114 4115 /* Initialize skew-specific constants */ 4116 err = ei->get_invariants(hw); 4117 if (err) 4118 goto err_sw_init; 4119 4120 /* setup the private structure */ 4121 err = igc_sw_init(adapter); 4122 if (err) 4123 goto err_sw_init; 4124 4125 /* copy netdev features into list of user selectable features */ 4126 netdev->hw_features |= NETIF_F_NTUPLE; 4127 4128 /* MTU range: 68 - 9216 */ 4129 netdev->min_mtu = ETH_MIN_MTU; 4130 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 4131 4132 /* before reading the NVM, reset the controller to put the device in a 4133 * known good starting state 4134 */ 4135 hw->mac.ops.reset_hw(hw); 4136 4137 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 4138 /* copy the MAC address out of the NVM */ 4139 if (hw->mac.ops.read_mac_addr(hw)) 4140 dev_err(&pdev->dev, "NVM Read Error\n"); 4141 } 4142 4143 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 4144 4145 if (!is_valid_ether_addr(netdev->dev_addr)) { 4146 dev_err(&pdev->dev, "Invalid MAC Address\n"); 4147 err = -EIO; 4148 goto err_eeprom; 4149 } 4150 4151 /* configure RXPBSIZE and TXPBSIZE */ 4152 wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT); 4153 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); 4154 4155 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 4156 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 4157 4158 INIT_WORK(&adapter->reset_task, igc_reset_task); 4159 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 4160 4161 /* Initialize link properties that are user-changeable */ 4162 adapter->fc_autoneg = true; 4163 hw->mac.autoneg = true; 4164 hw->phy.autoneg_advertised = 0xaf; 4165 4166 hw->fc.requested_mode = igc_fc_default; 4167 hw->fc.current_mode = igc_fc_default; 4168 4169 /* reset the hardware with the new settings */ 4170 igc_reset(adapter); 4171 4172 /* let the f/w know that the h/w is now under the control of the 4173 * driver. 4174 */ 4175 igc_get_hw_control(adapter); 4176 4177 strncpy(netdev->name, "eth%d", IFNAMSIZ); 4178 err = register_netdev(netdev); 4179 if (err) 4180 goto err_register; 4181 4182 /* carrier off reporting is important to ethtool even BEFORE open */ 4183 netif_carrier_off(netdev); 4184 4185 /* Check if Media Autosense is enabled */ 4186 adapter->ei = *ei; 4187 4188 /* print pcie link status and MAC address */ 4189 pcie_print_link_status(pdev); 4190 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 4191 4192 return 0; 4193 4194 err_register: 4195 igc_release_hw_control(adapter); 4196 err_eeprom: 4197 if (!igc_check_reset_block(hw)) 4198 igc_reset_phy(hw); 4199 err_sw_init: 4200 igc_clear_interrupt_scheme(adapter); 4201 iounmap(adapter->io_addr); 4202 err_ioremap: 4203 free_netdev(netdev); 4204 err_alloc_etherdev: 4205 pci_release_selected_regions(pdev, 4206 pci_select_bars(pdev, IORESOURCE_MEM)); 4207 err_pci_reg: 4208 err_dma: 4209 pci_disable_device(pdev); 4210 return err; 4211 } 4212 4213 /** 4214 * igc_remove - Device Removal Routine 4215 * @pdev: PCI device information struct 4216 * 4217 * igc_remove is called by the PCI subsystem to alert the driver 4218 * that it should release a PCI device. This could be caused by a 4219 * Hot-Plug event, or because the driver is going to be removed from 4220 * memory. 4221 */ 4222 static void igc_remove(struct pci_dev *pdev) 4223 { 4224 struct net_device *netdev = pci_get_drvdata(pdev); 4225 struct igc_adapter *adapter = netdev_priv(netdev); 4226 4227 set_bit(__IGC_DOWN, &adapter->state); 4228 4229 del_timer_sync(&adapter->watchdog_timer); 4230 del_timer_sync(&adapter->phy_info_timer); 4231 4232 cancel_work_sync(&adapter->reset_task); 4233 cancel_work_sync(&adapter->watchdog_task); 4234 4235 /* Release control of h/w to f/w. If f/w is AMT enabled, this 4236 * would have already happened in close and is redundant. 4237 */ 4238 igc_release_hw_control(adapter); 4239 unregister_netdev(netdev); 4240 4241 igc_clear_interrupt_scheme(adapter); 4242 pci_iounmap(pdev, adapter->io_addr); 4243 pci_release_mem_regions(pdev); 4244 4245 kfree(adapter->mac_table); 4246 kfree(adapter->shadow_vfta); 4247 free_netdev(netdev); 4248 4249 pci_disable_pcie_error_reporting(pdev); 4250 4251 pci_disable_device(pdev); 4252 } 4253 4254 static struct pci_driver igc_driver = { 4255 .name = igc_driver_name, 4256 .id_table = igc_pci_tbl, 4257 .probe = igc_probe, 4258 .remove = igc_remove, 4259 }; 4260 4261 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 4262 const u32 max_rss_queues) 4263 { 4264 /* Determine if we need to pair queues. */ 4265 /* If rss_queues > half of max_rss_queues, pair the queues in 4266 * order to conserve interrupts due to limited supply. 4267 */ 4268 if (adapter->rss_queues > (max_rss_queues / 2)) 4269 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 4270 else 4271 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 4272 } 4273 4274 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 4275 { 4276 unsigned int max_rss_queues; 4277 4278 /* Determine the maximum number of RSS queues supported. */ 4279 max_rss_queues = IGC_MAX_RX_QUEUES; 4280 4281 return max_rss_queues; 4282 } 4283 4284 static void igc_init_queue_configuration(struct igc_adapter *adapter) 4285 { 4286 u32 max_rss_queues; 4287 4288 max_rss_queues = igc_get_max_rss_queues(adapter); 4289 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 4290 4291 igc_set_flag_queue_pairs(adapter, max_rss_queues); 4292 } 4293 4294 /** 4295 * igc_sw_init - Initialize general software structures (struct igc_adapter) 4296 * @adapter: board private structure to initialize 4297 * 4298 * igc_sw_init initializes the Adapter private data structure. 4299 * Fields are initialized based on PCI device information and 4300 * OS network device settings (MTU size). 4301 */ 4302 static int igc_sw_init(struct igc_adapter *adapter) 4303 { 4304 struct net_device *netdev = adapter->netdev; 4305 struct pci_dev *pdev = adapter->pdev; 4306 struct igc_hw *hw = &adapter->hw; 4307 4308 int size = sizeof(struct igc_mac_addr) * hw->mac.rar_entry_count; 4309 4310 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 4311 4312 /* set default ring sizes */ 4313 adapter->tx_ring_count = IGC_DEFAULT_TXD; 4314 adapter->rx_ring_count = IGC_DEFAULT_RXD; 4315 4316 /* set default ITR values */ 4317 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 4318 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 4319 4320 /* set default work limits */ 4321 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 4322 4323 /* adjust max frame to be at least the size of a standard frame */ 4324 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 4325 VLAN_HLEN; 4326 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 4327 4328 spin_lock_init(&adapter->nfc_lock); 4329 spin_lock_init(&adapter->stats64_lock); 4330 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 4331 adapter->flags |= IGC_FLAG_HAS_MSIX; 4332 4333 adapter->mac_table = kzalloc(size, GFP_ATOMIC); 4334 if (!adapter->mac_table) 4335 return -ENOMEM; 4336 4337 igc_init_queue_configuration(adapter); 4338 4339 /* This call may decrease the number of queues */ 4340 if (igc_init_interrupt_scheme(adapter, true)) { 4341 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 4342 return -ENOMEM; 4343 } 4344 4345 /* Explicitly disable IRQ since the NIC can be in any state. */ 4346 igc_irq_disable(adapter); 4347 4348 set_bit(__IGC_DOWN, &adapter->state); 4349 4350 return 0; 4351 } 4352 4353 /** 4354 * igc_reinit_queues - return error 4355 * @adapter: pointer to adapter structure 4356 */ 4357 int igc_reinit_queues(struct igc_adapter *adapter) 4358 { 4359 struct net_device *netdev = adapter->netdev; 4360 struct pci_dev *pdev = adapter->pdev; 4361 int err = 0; 4362 4363 if (netif_running(netdev)) 4364 igc_close(netdev); 4365 4366 igc_reset_interrupt_capability(adapter); 4367 4368 if (igc_init_interrupt_scheme(adapter, true)) { 4369 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 4370 return -ENOMEM; 4371 } 4372 4373 if (netif_running(netdev)) 4374 err = igc_open(netdev); 4375 4376 return err; 4377 } 4378 4379 /** 4380 * igc_get_hw_dev - return device 4381 * @hw: pointer to hardware structure 4382 * 4383 * used by hardware layer to print debugging information 4384 */ 4385 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 4386 { 4387 struct igc_adapter *adapter = hw->back; 4388 4389 return adapter->netdev; 4390 } 4391 4392 /** 4393 * igc_init_module - Driver Registration Routine 4394 * 4395 * igc_init_module is the first routine called when the driver is 4396 * loaded. All it does is register with the PCI subsystem. 4397 */ 4398 static int __init igc_init_module(void) 4399 { 4400 int ret; 4401 4402 pr_info("%s - version %s\n", 4403 igc_driver_string, igc_driver_version); 4404 4405 pr_info("%s\n", igc_copyright); 4406 4407 ret = pci_register_driver(&igc_driver); 4408 return ret; 4409 } 4410 4411 module_init(igc_init_module); 4412 4413 /** 4414 * igc_exit_module - Driver Exit Cleanup Routine 4415 * 4416 * igc_exit_module is called just before the driver is removed 4417 * from memory. 4418 */ 4419 static void __exit igc_exit_module(void) 4420 { 4421 pci_unregister_driver(&igc_driver); 4422 } 4423 4424 module_exit(igc_exit_module); 4425 /* igc_main.c */ 4426