1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/aer.h> 8 #include <linux/tcp.h> 9 #include <linux/udp.h> 10 #include <linux/ip.h> 11 12 #include <net/ipv6.h> 13 14 #include "igc.h" 15 #include "igc_hw.h" 16 17 #define DRV_VERSION "0.0.1-k" 18 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 19 20 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 21 22 static int debug = -1; 23 24 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 25 MODULE_DESCRIPTION(DRV_SUMMARY); 26 MODULE_LICENSE("GPL v2"); 27 MODULE_VERSION(DRV_VERSION); 28 module_param(debug, int, 0); 29 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 30 31 char igc_driver_name[] = "igc"; 32 char igc_driver_version[] = DRV_VERSION; 33 static const char igc_driver_string[] = DRV_SUMMARY; 34 static const char igc_copyright[] = 35 "Copyright(c) 2018 Intel Corporation."; 36 37 static const struct igc_info *igc_info_tbl[] = { 38 [board_base] = &igc_base_info, 39 }; 40 41 static const struct pci_device_id igc_pci_tbl[] = { 42 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base }, 43 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base }, 44 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, 45 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, 46 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, 47 /* required last entry */ 48 {0, } 49 }; 50 51 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 52 53 /* forward declaration */ 54 static void igc_clean_tx_ring(struct igc_ring *tx_ring); 55 static int igc_sw_init(struct igc_adapter *); 56 static void igc_configure(struct igc_adapter *adapter); 57 static void igc_power_down_link(struct igc_adapter *adapter); 58 static void igc_set_default_mac_filter(struct igc_adapter *adapter); 59 static void igc_set_rx_mode(struct net_device *netdev); 60 static void igc_write_itr(struct igc_q_vector *q_vector); 61 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector); 62 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx); 63 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 64 bool msix); 65 static void igc_free_q_vectors(struct igc_adapter *adapter); 66 static void igc_irq_disable(struct igc_adapter *adapter); 67 static void igc_irq_enable(struct igc_adapter *adapter); 68 static void igc_configure_msix(struct igc_adapter *adapter); 69 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 70 struct igc_rx_buffer *bi); 71 72 enum latency_range { 73 lowest_latency = 0, 74 low_latency = 1, 75 bulk_latency = 2, 76 latency_invalid = 255 77 }; 78 79 void igc_reset(struct igc_adapter *adapter) 80 { 81 struct pci_dev *pdev = adapter->pdev; 82 struct igc_hw *hw = &adapter->hw; 83 struct igc_fc_info *fc = &hw->fc; 84 u32 pba, hwm; 85 86 /* Repartition PBA for greater than 9k MTU if required */ 87 pba = IGC_PBA_34K; 88 89 /* flow control settings 90 * The high water mark must be low enough to fit one full frame 91 * after transmitting the pause frame. As such we must have enough 92 * space to allow for us to complete our current transmit and then 93 * receive the frame that is in progress from the link partner. 94 * Set it to: 95 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 96 */ 97 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 98 99 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 100 fc->low_water = fc->high_water - 16; 101 fc->pause_time = 0xFFFF; 102 fc->send_xon = 1; 103 fc->current_mode = fc->requested_mode; 104 105 hw->mac.ops.reset_hw(hw); 106 107 if (hw->mac.ops.init_hw(hw)) 108 dev_err(&pdev->dev, "Hardware Error\n"); 109 110 if (!netif_running(adapter->netdev)) 111 igc_power_down_link(adapter); 112 113 igc_get_phy_info(hw); 114 } 115 116 /** 117 * igc_power_up_link - Power up the phy/serdes link 118 * @adapter: address of board private structure 119 */ 120 static void igc_power_up_link(struct igc_adapter *adapter) 121 { 122 igc_reset_phy(&adapter->hw); 123 124 if (adapter->hw.phy.media_type == igc_media_type_copper) 125 igc_power_up_phy_copper(&adapter->hw); 126 127 igc_setup_link(&adapter->hw); 128 } 129 130 /** 131 * igc_power_down_link - Power down the phy/serdes link 132 * @adapter: address of board private structure 133 */ 134 static void igc_power_down_link(struct igc_adapter *adapter) 135 { 136 if (adapter->hw.phy.media_type == igc_media_type_copper) 137 igc_power_down_phy_copper_base(&adapter->hw); 138 } 139 140 /** 141 * igc_release_hw_control - release control of the h/w to f/w 142 * @adapter: address of board private structure 143 * 144 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 145 * For ASF and Pass Through versions of f/w this means that the 146 * driver is no longer loaded. 147 */ 148 static void igc_release_hw_control(struct igc_adapter *adapter) 149 { 150 struct igc_hw *hw = &adapter->hw; 151 u32 ctrl_ext; 152 153 /* Let firmware take over control of h/w */ 154 ctrl_ext = rd32(IGC_CTRL_EXT); 155 wr32(IGC_CTRL_EXT, 156 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 157 } 158 159 /** 160 * igc_get_hw_control - get control of the h/w from f/w 161 * @adapter: address of board private structure 162 * 163 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 164 * For ASF and Pass Through versions of f/w this means that 165 * the driver is loaded. 166 */ 167 static void igc_get_hw_control(struct igc_adapter *adapter) 168 { 169 struct igc_hw *hw = &adapter->hw; 170 u32 ctrl_ext; 171 172 /* Let firmware know the driver has taken over */ 173 ctrl_ext = rd32(IGC_CTRL_EXT); 174 wr32(IGC_CTRL_EXT, 175 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 176 } 177 178 /** 179 * igc_free_tx_resources - Free Tx Resources per Queue 180 * @tx_ring: Tx descriptor ring for a specific queue 181 * 182 * Free all transmit software resources 183 */ 184 void igc_free_tx_resources(struct igc_ring *tx_ring) 185 { 186 igc_clean_tx_ring(tx_ring); 187 188 vfree(tx_ring->tx_buffer_info); 189 tx_ring->tx_buffer_info = NULL; 190 191 /* if not set, then don't free */ 192 if (!tx_ring->desc) 193 return; 194 195 dma_free_coherent(tx_ring->dev, tx_ring->size, 196 tx_ring->desc, tx_ring->dma); 197 198 tx_ring->desc = NULL; 199 } 200 201 /** 202 * igc_free_all_tx_resources - Free Tx Resources for All Queues 203 * @adapter: board private structure 204 * 205 * Free all transmit software resources 206 */ 207 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 208 { 209 int i; 210 211 for (i = 0; i < adapter->num_tx_queues; i++) 212 igc_free_tx_resources(adapter->tx_ring[i]); 213 } 214 215 /** 216 * igc_clean_tx_ring - Free Tx Buffers 217 * @tx_ring: ring to be cleaned 218 */ 219 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 220 { 221 u16 i = tx_ring->next_to_clean; 222 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 223 224 while (i != tx_ring->next_to_use) { 225 union igc_adv_tx_desc *eop_desc, *tx_desc; 226 227 /* Free all the Tx ring sk_buffs */ 228 dev_kfree_skb_any(tx_buffer->skb); 229 230 /* unmap skb header data */ 231 dma_unmap_single(tx_ring->dev, 232 dma_unmap_addr(tx_buffer, dma), 233 dma_unmap_len(tx_buffer, len), 234 DMA_TO_DEVICE); 235 236 /* check for eop_desc to determine the end of the packet */ 237 eop_desc = tx_buffer->next_to_watch; 238 tx_desc = IGC_TX_DESC(tx_ring, i); 239 240 /* unmap remaining buffers */ 241 while (tx_desc != eop_desc) { 242 tx_buffer++; 243 tx_desc++; 244 i++; 245 if (unlikely(i == tx_ring->count)) { 246 i = 0; 247 tx_buffer = tx_ring->tx_buffer_info; 248 tx_desc = IGC_TX_DESC(tx_ring, 0); 249 } 250 251 /* unmap any remaining paged data */ 252 if (dma_unmap_len(tx_buffer, len)) 253 dma_unmap_page(tx_ring->dev, 254 dma_unmap_addr(tx_buffer, dma), 255 dma_unmap_len(tx_buffer, len), 256 DMA_TO_DEVICE); 257 } 258 259 /* move us one more past the eop_desc for start of next pkt */ 260 tx_buffer++; 261 i++; 262 if (unlikely(i == tx_ring->count)) { 263 i = 0; 264 tx_buffer = tx_ring->tx_buffer_info; 265 } 266 } 267 268 /* reset BQL for queue */ 269 netdev_tx_reset_queue(txring_txq(tx_ring)); 270 271 /* reset next_to_use and next_to_clean */ 272 tx_ring->next_to_use = 0; 273 tx_ring->next_to_clean = 0; 274 } 275 276 /** 277 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 278 * @adapter: board private structure 279 */ 280 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 281 { 282 int i; 283 284 for (i = 0; i < adapter->num_tx_queues; i++) 285 if (adapter->tx_ring[i]) 286 igc_clean_tx_ring(adapter->tx_ring[i]); 287 } 288 289 /** 290 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 291 * @tx_ring: tx descriptor ring (for a specific queue) to setup 292 * 293 * Return 0 on success, negative on failure 294 */ 295 int igc_setup_tx_resources(struct igc_ring *tx_ring) 296 { 297 struct device *dev = tx_ring->dev; 298 int size = 0; 299 300 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 301 tx_ring->tx_buffer_info = vzalloc(size); 302 if (!tx_ring->tx_buffer_info) 303 goto err; 304 305 /* round up to nearest 4K */ 306 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 307 tx_ring->size = ALIGN(tx_ring->size, 4096); 308 309 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 310 &tx_ring->dma, GFP_KERNEL); 311 312 if (!tx_ring->desc) 313 goto err; 314 315 tx_ring->next_to_use = 0; 316 tx_ring->next_to_clean = 0; 317 318 return 0; 319 320 err: 321 vfree(tx_ring->tx_buffer_info); 322 dev_err(dev, 323 "Unable to allocate memory for the transmit descriptor ring\n"); 324 return -ENOMEM; 325 } 326 327 /** 328 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 329 * @adapter: board private structure 330 * 331 * Return 0 on success, negative on failure 332 */ 333 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 334 { 335 struct pci_dev *pdev = adapter->pdev; 336 int i, err = 0; 337 338 for (i = 0; i < adapter->num_tx_queues; i++) { 339 err = igc_setup_tx_resources(adapter->tx_ring[i]); 340 if (err) { 341 dev_err(&pdev->dev, 342 "Allocation for Tx Queue %u failed\n", i); 343 for (i--; i >= 0; i--) 344 igc_free_tx_resources(adapter->tx_ring[i]); 345 break; 346 } 347 } 348 349 return err; 350 } 351 352 /** 353 * igc_clean_rx_ring - Free Rx Buffers per Queue 354 * @rx_ring: ring to free buffers from 355 */ 356 static void igc_clean_rx_ring(struct igc_ring *rx_ring) 357 { 358 u16 i = rx_ring->next_to_clean; 359 360 dev_kfree_skb(rx_ring->skb); 361 rx_ring->skb = NULL; 362 363 /* Free all the Rx ring sk_buffs */ 364 while (i != rx_ring->next_to_alloc) { 365 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 366 367 /* Invalidate cache lines that may have been written to by 368 * device so that we avoid corrupting memory. 369 */ 370 dma_sync_single_range_for_cpu(rx_ring->dev, 371 buffer_info->dma, 372 buffer_info->page_offset, 373 igc_rx_bufsz(rx_ring), 374 DMA_FROM_DEVICE); 375 376 /* free resources associated with mapping */ 377 dma_unmap_page_attrs(rx_ring->dev, 378 buffer_info->dma, 379 igc_rx_pg_size(rx_ring), 380 DMA_FROM_DEVICE, 381 IGC_RX_DMA_ATTR); 382 __page_frag_cache_drain(buffer_info->page, 383 buffer_info->pagecnt_bias); 384 385 i++; 386 if (i == rx_ring->count) 387 i = 0; 388 } 389 390 rx_ring->next_to_alloc = 0; 391 rx_ring->next_to_clean = 0; 392 rx_ring->next_to_use = 0; 393 } 394 395 /** 396 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 397 * @adapter: board private structure 398 */ 399 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 400 { 401 int i; 402 403 for (i = 0; i < adapter->num_rx_queues; i++) 404 if (adapter->rx_ring[i]) 405 igc_clean_rx_ring(adapter->rx_ring[i]); 406 } 407 408 /** 409 * igc_free_rx_resources - Free Rx Resources 410 * @rx_ring: ring to clean the resources from 411 * 412 * Free all receive software resources 413 */ 414 void igc_free_rx_resources(struct igc_ring *rx_ring) 415 { 416 igc_clean_rx_ring(rx_ring); 417 418 vfree(rx_ring->rx_buffer_info); 419 rx_ring->rx_buffer_info = NULL; 420 421 /* if not set, then don't free */ 422 if (!rx_ring->desc) 423 return; 424 425 dma_free_coherent(rx_ring->dev, rx_ring->size, 426 rx_ring->desc, rx_ring->dma); 427 428 rx_ring->desc = NULL; 429 } 430 431 /** 432 * igc_free_all_rx_resources - Free Rx Resources for All Queues 433 * @adapter: board private structure 434 * 435 * Free all receive software resources 436 */ 437 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 438 { 439 int i; 440 441 for (i = 0; i < adapter->num_rx_queues; i++) 442 igc_free_rx_resources(adapter->rx_ring[i]); 443 } 444 445 /** 446 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 447 * @rx_ring: rx descriptor ring (for a specific queue) to setup 448 * 449 * Returns 0 on success, negative on failure 450 */ 451 int igc_setup_rx_resources(struct igc_ring *rx_ring) 452 { 453 struct device *dev = rx_ring->dev; 454 int size, desc_len; 455 456 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 457 rx_ring->rx_buffer_info = vzalloc(size); 458 if (!rx_ring->rx_buffer_info) 459 goto err; 460 461 desc_len = sizeof(union igc_adv_rx_desc); 462 463 /* Round up to nearest 4K */ 464 rx_ring->size = rx_ring->count * desc_len; 465 rx_ring->size = ALIGN(rx_ring->size, 4096); 466 467 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 468 &rx_ring->dma, GFP_KERNEL); 469 470 if (!rx_ring->desc) 471 goto err; 472 473 rx_ring->next_to_alloc = 0; 474 rx_ring->next_to_clean = 0; 475 rx_ring->next_to_use = 0; 476 477 return 0; 478 479 err: 480 vfree(rx_ring->rx_buffer_info); 481 rx_ring->rx_buffer_info = NULL; 482 dev_err(dev, 483 "Unable to allocate memory for the receive descriptor ring\n"); 484 return -ENOMEM; 485 } 486 487 /** 488 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 489 * (Descriptors) for all queues 490 * @adapter: board private structure 491 * 492 * Return 0 on success, negative on failure 493 */ 494 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 495 { 496 struct pci_dev *pdev = adapter->pdev; 497 int i, err = 0; 498 499 for (i = 0; i < adapter->num_rx_queues; i++) { 500 err = igc_setup_rx_resources(adapter->rx_ring[i]); 501 if (err) { 502 dev_err(&pdev->dev, 503 "Allocation for Rx Queue %u failed\n", i); 504 for (i--; i >= 0; i--) 505 igc_free_rx_resources(adapter->rx_ring[i]); 506 break; 507 } 508 } 509 510 return err; 511 } 512 513 /** 514 * igc_configure_rx_ring - Configure a receive ring after Reset 515 * @adapter: board private structure 516 * @ring: receive ring to be configured 517 * 518 * Configure the Rx unit of the MAC after a reset. 519 */ 520 static void igc_configure_rx_ring(struct igc_adapter *adapter, 521 struct igc_ring *ring) 522 { 523 struct igc_hw *hw = &adapter->hw; 524 union igc_adv_rx_desc *rx_desc; 525 int reg_idx = ring->reg_idx; 526 u32 srrctl = 0, rxdctl = 0; 527 u64 rdba = ring->dma; 528 529 /* disable the queue */ 530 wr32(IGC_RXDCTL(reg_idx), 0); 531 532 /* Set DMA base address registers */ 533 wr32(IGC_RDBAL(reg_idx), 534 rdba & 0x00000000ffffffffULL); 535 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 536 wr32(IGC_RDLEN(reg_idx), 537 ring->count * sizeof(union igc_adv_rx_desc)); 538 539 /* initialize head and tail */ 540 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 541 wr32(IGC_RDH(reg_idx), 0); 542 writel(0, ring->tail); 543 544 /* reset next-to- use/clean to place SW in sync with hardware */ 545 ring->next_to_clean = 0; 546 ring->next_to_use = 0; 547 548 /* set descriptor configuration */ 549 srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; 550 if (ring_uses_large_buffer(ring)) 551 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 552 else 553 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 554 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 555 556 wr32(IGC_SRRCTL(reg_idx), srrctl); 557 558 rxdctl |= IGC_RX_PTHRESH; 559 rxdctl |= IGC_RX_HTHRESH << 8; 560 rxdctl |= IGC_RX_WTHRESH << 16; 561 562 /* initialize rx_buffer_info */ 563 memset(ring->rx_buffer_info, 0, 564 sizeof(struct igc_rx_buffer) * ring->count); 565 566 /* initialize Rx descriptor 0 */ 567 rx_desc = IGC_RX_DESC(ring, 0); 568 rx_desc->wb.upper.length = 0; 569 570 /* enable receive descriptor fetching */ 571 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 572 573 wr32(IGC_RXDCTL(reg_idx), rxdctl); 574 } 575 576 /** 577 * igc_configure_rx - Configure receive Unit after Reset 578 * @adapter: board private structure 579 * 580 * Configure the Rx unit of the MAC after a reset. 581 */ 582 static void igc_configure_rx(struct igc_adapter *adapter) 583 { 584 int i; 585 586 /* Setup the HW Rx Head and Tail Descriptor Pointers and 587 * the Base and Length of the Rx Descriptor Ring 588 */ 589 for (i = 0; i < adapter->num_rx_queues; i++) 590 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 591 } 592 593 /** 594 * igc_configure_tx_ring - Configure transmit ring after Reset 595 * @adapter: board private structure 596 * @ring: tx ring to configure 597 * 598 * Configure a transmit ring after a reset. 599 */ 600 static void igc_configure_tx_ring(struct igc_adapter *adapter, 601 struct igc_ring *ring) 602 { 603 struct igc_hw *hw = &adapter->hw; 604 int reg_idx = ring->reg_idx; 605 u64 tdba = ring->dma; 606 u32 txdctl = 0; 607 608 /* disable the queue */ 609 wr32(IGC_TXDCTL(reg_idx), 0); 610 wrfl(); 611 mdelay(10); 612 613 wr32(IGC_TDLEN(reg_idx), 614 ring->count * sizeof(union igc_adv_tx_desc)); 615 wr32(IGC_TDBAL(reg_idx), 616 tdba & 0x00000000ffffffffULL); 617 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 618 619 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 620 wr32(IGC_TDH(reg_idx), 0); 621 writel(0, ring->tail); 622 623 txdctl |= IGC_TX_PTHRESH; 624 txdctl |= IGC_TX_HTHRESH << 8; 625 txdctl |= IGC_TX_WTHRESH << 16; 626 627 txdctl |= IGC_TXDCTL_QUEUE_ENABLE; 628 wr32(IGC_TXDCTL(reg_idx), txdctl); 629 } 630 631 /** 632 * igc_configure_tx - Configure transmit Unit after Reset 633 * @adapter: board private structure 634 * 635 * Configure the Tx unit of the MAC after a reset. 636 */ 637 static void igc_configure_tx(struct igc_adapter *adapter) 638 { 639 int i; 640 641 for (i = 0; i < adapter->num_tx_queues; i++) 642 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 643 } 644 645 /** 646 * igc_setup_mrqc - configure the multiple receive queue control registers 647 * @adapter: Board private structure 648 */ 649 static void igc_setup_mrqc(struct igc_adapter *adapter) 650 { 651 struct igc_hw *hw = &adapter->hw; 652 u32 j, num_rx_queues; 653 u32 mrqc, rxcsum; 654 u32 rss_key[10]; 655 656 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 657 for (j = 0; j < 10; j++) 658 wr32(IGC_RSSRK(j), rss_key[j]); 659 660 num_rx_queues = adapter->rss_queues; 661 662 if (adapter->rss_indir_tbl_init != num_rx_queues) { 663 for (j = 0; j < IGC_RETA_SIZE; j++) 664 adapter->rss_indir_tbl[j] = 665 (j * num_rx_queues) / IGC_RETA_SIZE; 666 adapter->rss_indir_tbl_init = num_rx_queues; 667 } 668 igc_write_rss_indir_tbl(adapter); 669 670 /* Disable raw packet checksumming so that RSS hash is placed in 671 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 672 * offloads as they are enabled by default 673 */ 674 rxcsum = rd32(IGC_RXCSUM); 675 rxcsum |= IGC_RXCSUM_PCSD; 676 677 /* Enable Receive Checksum Offload for SCTP */ 678 rxcsum |= IGC_RXCSUM_CRCOFL; 679 680 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 681 wr32(IGC_RXCSUM, rxcsum); 682 683 /* Generate RSS hash based on packet types, TCP/UDP 684 * port numbers and/or IPv4/v6 src and dst addresses 685 */ 686 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 687 IGC_MRQC_RSS_FIELD_IPV4_TCP | 688 IGC_MRQC_RSS_FIELD_IPV6 | 689 IGC_MRQC_RSS_FIELD_IPV6_TCP | 690 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 691 692 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 693 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 694 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 695 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 696 697 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 698 699 wr32(IGC_MRQC, mrqc); 700 } 701 702 /** 703 * igc_setup_rctl - configure the receive control registers 704 * @adapter: Board private structure 705 */ 706 static void igc_setup_rctl(struct igc_adapter *adapter) 707 { 708 struct igc_hw *hw = &adapter->hw; 709 u32 rctl; 710 711 rctl = rd32(IGC_RCTL); 712 713 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 714 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 715 716 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 717 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 718 719 /* enable stripping of CRC. Newer features require 720 * that the HW strips the CRC. 721 */ 722 rctl |= IGC_RCTL_SECRC; 723 724 /* disable store bad packets and clear size bits. */ 725 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 726 727 /* enable LPE to allow for reception of jumbo frames */ 728 rctl |= IGC_RCTL_LPE; 729 730 /* disable queue 0 to prevent tail write w/o re-config */ 731 wr32(IGC_RXDCTL(0), 0); 732 733 /* This is useful for sniffing bad packets. */ 734 if (adapter->netdev->features & NETIF_F_RXALL) { 735 /* UPE and MPE will be handled by normal PROMISC logic 736 * in set_rx_mode 737 */ 738 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 739 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 740 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 741 742 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 743 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 744 } 745 746 wr32(IGC_RCTL, rctl); 747 } 748 749 /** 750 * igc_setup_tctl - configure the transmit control registers 751 * @adapter: Board private structure 752 */ 753 static void igc_setup_tctl(struct igc_adapter *adapter) 754 { 755 struct igc_hw *hw = &adapter->hw; 756 u32 tctl; 757 758 /* disable queue 0 which icould be enabled by default */ 759 wr32(IGC_TXDCTL(0), 0); 760 761 /* Program the Transmit Control Register */ 762 tctl = rd32(IGC_TCTL); 763 tctl &= ~IGC_TCTL_CT; 764 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 765 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 766 767 /* Enable transmits */ 768 tctl |= IGC_TCTL_EN; 769 770 wr32(IGC_TCTL, tctl); 771 } 772 773 /** 774 * igc_set_mac - Change the Ethernet Address of the NIC 775 * @netdev: network interface device structure 776 * @p: pointer to an address structure 777 * 778 * Returns 0 on success, negative on failure 779 */ 780 static int igc_set_mac(struct net_device *netdev, void *p) 781 { 782 struct igc_adapter *adapter = netdev_priv(netdev); 783 struct igc_hw *hw = &adapter->hw; 784 struct sockaddr *addr = p; 785 786 if (!is_valid_ether_addr(addr->sa_data)) 787 return -EADDRNOTAVAIL; 788 789 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 790 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 791 792 /* set the correct pool for the new PF MAC address in entry 0 */ 793 igc_set_default_mac_filter(adapter); 794 795 return 0; 796 } 797 798 /** 799 * igc_write_mc_addr_list - write multicast addresses to MTA 800 * @netdev: network interface device structure 801 * 802 * Writes multicast address list to the MTA hash table. 803 * Returns: -ENOMEM on failure 804 * 0 on no addresses written 805 * X on writing X addresses to MTA 806 **/ 807 static int igc_write_mc_addr_list(struct net_device *netdev) 808 { 809 struct igc_adapter *adapter = netdev_priv(netdev); 810 struct igc_hw *hw = &adapter->hw; 811 struct netdev_hw_addr *ha; 812 u8 *mta_list; 813 int i; 814 815 if (netdev_mc_empty(netdev)) { 816 /* nothing to program, so clear mc list */ 817 igc_update_mc_addr_list(hw, NULL, 0); 818 return 0; 819 } 820 821 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 822 if (!mta_list) 823 return -ENOMEM; 824 825 /* The shared function expects a packed array of only addresses. */ 826 i = 0; 827 netdev_for_each_mc_addr(ha, netdev) 828 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 829 830 igc_update_mc_addr_list(hw, mta_list, i); 831 kfree(mta_list); 832 833 return netdev_mc_count(netdev); 834 } 835 836 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, 837 struct igc_tx_buffer *first, 838 u32 vlan_macip_lens, u32 type_tucmd, 839 u32 mss_l4len_idx) 840 { 841 struct igc_adv_tx_context_desc *context_desc; 842 u16 i = tx_ring->next_to_use; 843 struct timespec64 ts; 844 845 context_desc = IGC_TX_CTXTDESC(tx_ring, i); 846 847 i++; 848 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 849 850 /* set bits to identify this as an advanced context descriptor */ 851 type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 852 853 /* For 82575, context index must be unique per ring. */ 854 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 855 mss_l4len_idx |= tx_ring->reg_idx << 4; 856 857 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 858 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 859 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 860 861 /* We assume there is always a valid Tx time available. Invalid times 862 * should have been handled by the upper layers. 863 */ 864 if (tx_ring->launchtime_enable) { 865 ts = ktime_to_timespec64(first->skb->tstamp); 866 first->skb->tstamp = ktime_set(0, 0); 867 context_desc->launch_time = cpu_to_le32(ts.tv_nsec / 32); 868 } else { 869 context_desc->launch_time = 0; 870 } 871 } 872 873 static inline bool igc_ipv6_csum_is_sctp(struct sk_buff *skb) 874 { 875 unsigned int offset = 0; 876 877 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 878 879 return offset == skb_checksum_start_offset(skb); 880 } 881 882 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first) 883 { 884 struct sk_buff *skb = first->skb; 885 u32 vlan_macip_lens = 0; 886 u32 type_tucmd = 0; 887 888 if (skb->ip_summed != CHECKSUM_PARTIAL) { 889 csum_failed: 890 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && 891 !tx_ring->launchtime_enable) 892 return; 893 goto no_csum; 894 } 895 896 switch (skb->csum_offset) { 897 case offsetof(struct tcphdr, check): 898 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 899 /* fall through */ 900 case offsetof(struct udphdr, check): 901 break; 902 case offsetof(struct sctphdr, checksum): 903 /* validate that this is actually an SCTP request */ 904 if ((first->protocol == htons(ETH_P_IP) && 905 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 906 (first->protocol == htons(ETH_P_IPV6) && 907 igc_ipv6_csum_is_sctp(skb))) { 908 type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; 909 break; 910 } 911 /* fall through */ 912 default: 913 skb_checksum_help(skb); 914 goto csum_failed; 915 } 916 917 /* update TX checksum flag */ 918 first->tx_flags |= IGC_TX_FLAGS_CSUM; 919 vlan_macip_lens = skb_checksum_start_offset(skb) - 920 skb_network_offset(skb); 921 no_csum: 922 vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; 923 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 924 925 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0); 926 } 927 928 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 929 { 930 struct net_device *netdev = tx_ring->netdev; 931 932 netif_stop_subqueue(netdev, tx_ring->queue_index); 933 934 /* memory barriier comment */ 935 smp_mb(); 936 937 /* We need to check again in a case another CPU has just 938 * made room available. 939 */ 940 if (igc_desc_unused(tx_ring) < size) 941 return -EBUSY; 942 943 /* A reprieve! */ 944 netif_wake_subqueue(netdev, tx_ring->queue_index); 945 946 u64_stats_update_begin(&tx_ring->tx_syncp2); 947 tx_ring->tx_stats.restart_queue2++; 948 u64_stats_update_end(&tx_ring->tx_syncp2); 949 950 return 0; 951 } 952 953 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 954 { 955 if (igc_desc_unused(tx_ring) >= size) 956 return 0; 957 return __igc_maybe_stop_tx(tx_ring, size); 958 } 959 960 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 961 { 962 /* set type for advanced descriptor with frame checksum insertion */ 963 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 964 IGC_ADVTXD_DCMD_DEXT | 965 IGC_ADVTXD_DCMD_IFCS; 966 967 return cmd_type; 968 } 969 970 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 971 union igc_adv_tx_desc *tx_desc, 972 u32 tx_flags, unsigned int paylen) 973 { 974 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 975 976 /* insert L4 checksum */ 977 olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) * 978 ((IGC_TXD_POPTS_TXSM << 8) / 979 IGC_TX_FLAGS_CSUM); 980 981 /* insert IPv4 checksum */ 982 olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) * 983 (((IGC_TXD_POPTS_IXSM << 8)) / 984 IGC_TX_FLAGS_IPV4); 985 986 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 987 } 988 989 static int igc_tx_map(struct igc_ring *tx_ring, 990 struct igc_tx_buffer *first, 991 const u8 hdr_len) 992 { 993 struct sk_buff *skb = first->skb; 994 struct igc_tx_buffer *tx_buffer; 995 union igc_adv_tx_desc *tx_desc; 996 u32 tx_flags = first->tx_flags; 997 skb_frag_t *frag; 998 u16 i = tx_ring->next_to_use; 999 unsigned int data_len, size; 1000 dma_addr_t dma; 1001 u32 cmd_type = igc_tx_cmd_type(skb, tx_flags); 1002 1003 tx_desc = IGC_TX_DESC(tx_ring, i); 1004 1005 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 1006 1007 size = skb_headlen(skb); 1008 data_len = skb->data_len; 1009 1010 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1011 1012 tx_buffer = first; 1013 1014 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1015 if (dma_mapping_error(tx_ring->dev, dma)) 1016 goto dma_error; 1017 1018 /* record length, and DMA address */ 1019 dma_unmap_len_set(tx_buffer, len, size); 1020 dma_unmap_addr_set(tx_buffer, dma, dma); 1021 1022 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1023 1024 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 1025 tx_desc->read.cmd_type_len = 1026 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 1027 1028 i++; 1029 tx_desc++; 1030 if (i == tx_ring->count) { 1031 tx_desc = IGC_TX_DESC(tx_ring, 0); 1032 i = 0; 1033 } 1034 tx_desc->read.olinfo_status = 0; 1035 1036 dma += IGC_MAX_DATA_PER_TXD; 1037 size -= IGC_MAX_DATA_PER_TXD; 1038 1039 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1040 } 1041 1042 if (likely(!data_len)) 1043 break; 1044 1045 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 1046 1047 i++; 1048 tx_desc++; 1049 if (i == tx_ring->count) { 1050 tx_desc = IGC_TX_DESC(tx_ring, 0); 1051 i = 0; 1052 } 1053 tx_desc->read.olinfo_status = 0; 1054 1055 size = skb_frag_size(frag); 1056 data_len -= size; 1057 1058 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 1059 size, DMA_TO_DEVICE); 1060 1061 tx_buffer = &tx_ring->tx_buffer_info[i]; 1062 } 1063 1064 /* write last descriptor with RS and EOP bits */ 1065 cmd_type |= size | IGC_TXD_DCMD; 1066 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1067 1068 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1069 1070 /* set the timestamp */ 1071 first->time_stamp = jiffies; 1072 1073 skb_tx_timestamp(skb); 1074 1075 /* Force memory writes to complete before letting h/w know there 1076 * are new descriptors to fetch. (Only applicable for weak-ordered 1077 * memory model archs, such as IA-64). 1078 * 1079 * We also need this memory barrier to make certain all of the 1080 * status bits have been updated before next_to_watch is written. 1081 */ 1082 wmb(); 1083 1084 /* set next_to_watch value indicating a packet is present */ 1085 first->next_to_watch = tx_desc; 1086 1087 i++; 1088 if (i == tx_ring->count) 1089 i = 0; 1090 1091 tx_ring->next_to_use = i; 1092 1093 /* Make sure there is space in the ring for the next send. */ 1094 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 1095 1096 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1097 writel(i, tx_ring->tail); 1098 } 1099 1100 return 0; 1101 dma_error: 1102 dev_err(tx_ring->dev, "TX DMA map failed\n"); 1103 tx_buffer = &tx_ring->tx_buffer_info[i]; 1104 1105 /* clear dma mappings for failed tx_buffer_info map */ 1106 while (tx_buffer != first) { 1107 if (dma_unmap_len(tx_buffer, len)) 1108 dma_unmap_page(tx_ring->dev, 1109 dma_unmap_addr(tx_buffer, dma), 1110 dma_unmap_len(tx_buffer, len), 1111 DMA_TO_DEVICE); 1112 dma_unmap_len_set(tx_buffer, len, 0); 1113 1114 if (i-- == 0) 1115 i += tx_ring->count; 1116 tx_buffer = &tx_ring->tx_buffer_info[i]; 1117 } 1118 1119 if (dma_unmap_len(tx_buffer, len)) 1120 dma_unmap_single(tx_ring->dev, 1121 dma_unmap_addr(tx_buffer, dma), 1122 dma_unmap_len(tx_buffer, len), 1123 DMA_TO_DEVICE); 1124 dma_unmap_len_set(tx_buffer, len, 0); 1125 1126 dev_kfree_skb_any(tx_buffer->skb); 1127 tx_buffer->skb = NULL; 1128 1129 tx_ring->next_to_use = i; 1130 1131 return -1; 1132 } 1133 1134 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1135 struct igc_ring *tx_ring) 1136 { 1137 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1138 __be16 protocol = vlan_get_protocol(skb); 1139 struct igc_tx_buffer *first; 1140 u32 tx_flags = 0; 1141 unsigned short f; 1142 u8 hdr_len = 0; 1143 1144 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1145 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1146 * + 2 desc gap to keep tail from touching head, 1147 * + 1 desc for context descriptor, 1148 * otherwise try next time 1149 */ 1150 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1151 count += TXD_USE_COUNT(skb_frag_size( 1152 &skb_shinfo(skb)->frags[f])); 1153 1154 if (igc_maybe_stop_tx(tx_ring, count + 3)) { 1155 /* this is a hard error */ 1156 return NETDEV_TX_BUSY; 1157 } 1158 1159 /* record the location of the first descriptor for this packet */ 1160 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1161 first->skb = skb; 1162 first->bytecount = skb->len; 1163 first->gso_segs = 1; 1164 1165 /* record initial flags and protocol */ 1166 first->tx_flags = tx_flags; 1167 first->protocol = protocol; 1168 1169 igc_tx_csum(tx_ring, first); 1170 1171 igc_tx_map(tx_ring, first, hdr_len); 1172 1173 return NETDEV_TX_OK; 1174 } 1175 1176 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1177 struct sk_buff *skb) 1178 { 1179 unsigned int r_idx = skb->queue_mapping; 1180 1181 if (r_idx >= adapter->num_tx_queues) 1182 r_idx = r_idx % adapter->num_tx_queues; 1183 1184 return adapter->tx_ring[r_idx]; 1185 } 1186 1187 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1188 struct net_device *netdev) 1189 { 1190 struct igc_adapter *adapter = netdev_priv(netdev); 1191 1192 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1193 * in order to meet this minimum size requirement. 1194 */ 1195 if (skb->len < 17) { 1196 if (skb_padto(skb, 17)) 1197 return NETDEV_TX_OK; 1198 skb->len = 17; 1199 } 1200 1201 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1202 } 1203 1204 static void igc_rx_checksum(struct igc_ring *ring, 1205 union igc_adv_rx_desc *rx_desc, 1206 struct sk_buff *skb) 1207 { 1208 skb_checksum_none_assert(skb); 1209 1210 /* Ignore Checksum bit is set */ 1211 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) 1212 return; 1213 1214 /* Rx checksum disabled via ethtool */ 1215 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1216 return; 1217 1218 /* TCP/UDP checksum error bit is set */ 1219 if (igc_test_staterr(rx_desc, 1220 IGC_RXDEXT_STATERR_TCPE | 1221 IGC_RXDEXT_STATERR_IPE)) { 1222 /* work around errata with sctp packets where the TCPE aka 1223 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 1224 * packets (aka let the stack check the crc32c) 1225 */ 1226 if (!(skb->len == 60 && 1227 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 1228 u64_stats_update_begin(&ring->rx_syncp); 1229 ring->rx_stats.csum_err++; 1230 u64_stats_update_end(&ring->rx_syncp); 1231 } 1232 /* let the stack verify checksum errors */ 1233 return; 1234 } 1235 /* It must be a TCP or UDP packet with a valid checksum */ 1236 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | 1237 IGC_RXD_STAT_UDPCS)) 1238 skb->ip_summed = CHECKSUM_UNNECESSARY; 1239 1240 dev_dbg(ring->dev, "cksum success: bits %08X\n", 1241 le32_to_cpu(rx_desc->wb.upper.status_error)); 1242 } 1243 1244 static inline void igc_rx_hash(struct igc_ring *ring, 1245 union igc_adv_rx_desc *rx_desc, 1246 struct sk_buff *skb) 1247 { 1248 if (ring->netdev->features & NETIF_F_RXHASH) 1249 skb_set_hash(skb, 1250 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 1251 PKT_HASH_TYPE_L3); 1252 } 1253 1254 /** 1255 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1256 * @rx_ring: rx descriptor ring packet is being transacted on 1257 * @rx_desc: pointer to the EOP Rx descriptor 1258 * @skb: pointer to current skb being populated 1259 * 1260 * This function checks the ring, descriptor, and packet information in 1261 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 1262 * other fields within the skb. 1263 */ 1264 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1265 union igc_adv_rx_desc *rx_desc, 1266 struct sk_buff *skb) 1267 { 1268 igc_rx_hash(rx_ring, rx_desc, skb); 1269 1270 igc_rx_checksum(rx_ring, rx_desc, skb); 1271 1272 skb_record_rx_queue(skb, rx_ring->queue_index); 1273 1274 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1275 } 1276 1277 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1278 const unsigned int size) 1279 { 1280 struct igc_rx_buffer *rx_buffer; 1281 1282 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1283 prefetchw(rx_buffer->page); 1284 1285 /* we are reusing so sync this buffer for CPU use */ 1286 dma_sync_single_range_for_cpu(rx_ring->dev, 1287 rx_buffer->dma, 1288 rx_buffer->page_offset, 1289 size, 1290 DMA_FROM_DEVICE); 1291 1292 rx_buffer->pagecnt_bias--; 1293 1294 return rx_buffer; 1295 } 1296 1297 /** 1298 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1299 * @rx_ring: rx descriptor ring to transact packets on 1300 * @rx_buffer: buffer containing page to add 1301 * @skb: sk_buff to place the data into 1302 * @size: size of buffer to be added 1303 * 1304 * This function will add the data contained in rx_buffer->page to the skb. 1305 */ 1306 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1307 struct igc_rx_buffer *rx_buffer, 1308 struct sk_buff *skb, 1309 unsigned int size) 1310 { 1311 #if (PAGE_SIZE < 8192) 1312 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1313 1314 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1315 rx_buffer->page_offset, size, truesize); 1316 rx_buffer->page_offset ^= truesize; 1317 #else 1318 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1319 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1320 SKB_DATA_ALIGN(size); 1321 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1322 rx_buffer->page_offset, size, truesize); 1323 rx_buffer->page_offset += truesize; 1324 #endif 1325 } 1326 1327 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1328 struct igc_rx_buffer *rx_buffer, 1329 union igc_adv_rx_desc *rx_desc, 1330 unsigned int size) 1331 { 1332 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1333 #if (PAGE_SIZE < 8192) 1334 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1335 #else 1336 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1337 SKB_DATA_ALIGN(IGC_SKB_PAD + size); 1338 #endif 1339 struct sk_buff *skb; 1340 1341 /* prefetch first cache line of first page */ 1342 prefetch(va); 1343 #if L1_CACHE_BYTES < 128 1344 prefetch(va + L1_CACHE_BYTES); 1345 #endif 1346 1347 /* build an skb around the page buffer */ 1348 skb = build_skb(va - IGC_SKB_PAD, truesize); 1349 if (unlikely(!skb)) 1350 return NULL; 1351 1352 /* update pointers within the skb to store the data */ 1353 skb_reserve(skb, IGC_SKB_PAD); 1354 __skb_put(skb, size); 1355 1356 /* update buffer offset */ 1357 #if (PAGE_SIZE < 8192) 1358 rx_buffer->page_offset ^= truesize; 1359 #else 1360 rx_buffer->page_offset += truesize; 1361 #endif 1362 1363 return skb; 1364 } 1365 1366 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1367 struct igc_rx_buffer *rx_buffer, 1368 union igc_adv_rx_desc *rx_desc, 1369 unsigned int size) 1370 { 1371 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1372 #if (PAGE_SIZE < 8192) 1373 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1374 #else 1375 unsigned int truesize = SKB_DATA_ALIGN(size); 1376 #endif 1377 unsigned int headlen; 1378 struct sk_buff *skb; 1379 1380 /* prefetch first cache line of first page */ 1381 prefetch(va); 1382 #if L1_CACHE_BYTES < 128 1383 prefetch(va + L1_CACHE_BYTES); 1384 #endif 1385 1386 /* allocate a skb to store the frags */ 1387 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); 1388 if (unlikely(!skb)) 1389 return NULL; 1390 1391 /* Determine available headroom for copy */ 1392 headlen = size; 1393 if (headlen > IGC_RX_HDR_LEN) 1394 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 1395 1396 /* align pull length to size of long to optimize memcpy performance */ 1397 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 1398 1399 /* update all of the pointers */ 1400 size -= headlen; 1401 if (size) { 1402 skb_add_rx_frag(skb, 0, rx_buffer->page, 1403 (va + headlen) - page_address(rx_buffer->page), 1404 size, truesize); 1405 #if (PAGE_SIZE < 8192) 1406 rx_buffer->page_offset ^= truesize; 1407 #else 1408 rx_buffer->page_offset += truesize; 1409 #endif 1410 } else { 1411 rx_buffer->pagecnt_bias++; 1412 } 1413 1414 return skb; 1415 } 1416 1417 /** 1418 * igc_reuse_rx_page - page flip buffer and store it back on the ring 1419 * @rx_ring: rx descriptor ring to store buffers on 1420 * @old_buff: donor buffer to have page reused 1421 * 1422 * Synchronizes page for reuse by the adapter 1423 */ 1424 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 1425 struct igc_rx_buffer *old_buff) 1426 { 1427 u16 nta = rx_ring->next_to_alloc; 1428 struct igc_rx_buffer *new_buff; 1429 1430 new_buff = &rx_ring->rx_buffer_info[nta]; 1431 1432 /* update, and store next to alloc */ 1433 nta++; 1434 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1435 1436 /* Transfer page from old buffer to new buffer. 1437 * Move each member individually to avoid possible store 1438 * forwarding stalls. 1439 */ 1440 new_buff->dma = old_buff->dma; 1441 new_buff->page = old_buff->page; 1442 new_buff->page_offset = old_buff->page_offset; 1443 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1444 } 1445 1446 static inline bool igc_page_is_reserved(struct page *page) 1447 { 1448 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 1449 } 1450 1451 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) 1452 { 1453 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1454 struct page *page = rx_buffer->page; 1455 1456 /* avoid re-using remote pages */ 1457 if (unlikely(igc_page_is_reserved(page))) 1458 return false; 1459 1460 #if (PAGE_SIZE < 8192) 1461 /* if we are only owner of page we can reuse it */ 1462 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 1463 return false; 1464 #else 1465 #define IGC_LAST_OFFSET \ 1466 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 1467 1468 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 1469 return false; 1470 #endif 1471 1472 /* If we have drained the page fragment pool we need to update 1473 * the pagecnt_bias and page count so that we fully restock the 1474 * number of references the driver holds. 1475 */ 1476 if (unlikely(!pagecnt_bias)) { 1477 page_ref_add(page, USHRT_MAX); 1478 rx_buffer->pagecnt_bias = USHRT_MAX; 1479 } 1480 1481 return true; 1482 } 1483 1484 /** 1485 * igc_is_non_eop - process handling of non-EOP buffers 1486 * @rx_ring: Rx ring being processed 1487 * @rx_desc: Rx descriptor for current buffer 1488 * @skb: current socket buffer containing buffer in progress 1489 * 1490 * This function updates next to clean. If the buffer is an EOP buffer 1491 * this function exits returning false, otherwise it will place the 1492 * sk_buff in the next buffer to be chained and return true indicating 1493 * that this is in fact a non-EOP buffer. 1494 */ 1495 static bool igc_is_non_eop(struct igc_ring *rx_ring, 1496 union igc_adv_rx_desc *rx_desc) 1497 { 1498 u32 ntc = rx_ring->next_to_clean + 1; 1499 1500 /* fetch, update, and store next to clean */ 1501 ntc = (ntc < rx_ring->count) ? ntc : 0; 1502 rx_ring->next_to_clean = ntc; 1503 1504 prefetch(IGC_RX_DESC(rx_ring, ntc)); 1505 1506 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 1507 return false; 1508 1509 return true; 1510 } 1511 1512 /** 1513 * igc_cleanup_headers - Correct corrupted or empty headers 1514 * @rx_ring: rx descriptor ring packet is being transacted on 1515 * @rx_desc: pointer to the EOP Rx descriptor 1516 * @skb: pointer to current skb being fixed 1517 * 1518 * Address the case where we are pulling data in on pages only 1519 * and as such no data is present in the skb header. 1520 * 1521 * In addition if skb is not at least 60 bytes we need to pad it so that 1522 * it is large enough to qualify as a valid Ethernet frame. 1523 * 1524 * Returns true if an error was encountered and skb was freed. 1525 */ 1526 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 1527 union igc_adv_rx_desc *rx_desc, 1528 struct sk_buff *skb) 1529 { 1530 if (unlikely((igc_test_staterr(rx_desc, 1531 IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) { 1532 struct net_device *netdev = rx_ring->netdev; 1533 1534 if (!(netdev->features & NETIF_F_RXALL)) { 1535 dev_kfree_skb_any(skb); 1536 return true; 1537 } 1538 } 1539 1540 /* if eth_skb_pad returns an error the skb was freed */ 1541 if (eth_skb_pad(skb)) 1542 return true; 1543 1544 return false; 1545 } 1546 1547 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 1548 struct igc_rx_buffer *rx_buffer) 1549 { 1550 if (igc_can_reuse_rx_page(rx_buffer)) { 1551 /* hand second half of page back to the ring */ 1552 igc_reuse_rx_page(rx_ring, rx_buffer); 1553 } else { 1554 /* We are not reusing the buffer so unmap it and free 1555 * any references we are holding to it 1556 */ 1557 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1558 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 1559 IGC_RX_DMA_ATTR); 1560 __page_frag_cache_drain(rx_buffer->page, 1561 rx_buffer->pagecnt_bias); 1562 } 1563 1564 /* clear contents of rx_buffer */ 1565 rx_buffer->page = NULL; 1566 } 1567 1568 /** 1569 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 1570 * @adapter: address of board private structure 1571 */ 1572 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 1573 { 1574 union igc_adv_rx_desc *rx_desc; 1575 u16 i = rx_ring->next_to_use; 1576 struct igc_rx_buffer *bi; 1577 u16 bufsz; 1578 1579 /* nothing to do */ 1580 if (!cleaned_count) 1581 return; 1582 1583 rx_desc = IGC_RX_DESC(rx_ring, i); 1584 bi = &rx_ring->rx_buffer_info[i]; 1585 i -= rx_ring->count; 1586 1587 bufsz = igc_rx_bufsz(rx_ring); 1588 1589 do { 1590 if (!igc_alloc_mapped_page(rx_ring, bi)) 1591 break; 1592 1593 /* sync the buffer for use by the device */ 1594 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1595 bi->page_offset, bufsz, 1596 DMA_FROM_DEVICE); 1597 1598 /* Refresh the desc even if buffer_addrs didn't change 1599 * because each write-back erases this info. 1600 */ 1601 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1602 1603 rx_desc++; 1604 bi++; 1605 i++; 1606 if (unlikely(!i)) { 1607 rx_desc = IGC_RX_DESC(rx_ring, 0); 1608 bi = rx_ring->rx_buffer_info; 1609 i -= rx_ring->count; 1610 } 1611 1612 /* clear the length for the next_to_use descriptor */ 1613 rx_desc->wb.upper.length = 0; 1614 1615 cleaned_count--; 1616 } while (cleaned_count); 1617 1618 i += rx_ring->count; 1619 1620 if (rx_ring->next_to_use != i) { 1621 /* record the next descriptor to use */ 1622 rx_ring->next_to_use = i; 1623 1624 /* update next to alloc since we have filled the ring */ 1625 rx_ring->next_to_alloc = i; 1626 1627 /* Force memory writes to complete before letting h/w 1628 * know there are new descriptors to fetch. (Only 1629 * applicable for weak-ordered memory model archs, 1630 * such as IA-64). 1631 */ 1632 wmb(); 1633 writel(i, rx_ring->tail); 1634 } 1635 } 1636 1637 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 1638 { 1639 unsigned int total_bytes = 0, total_packets = 0; 1640 struct igc_ring *rx_ring = q_vector->rx.ring; 1641 struct sk_buff *skb = rx_ring->skb; 1642 u16 cleaned_count = igc_desc_unused(rx_ring); 1643 1644 while (likely(total_packets < budget)) { 1645 union igc_adv_rx_desc *rx_desc; 1646 struct igc_rx_buffer *rx_buffer; 1647 unsigned int size; 1648 1649 /* return some buffers to hardware, one at a time is too slow */ 1650 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 1651 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1652 cleaned_count = 0; 1653 } 1654 1655 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 1656 size = le16_to_cpu(rx_desc->wb.upper.length); 1657 if (!size) 1658 break; 1659 1660 /* This memory barrier is needed to keep us from reading 1661 * any other fields out of the rx_desc until we know the 1662 * descriptor has been written back 1663 */ 1664 dma_rmb(); 1665 1666 rx_buffer = igc_get_rx_buffer(rx_ring, size); 1667 1668 /* retrieve a buffer from the ring */ 1669 if (skb) 1670 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 1671 else if (ring_uses_build_skb(rx_ring)) 1672 skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); 1673 else 1674 skb = igc_construct_skb(rx_ring, rx_buffer, 1675 rx_desc, size); 1676 1677 /* exit if we failed to retrieve a buffer */ 1678 if (!skb) { 1679 rx_ring->rx_stats.alloc_failed++; 1680 rx_buffer->pagecnt_bias++; 1681 break; 1682 } 1683 1684 igc_put_rx_buffer(rx_ring, rx_buffer); 1685 cleaned_count++; 1686 1687 /* fetch next buffer in frame if non-eop */ 1688 if (igc_is_non_eop(rx_ring, rx_desc)) 1689 continue; 1690 1691 /* verify the packet layout is correct */ 1692 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) { 1693 skb = NULL; 1694 continue; 1695 } 1696 1697 /* probably a little skewed due to removing CRC */ 1698 total_bytes += skb->len; 1699 1700 /* populate checksum, timestamp, VLAN, and protocol */ 1701 igc_process_skb_fields(rx_ring, rx_desc, skb); 1702 1703 napi_gro_receive(&q_vector->napi, skb); 1704 1705 /* reset skb pointer */ 1706 skb = NULL; 1707 1708 /* update budget accounting */ 1709 total_packets++; 1710 } 1711 1712 /* place incomplete frames back on ring for completion */ 1713 rx_ring->skb = skb; 1714 1715 u64_stats_update_begin(&rx_ring->rx_syncp); 1716 rx_ring->rx_stats.packets += total_packets; 1717 rx_ring->rx_stats.bytes += total_bytes; 1718 u64_stats_update_end(&rx_ring->rx_syncp); 1719 q_vector->rx.total_packets += total_packets; 1720 q_vector->rx.total_bytes += total_bytes; 1721 1722 if (cleaned_count) 1723 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1724 1725 return total_packets; 1726 } 1727 1728 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 1729 { 1730 return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; 1731 } 1732 1733 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 1734 struct igc_rx_buffer *bi) 1735 { 1736 struct page *page = bi->page; 1737 dma_addr_t dma; 1738 1739 /* since we are recycling buffers we should seldom need to alloc */ 1740 if (likely(page)) 1741 return true; 1742 1743 /* alloc new page for storage */ 1744 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 1745 if (unlikely(!page)) { 1746 rx_ring->rx_stats.alloc_failed++; 1747 return false; 1748 } 1749 1750 /* map page for use */ 1751 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1752 igc_rx_pg_size(rx_ring), 1753 DMA_FROM_DEVICE, 1754 IGC_RX_DMA_ATTR); 1755 1756 /* if mapping failed free memory back to system since 1757 * there isn't much point in holding memory we can't use 1758 */ 1759 if (dma_mapping_error(rx_ring->dev, dma)) { 1760 __free_page(page); 1761 1762 rx_ring->rx_stats.alloc_failed++; 1763 return false; 1764 } 1765 1766 bi->dma = dma; 1767 bi->page = page; 1768 bi->page_offset = igc_rx_offset(rx_ring); 1769 bi->pagecnt_bias = 1; 1770 1771 return true; 1772 } 1773 1774 /** 1775 * igc_clean_tx_irq - Reclaim resources after transmit completes 1776 * @q_vector: pointer to q_vector containing needed info 1777 * @napi_budget: Used to determine if we are in netpoll 1778 * 1779 * returns true if ring is completely cleaned 1780 */ 1781 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 1782 { 1783 struct igc_adapter *adapter = q_vector->adapter; 1784 unsigned int total_bytes = 0, total_packets = 0; 1785 unsigned int budget = q_vector->tx.work_limit; 1786 struct igc_ring *tx_ring = q_vector->tx.ring; 1787 unsigned int i = tx_ring->next_to_clean; 1788 struct igc_tx_buffer *tx_buffer; 1789 union igc_adv_tx_desc *tx_desc; 1790 1791 if (test_bit(__IGC_DOWN, &adapter->state)) 1792 return true; 1793 1794 tx_buffer = &tx_ring->tx_buffer_info[i]; 1795 tx_desc = IGC_TX_DESC(tx_ring, i); 1796 i -= tx_ring->count; 1797 1798 do { 1799 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 1800 1801 /* if next_to_watch is not set then there is no work pending */ 1802 if (!eop_desc) 1803 break; 1804 1805 /* prevent any other reads prior to eop_desc */ 1806 smp_rmb(); 1807 1808 /* if DD is not set pending work has not been completed */ 1809 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 1810 break; 1811 1812 /* clear next_to_watch to prevent false hangs */ 1813 tx_buffer->next_to_watch = NULL; 1814 1815 /* update the statistics for this packet */ 1816 total_bytes += tx_buffer->bytecount; 1817 total_packets += tx_buffer->gso_segs; 1818 1819 /* free the skb */ 1820 napi_consume_skb(tx_buffer->skb, napi_budget); 1821 1822 /* unmap skb header data */ 1823 dma_unmap_single(tx_ring->dev, 1824 dma_unmap_addr(tx_buffer, dma), 1825 dma_unmap_len(tx_buffer, len), 1826 DMA_TO_DEVICE); 1827 1828 /* clear tx_buffer data */ 1829 dma_unmap_len_set(tx_buffer, len, 0); 1830 1831 /* clear last DMA location and unmap remaining buffers */ 1832 while (tx_desc != eop_desc) { 1833 tx_buffer++; 1834 tx_desc++; 1835 i++; 1836 if (unlikely(!i)) { 1837 i -= tx_ring->count; 1838 tx_buffer = tx_ring->tx_buffer_info; 1839 tx_desc = IGC_TX_DESC(tx_ring, 0); 1840 } 1841 1842 /* unmap any remaining paged data */ 1843 if (dma_unmap_len(tx_buffer, len)) { 1844 dma_unmap_page(tx_ring->dev, 1845 dma_unmap_addr(tx_buffer, dma), 1846 dma_unmap_len(tx_buffer, len), 1847 DMA_TO_DEVICE); 1848 dma_unmap_len_set(tx_buffer, len, 0); 1849 } 1850 } 1851 1852 /* move us one more past the eop_desc for start of next pkt */ 1853 tx_buffer++; 1854 tx_desc++; 1855 i++; 1856 if (unlikely(!i)) { 1857 i -= tx_ring->count; 1858 tx_buffer = tx_ring->tx_buffer_info; 1859 tx_desc = IGC_TX_DESC(tx_ring, 0); 1860 } 1861 1862 /* issue prefetch for next Tx descriptor */ 1863 prefetch(tx_desc); 1864 1865 /* update budget accounting */ 1866 budget--; 1867 } while (likely(budget)); 1868 1869 netdev_tx_completed_queue(txring_txq(tx_ring), 1870 total_packets, total_bytes); 1871 1872 i += tx_ring->count; 1873 tx_ring->next_to_clean = i; 1874 u64_stats_update_begin(&tx_ring->tx_syncp); 1875 tx_ring->tx_stats.bytes += total_bytes; 1876 tx_ring->tx_stats.packets += total_packets; 1877 u64_stats_update_end(&tx_ring->tx_syncp); 1878 q_vector->tx.total_bytes += total_bytes; 1879 q_vector->tx.total_packets += total_packets; 1880 1881 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 1882 struct igc_hw *hw = &adapter->hw; 1883 1884 /* Detect a transmit hang in hardware, this serializes the 1885 * check with the clearing of time_stamp and movement of i 1886 */ 1887 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 1888 if (tx_buffer->next_to_watch && 1889 time_after(jiffies, tx_buffer->time_stamp + 1890 (adapter->tx_timeout_factor * HZ)) && 1891 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { 1892 /* detected Tx unit hang */ 1893 dev_err(tx_ring->dev, 1894 "Detected Tx Unit Hang\n" 1895 " Tx Queue <%d>\n" 1896 " TDH <%x>\n" 1897 " TDT <%x>\n" 1898 " next_to_use <%x>\n" 1899 " next_to_clean <%x>\n" 1900 "buffer_info[next_to_clean]\n" 1901 " time_stamp <%lx>\n" 1902 " next_to_watch <%p>\n" 1903 " jiffies <%lx>\n" 1904 " desc.status <%x>\n", 1905 tx_ring->queue_index, 1906 rd32(IGC_TDH(tx_ring->reg_idx)), 1907 readl(tx_ring->tail), 1908 tx_ring->next_to_use, 1909 tx_ring->next_to_clean, 1910 tx_buffer->time_stamp, 1911 tx_buffer->next_to_watch, 1912 jiffies, 1913 tx_buffer->next_to_watch->wb.status); 1914 netif_stop_subqueue(tx_ring->netdev, 1915 tx_ring->queue_index); 1916 1917 /* we are about to reset, no point in enabling stuff */ 1918 return true; 1919 } 1920 } 1921 1922 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 1923 if (unlikely(total_packets && 1924 netif_carrier_ok(tx_ring->netdev) && 1925 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 1926 /* Make sure that anybody stopping the queue after this 1927 * sees the new next_to_clean. 1928 */ 1929 smp_mb(); 1930 if (__netif_subqueue_stopped(tx_ring->netdev, 1931 tx_ring->queue_index) && 1932 !(test_bit(__IGC_DOWN, &adapter->state))) { 1933 netif_wake_subqueue(tx_ring->netdev, 1934 tx_ring->queue_index); 1935 1936 u64_stats_update_begin(&tx_ring->tx_syncp); 1937 tx_ring->tx_stats.restart_queue++; 1938 u64_stats_update_end(&tx_ring->tx_syncp); 1939 } 1940 } 1941 1942 return !!budget; 1943 } 1944 1945 /** 1946 * igc_up - Open the interface and prepare it to handle traffic 1947 * @adapter: board private structure 1948 */ 1949 void igc_up(struct igc_adapter *adapter) 1950 { 1951 struct igc_hw *hw = &adapter->hw; 1952 int i = 0; 1953 1954 /* hardware has been reset, we need to reload some things */ 1955 igc_configure(adapter); 1956 1957 clear_bit(__IGC_DOWN, &adapter->state); 1958 1959 for (i = 0; i < adapter->num_q_vectors; i++) 1960 napi_enable(&adapter->q_vector[i]->napi); 1961 1962 if (adapter->msix_entries) 1963 igc_configure_msix(adapter); 1964 else 1965 igc_assign_vector(adapter->q_vector[0], 0); 1966 1967 /* Clear any pending interrupts. */ 1968 rd32(IGC_ICR); 1969 igc_irq_enable(adapter); 1970 1971 netif_tx_start_all_queues(adapter->netdev); 1972 1973 /* start the watchdog. */ 1974 hw->mac.get_link_status = 1; 1975 schedule_work(&adapter->watchdog_task); 1976 } 1977 1978 /** 1979 * igc_update_stats - Update the board statistics counters 1980 * @adapter: board private structure 1981 */ 1982 void igc_update_stats(struct igc_adapter *adapter) 1983 { 1984 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 1985 struct pci_dev *pdev = adapter->pdev; 1986 struct igc_hw *hw = &adapter->hw; 1987 u64 _bytes, _packets; 1988 u64 bytes, packets; 1989 unsigned int start; 1990 u32 mpc; 1991 int i; 1992 1993 /* Prevent stats update while adapter is being reset, or if the pci 1994 * connection is down. 1995 */ 1996 if (adapter->link_speed == 0) 1997 return; 1998 if (pci_channel_offline(pdev)) 1999 return; 2000 2001 packets = 0; 2002 bytes = 0; 2003 2004 rcu_read_lock(); 2005 for (i = 0; i < adapter->num_rx_queues; i++) { 2006 struct igc_ring *ring = adapter->rx_ring[i]; 2007 u32 rqdpc = rd32(IGC_RQDPC(i)); 2008 2009 if (hw->mac.type >= igc_i225) 2010 wr32(IGC_RQDPC(i), 0); 2011 2012 if (rqdpc) { 2013 ring->rx_stats.drops += rqdpc; 2014 net_stats->rx_fifo_errors += rqdpc; 2015 } 2016 2017 do { 2018 start = u64_stats_fetch_begin_irq(&ring->rx_syncp); 2019 _bytes = ring->rx_stats.bytes; 2020 _packets = ring->rx_stats.packets; 2021 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); 2022 bytes += _bytes; 2023 packets += _packets; 2024 } 2025 2026 net_stats->rx_bytes = bytes; 2027 net_stats->rx_packets = packets; 2028 2029 packets = 0; 2030 bytes = 0; 2031 for (i = 0; i < adapter->num_tx_queues; i++) { 2032 struct igc_ring *ring = adapter->tx_ring[i]; 2033 2034 do { 2035 start = u64_stats_fetch_begin_irq(&ring->tx_syncp); 2036 _bytes = ring->tx_stats.bytes; 2037 _packets = ring->tx_stats.packets; 2038 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); 2039 bytes += _bytes; 2040 packets += _packets; 2041 } 2042 net_stats->tx_bytes = bytes; 2043 net_stats->tx_packets = packets; 2044 rcu_read_unlock(); 2045 2046 /* read stats registers */ 2047 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 2048 adapter->stats.gprc += rd32(IGC_GPRC); 2049 adapter->stats.gorc += rd32(IGC_GORCL); 2050 rd32(IGC_GORCH); /* clear GORCL */ 2051 adapter->stats.bprc += rd32(IGC_BPRC); 2052 adapter->stats.mprc += rd32(IGC_MPRC); 2053 adapter->stats.roc += rd32(IGC_ROC); 2054 2055 adapter->stats.prc64 += rd32(IGC_PRC64); 2056 adapter->stats.prc127 += rd32(IGC_PRC127); 2057 adapter->stats.prc255 += rd32(IGC_PRC255); 2058 adapter->stats.prc511 += rd32(IGC_PRC511); 2059 adapter->stats.prc1023 += rd32(IGC_PRC1023); 2060 adapter->stats.prc1522 += rd32(IGC_PRC1522); 2061 adapter->stats.symerrs += rd32(IGC_SYMERRS); 2062 adapter->stats.sec += rd32(IGC_SEC); 2063 2064 mpc = rd32(IGC_MPC); 2065 adapter->stats.mpc += mpc; 2066 net_stats->rx_fifo_errors += mpc; 2067 adapter->stats.scc += rd32(IGC_SCC); 2068 adapter->stats.ecol += rd32(IGC_ECOL); 2069 adapter->stats.mcc += rd32(IGC_MCC); 2070 adapter->stats.latecol += rd32(IGC_LATECOL); 2071 adapter->stats.dc += rd32(IGC_DC); 2072 adapter->stats.rlec += rd32(IGC_RLEC); 2073 adapter->stats.xonrxc += rd32(IGC_XONRXC); 2074 adapter->stats.xontxc += rd32(IGC_XONTXC); 2075 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 2076 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 2077 adapter->stats.fcruc += rd32(IGC_FCRUC); 2078 adapter->stats.gptc += rd32(IGC_GPTC); 2079 adapter->stats.gotc += rd32(IGC_GOTCL); 2080 rd32(IGC_GOTCH); /* clear GOTCL */ 2081 adapter->stats.rnbc += rd32(IGC_RNBC); 2082 adapter->stats.ruc += rd32(IGC_RUC); 2083 adapter->stats.rfc += rd32(IGC_RFC); 2084 adapter->stats.rjc += rd32(IGC_RJC); 2085 adapter->stats.tor += rd32(IGC_TORH); 2086 adapter->stats.tot += rd32(IGC_TOTH); 2087 adapter->stats.tpr += rd32(IGC_TPR); 2088 2089 adapter->stats.ptc64 += rd32(IGC_PTC64); 2090 adapter->stats.ptc127 += rd32(IGC_PTC127); 2091 adapter->stats.ptc255 += rd32(IGC_PTC255); 2092 adapter->stats.ptc511 += rd32(IGC_PTC511); 2093 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 2094 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 2095 2096 adapter->stats.mptc += rd32(IGC_MPTC); 2097 adapter->stats.bptc += rd32(IGC_BPTC); 2098 2099 adapter->stats.tpt += rd32(IGC_TPT); 2100 adapter->stats.colc += rd32(IGC_COLC); 2101 2102 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 2103 2104 adapter->stats.tsctc += rd32(IGC_TSCTC); 2105 adapter->stats.tsctfc += rd32(IGC_TSCTFC); 2106 2107 adapter->stats.iac += rd32(IGC_IAC); 2108 adapter->stats.icrxoc += rd32(IGC_ICRXOC); 2109 adapter->stats.icrxptc += rd32(IGC_ICRXPTC); 2110 adapter->stats.icrxatc += rd32(IGC_ICRXATC); 2111 adapter->stats.ictxptc += rd32(IGC_ICTXPTC); 2112 adapter->stats.ictxatc += rd32(IGC_ICTXATC); 2113 adapter->stats.ictxqec += rd32(IGC_ICTXQEC); 2114 adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC); 2115 adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC); 2116 2117 /* Fill out the OS statistics structure */ 2118 net_stats->multicast = adapter->stats.mprc; 2119 net_stats->collisions = adapter->stats.colc; 2120 2121 /* Rx Errors */ 2122 2123 /* RLEC on some newer hardware can be incorrect so build 2124 * our own version based on RUC and ROC 2125 */ 2126 net_stats->rx_errors = adapter->stats.rxerrc + 2127 adapter->stats.crcerrs + adapter->stats.algnerrc + 2128 adapter->stats.ruc + adapter->stats.roc + 2129 adapter->stats.cexterr; 2130 net_stats->rx_length_errors = adapter->stats.ruc + 2131 adapter->stats.roc; 2132 net_stats->rx_crc_errors = adapter->stats.crcerrs; 2133 net_stats->rx_frame_errors = adapter->stats.algnerrc; 2134 net_stats->rx_missed_errors = adapter->stats.mpc; 2135 2136 /* Tx Errors */ 2137 net_stats->tx_errors = adapter->stats.ecol + 2138 adapter->stats.latecol; 2139 net_stats->tx_aborted_errors = adapter->stats.ecol; 2140 net_stats->tx_window_errors = adapter->stats.latecol; 2141 net_stats->tx_carrier_errors = adapter->stats.tncrs; 2142 2143 /* Tx Dropped needs to be maintained elsewhere */ 2144 2145 /* Management Stats */ 2146 adapter->stats.mgptc += rd32(IGC_MGTPTC); 2147 adapter->stats.mgprc += rd32(IGC_MGTPRC); 2148 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 2149 } 2150 2151 static void igc_nfc_filter_exit(struct igc_adapter *adapter) 2152 { 2153 struct igc_nfc_filter *rule; 2154 2155 spin_lock(&adapter->nfc_lock); 2156 2157 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) 2158 igc_erase_filter(adapter, rule); 2159 2160 hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node) 2161 igc_erase_filter(adapter, rule); 2162 2163 spin_unlock(&adapter->nfc_lock); 2164 } 2165 2166 static void igc_nfc_filter_restore(struct igc_adapter *adapter) 2167 { 2168 struct igc_nfc_filter *rule; 2169 2170 spin_lock(&adapter->nfc_lock); 2171 2172 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) 2173 igc_add_filter(adapter, rule); 2174 2175 spin_unlock(&adapter->nfc_lock); 2176 } 2177 2178 /** 2179 * igc_down - Close the interface 2180 * @adapter: board private structure 2181 */ 2182 void igc_down(struct igc_adapter *adapter) 2183 { 2184 struct net_device *netdev = adapter->netdev; 2185 struct igc_hw *hw = &adapter->hw; 2186 u32 tctl, rctl; 2187 int i = 0; 2188 2189 set_bit(__IGC_DOWN, &adapter->state); 2190 2191 /* disable receives in the hardware */ 2192 rctl = rd32(IGC_RCTL); 2193 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 2194 /* flush and sleep below */ 2195 2196 igc_nfc_filter_exit(adapter); 2197 2198 /* set trans_start so we don't get spurious watchdogs during reset */ 2199 netif_trans_update(netdev); 2200 2201 netif_carrier_off(netdev); 2202 netif_tx_stop_all_queues(netdev); 2203 2204 /* disable transmits in the hardware */ 2205 tctl = rd32(IGC_TCTL); 2206 tctl &= ~IGC_TCTL_EN; 2207 wr32(IGC_TCTL, tctl); 2208 /* flush both disables and wait for them to finish */ 2209 wrfl(); 2210 usleep_range(10000, 20000); 2211 2212 igc_irq_disable(adapter); 2213 2214 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 2215 2216 for (i = 0; i < adapter->num_q_vectors; i++) { 2217 if (adapter->q_vector[i]) { 2218 napi_synchronize(&adapter->q_vector[i]->napi); 2219 napi_disable(&adapter->q_vector[i]->napi); 2220 } 2221 } 2222 2223 del_timer_sync(&adapter->watchdog_timer); 2224 del_timer_sync(&adapter->phy_info_timer); 2225 2226 /* record the stats before reset*/ 2227 spin_lock(&adapter->stats64_lock); 2228 igc_update_stats(adapter); 2229 spin_unlock(&adapter->stats64_lock); 2230 2231 adapter->link_speed = 0; 2232 adapter->link_duplex = 0; 2233 2234 if (!pci_channel_offline(adapter->pdev)) 2235 igc_reset(adapter); 2236 2237 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 2238 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 2239 2240 igc_clean_all_tx_rings(adapter); 2241 igc_clean_all_rx_rings(adapter); 2242 } 2243 2244 void igc_reinit_locked(struct igc_adapter *adapter) 2245 { 2246 WARN_ON(in_interrupt()); 2247 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 2248 usleep_range(1000, 2000); 2249 igc_down(adapter); 2250 igc_up(adapter); 2251 clear_bit(__IGC_RESETTING, &adapter->state); 2252 } 2253 2254 static void igc_reset_task(struct work_struct *work) 2255 { 2256 struct igc_adapter *adapter; 2257 2258 adapter = container_of(work, struct igc_adapter, reset_task); 2259 2260 netdev_err(adapter->netdev, "Reset adapter\n"); 2261 igc_reinit_locked(adapter); 2262 } 2263 2264 /** 2265 * igc_change_mtu - Change the Maximum Transfer Unit 2266 * @netdev: network interface device structure 2267 * @new_mtu: new value for maximum frame size 2268 * 2269 * Returns 0 on success, negative on failure 2270 */ 2271 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 2272 { 2273 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 2274 struct igc_adapter *adapter = netdev_priv(netdev); 2275 2276 /* adjust max frame to be at least the size of a standard frame */ 2277 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 2278 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 2279 2280 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 2281 usleep_range(1000, 2000); 2282 2283 /* igc_down has a dependency on max_frame_size */ 2284 adapter->max_frame_size = max_frame; 2285 2286 if (netif_running(netdev)) 2287 igc_down(adapter); 2288 2289 netdev_dbg(netdev, "changing MTU from %d to %d\n", 2290 netdev->mtu, new_mtu); 2291 netdev->mtu = new_mtu; 2292 2293 if (netif_running(netdev)) 2294 igc_up(adapter); 2295 else 2296 igc_reset(adapter); 2297 2298 clear_bit(__IGC_RESETTING, &adapter->state); 2299 2300 return 0; 2301 } 2302 2303 /** 2304 * igc_get_stats - Get System Network Statistics 2305 * @netdev: network interface device structure 2306 * 2307 * Returns the address of the device statistics structure. 2308 * The statistics are updated here and also from the timer callback. 2309 */ 2310 static struct net_device_stats *igc_get_stats(struct net_device *netdev) 2311 { 2312 struct igc_adapter *adapter = netdev_priv(netdev); 2313 2314 if (!test_bit(__IGC_RESETTING, &adapter->state)) 2315 igc_update_stats(adapter); 2316 2317 /* only return the current stats */ 2318 return &netdev->stats; 2319 } 2320 2321 static netdev_features_t igc_fix_features(struct net_device *netdev, 2322 netdev_features_t features) 2323 { 2324 /* Since there is no support for separate Rx/Tx vlan accel 2325 * enable/disable make sure Tx flag is always in same state as Rx. 2326 */ 2327 if (features & NETIF_F_HW_VLAN_CTAG_RX) 2328 features |= NETIF_F_HW_VLAN_CTAG_TX; 2329 else 2330 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 2331 2332 return features; 2333 } 2334 2335 static int igc_set_features(struct net_device *netdev, 2336 netdev_features_t features) 2337 { 2338 netdev_features_t changed = netdev->features ^ features; 2339 struct igc_adapter *adapter = netdev_priv(netdev); 2340 2341 /* Add VLAN support */ 2342 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 2343 return 0; 2344 2345 if (!(features & NETIF_F_NTUPLE)) { 2346 struct hlist_node *node2; 2347 struct igc_nfc_filter *rule; 2348 2349 spin_lock(&adapter->nfc_lock); 2350 hlist_for_each_entry_safe(rule, node2, 2351 &adapter->nfc_filter_list, nfc_node) { 2352 igc_erase_filter(adapter, rule); 2353 hlist_del(&rule->nfc_node); 2354 kfree(rule); 2355 } 2356 spin_unlock(&adapter->nfc_lock); 2357 adapter->nfc_filter_count = 0; 2358 } 2359 2360 netdev->features = features; 2361 2362 if (netif_running(netdev)) 2363 igc_reinit_locked(adapter); 2364 else 2365 igc_reset(adapter); 2366 2367 return 1; 2368 } 2369 2370 static netdev_features_t 2371 igc_features_check(struct sk_buff *skb, struct net_device *dev, 2372 netdev_features_t features) 2373 { 2374 unsigned int network_hdr_len, mac_hdr_len; 2375 2376 /* Make certain the headers can be described by a context descriptor */ 2377 mac_hdr_len = skb_network_header(skb) - skb->data; 2378 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 2379 return features & ~(NETIF_F_HW_CSUM | 2380 NETIF_F_SCTP_CRC | 2381 NETIF_F_HW_VLAN_CTAG_TX | 2382 NETIF_F_TSO | 2383 NETIF_F_TSO6); 2384 2385 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 2386 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 2387 return features & ~(NETIF_F_HW_CSUM | 2388 NETIF_F_SCTP_CRC | 2389 NETIF_F_TSO | 2390 NETIF_F_TSO6); 2391 2392 /* We can only support IPv4 TSO in tunnels if we can mangle the 2393 * inner IP ID field, so strip TSO if MANGLEID is not supported. 2394 */ 2395 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 2396 features &= ~NETIF_F_TSO; 2397 2398 return features; 2399 } 2400 2401 /** 2402 * igc_configure - configure the hardware for RX and TX 2403 * @adapter: private board structure 2404 */ 2405 static void igc_configure(struct igc_adapter *adapter) 2406 { 2407 struct net_device *netdev = adapter->netdev; 2408 int i = 0; 2409 2410 igc_get_hw_control(adapter); 2411 igc_set_rx_mode(netdev); 2412 2413 igc_setup_tctl(adapter); 2414 igc_setup_mrqc(adapter); 2415 igc_setup_rctl(adapter); 2416 2417 igc_nfc_filter_restore(adapter); 2418 igc_configure_tx(adapter); 2419 igc_configure_rx(adapter); 2420 2421 igc_rx_fifo_flush_base(&adapter->hw); 2422 2423 /* call igc_desc_unused which always leaves 2424 * at least 1 descriptor unused to make sure 2425 * next_to_use != next_to_clean 2426 */ 2427 for (i = 0; i < adapter->num_rx_queues; i++) { 2428 struct igc_ring *ring = adapter->rx_ring[i]; 2429 2430 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 2431 } 2432 } 2433 2434 /** 2435 * igc_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table 2436 * @adapter: address of board private structure 2437 * @index: Index of the RAR entry which need to be synced with MAC table 2438 */ 2439 static void igc_rar_set_index(struct igc_adapter *adapter, u32 index) 2440 { 2441 u8 *addr = adapter->mac_table[index].addr; 2442 struct igc_hw *hw = &adapter->hw; 2443 u32 rar_low, rar_high; 2444 2445 /* HW expects these to be in network order when they are plugged 2446 * into the registers which are little endian. In order to guarantee 2447 * that ordering we need to do an leXX_to_cpup here in order to be 2448 * ready for the byteswap that occurs with writel 2449 */ 2450 rar_low = le32_to_cpup((__le32 *)(addr)); 2451 rar_high = le16_to_cpup((__le16 *)(addr + 4)); 2452 2453 /* Indicate to hardware the Address is Valid. */ 2454 if (adapter->mac_table[index].state & IGC_MAC_STATE_IN_USE) { 2455 if (is_valid_ether_addr(addr)) 2456 rar_high |= IGC_RAH_AV; 2457 2458 rar_high |= IGC_RAH_POOL_1 << 2459 adapter->mac_table[index].queue; 2460 } 2461 2462 wr32(IGC_RAL(index), rar_low); 2463 wrfl(); 2464 wr32(IGC_RAH(index), rar_high); 2465 wrfl(); 2466 } 2467 2468 /* Set default MAC address for the PF in the first RAR entry */ 2469 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 2470 { 2471 struct igc_mac_addr *mac_table = &adapter->mac_table[0]; 2472 2473 ether_addr_copy(mac_table->addr, adapter->hw.mac.addr); 2474 mac_table->state = IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; 2475 2476 igc_rar_set_index(adapter, 0); 2477 } 2478 2479 /* If the filter to be added and an already existing filter express 2480 * the same address and address type, it should be possible to only 2481 * override the other configurations, for example the queue to steer 2482 * traffic. 2483 */ 2484 static bool igc_mac_entry_can_be_used(const struct igc_mac_addr *entry, 2485 const u8 *addr, const u8 flags) 2486 { 2487 if (!(entry->state & IGC_MAC_STATE_IN_USE)) 2488 return true; 2489 2490 if ((entry->state & IGC_MAC_STATE_SRC_ADDR) != 2491 (flags & IGC_MAC_STATE_SRC_ADDR)) 2492 return false; 2493 2494 if (!ether_addr_equal(addr, entry->addr)) 2495 return false; 2496 2497 return true; 2498 } 2499 2500 /* Add a MAC filter for 'addr' directing matching traffic to 'queue', 2501 * 'flags' is used to indicate what kind of match is made, match is by 2502 * default for the destination address, if matching by source address 2503 * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. 2504 */ 2505 static int igc_add_mac_filter_flags(struct igc_adapter *adapter, 2506 const u8 *addr, const u8 queue, 2507 const u8 flags) 2508 { 2509 struct igc_hw *hw = &adapter->hw; 2510 int rar_entries = hw->mac.rar_entry_count; 2511 int i; 2512 2513 if (is_zero_ether_addr(addr)) 2514 return -EINVAL; 2515 2516 /* Search for the first empty entry in the MAC table. 2517 * Do not touch entries at the end of the table reserved for the VF MAC 2518 * addresses. 2519 */ 2520 for (i = 0; i < rar_entries; i++) { 2521 if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], 2522 addr, flags)) 2523 continue; 2524 2525 ether_addr_copy(adapter->mac_table[i].addr, addr); 2526 adapter->mac_table[i].queue = queue; 2527 adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE | flags; 2528 2529 igc_rar_set_index(adapter, i); 2530 return i; 2531 } 2532 2533 return -ENOSPC; 2534 } 2535 2536 int igc_add_mac_steering_filter(struct igc_adapter *adapter, 2537 const u8 *addr, u8 queue, u8 flags) 2538 { 2539 return igc_add_mac_filter_flags(adapter, addr, queue, 2540 IGC_MAC_STATE_QUEUE_STEERING | flags); 2541 } 2542 2543 /* Remove a MAC filter for 'addr' directing matching traffic to 2544 * 'queue', 'flags' is used to indicate what kind of match need to be 2545 * removed, match is by default for the destination address, if 2546 * matching by source address is to be removed the flag 2547 * IGC_MAC_STATE_SRC_ADDR can be used. 2548 */ 2549 static int igc_del_mac_filter_flags(struct igc_adapter *adapter, 2550 const u8 *addr, const u8 queue, 2551 const u8 flags) 2552 { 2553 struct igc_hw *hw = &adapter->hw; 2554 int rar_entries = hw->mac.rar_entry_count; 2555 int i; 2556 2557 if (is_zero_ether_addr(addr)) 2558 return -EINVAL; 2559 2560 /* Search for matching entry in the MAC table based on given address 2561 * and queue. Do not touch entries at the end of the table reserved 2562 * for the VF MAC addresses. 2563 */ 2564 for (i = 0; i < rar_entries; i++) { 2565 if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) 2566 continue; 2567 if ((adapter->mac_table[i].state & flags) != flags) 2568 continue; 2569 if (adapter->mac_table[i].queue != queue) 2570 continue; 2571 if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) 2572 continue; 2573 2574 /* When a filter for the default address is "deleted", 2575 * we return it to its initial configuration 2576 */ 2577 if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { 2578 adapter->mac_table[i].state = 2579 IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; 2580 } else { 2581 adapter->mac_table[i].state = 0; 2582 adapter->mac_table[i].queue = 0; 2583 memset(adapter->mac_table[i].addr, 0, ETH_ALEN); 2584 } 2585 2586 igc_rar_set_index(adapter, i); 2587 return 0; 2588 } 2589 2590 return -ENOENT; 2591 } 2592 2593 int igc_del_mac_steering_filter(struct igc_adapter *adapter, 2594 const u8 *addr, u8 queue, u8 flags) 2595 { 2596 return igc_del_mac_filter_flags(adapter, addr, queue, 2597 IGC_MAC_STATE_QUEUE_STEERING | flags); 2598 } 2599 2600 /* Add a MAC filter for 'addr' directing matching traffic to 'queue', 2601 * 'flags' is used to indicate what kind of match is made, match is by 2602 * default for the destination address, if matching by source address 2603 * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. 2604 */ 2605 static int igc_add_mac_filter(struct igc_adapter *adapter, 2606 const u8 *addr, const u8 queue) 2607 { 2608 struct igc_hw *hw = &adapter->hw; 2609 int rar_entries = hw->mac.rar_entry_count; 2610 int i; 2611 2612 if (is_zero_ether_addr(addr)) 2613 return -EINVAL; 2614 2615 /* Search for the first empty entry in the MAC table. 2616 * Do not touch entries at the end of the table reserved for the VF MAC 2617 * addresses. 2618 */ 2619 for (i = 0; i < rar_entries; i++) { 2620 if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], 2621 addr, 0)) 2622 continue; 2623 2624 ether_addr_copy(adapter->mac_table[i].addr, addr); 2625 adapter->mac_table[i].queue = queue; 2626 adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE; 2627 2628 igc_rar_set_index(adapter, i); 2629 return i; 2630 } 2631 2632 return -ENOSPC; 2633 } 2634 2635 /* Remove a MAC filter for 'addr' directing matching traffic to 2636 * 'queue', 'flags' is used to indicate what kind of match need to be 2637 * removed, match is by default for the destination address, if 2638 * matching by source address is to be removed the flag 2639 * IGC_MAC_STATE_SRC_ADDR can be used. 2640 */ 2641 static int igc_del_mac_filter(struct igc_adapter *adapter, 2642 const u8 *addr, const u8 queue) 2643 { 2644 struct igc_hw *hw = &adapter->hw; 2645 int rar_entries = hw->mac.rar_entry_count; 2646 int i; 2647 2648 if (is_zero_ether_addr(addr)) 2649 return -EINVAL; 2650 2651 /* Search for matching entry in the MAC table based on given address 2652 * and queue. Do not touch entries at the end of the table reserved 2653 * for the VF MAC addresses. 2654 */ 2655 for (i = 0; i < rar_entries; i++) { 2656 if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) 2657 continue; 2658 if (adapter->mac_table[i].state != 0) 2659 continue; 2660 if (adapter->mac_table[i].queue != queue) 2661 continue; 2662 if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) 2663 continue; 2664 2665 /* When a filter for the default address is "deleted", 2666 * we return it to its initial configuration 2667 */ 2668 if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { 2669 adapter->mac_table[i].state = 2670 IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; 2671 adapter->mac_table[i].queue = 0; 2672 } else { 2673 adapter->mac_table[i].state = 0; 2674 adapter->mac_table[i].queue = 0; 2675 memset(adapter->mac_table[i].addr, 0, ETH_ALEN); 2676 } 2677 2678 igc_rar_set_index(adapter, i); 2679 return 0; 2680 } 2681 2682 return -ENOENT; 2683 } 2684 2685 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) 2686 { 2687 struct igc_adapter *adapter = netdev_priv(netdev); 2688 int ret; 2689 2690 ret = igc_add_mac_filter(adapter, addr, adapter->num_rx_queues); 2691 2692 return min_t(int, ret, 0); 2693 } 2694 2695 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) 2696 { 2697 struct igc_adapter *adapter = netdev_priv(netdev); 2698 2699 igc_del_mac_filter(adapter, addr, adapter->num_rx_queues); 2700 2701 return 0; 2702 } 2703 2704 /** 2705 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 2706 * @netdev: network interface device structure 2707 * 2708 * The set_rx_mode entry point is called whenever the unicast or multicast 2709 * address lists or the network interface flags are updated. This routine is 2710 * responsible for configuring the hardware for proper unicast, multicast, 2711 * promiscuous mode, and all-multi behavior. 2712 */ 2713 static void igc_set_rx_mode(struct net_device *netdev) 2714 { 2715 struct igc_adapter *adapter = netdev_priv(netdev); 2716 struct igc_hw *hw = &adapter->hw; 2717 u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 2718 int count; 2719 2720 /* Check for Promiscuous and All Multicast modes */ 2721 if (netdev->flags & IFF_PROMISC) { 2722 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; 2723 } else { 2724 if (netdev->flags & IFF_ALLMULTI) { 2725 rctl |= IGC_RCTL_MPE; 2726 } else { 2727 /* Write addresses to the MTA, if the attempt fails 2728 * then we should just turn on promiscuous mode so 2729 * that we can at least receive multicast traffic 2730 */ 2731 count = igc_write_mc_addr_list(netdev); 2732 if (count < 0) 2733 rctl |= IGC_RCTL_MPE; 2734 } 2735 } 2736 2737 /* Write addresses to available RAR registers, if there is not 2738 * sufficient space to store all the addresses then enable 2739 * unicast promiscuous mode 2740 */ 2741 if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) 2742 rctl |= IGC_RCTL_UPE; 2743 2744 /* update state of unicast and multicast */ 2745 rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 2746 wr32(IGC_RCTL, rctl); 2747 2748 #if (PAGE_SIZE < 8192) 2749 if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) 2750 rlpml = IGC_MAX_FRAME_BUILD_SKB; 2751 #endif 2752 wr32(IGC_RLPML, rlpml); 2753 } 2754 2755 /** 2756 * igc_msix_other - msix other interrupt handler 2757 * @irq: interrupt number 2758 * @data: pointer to a q_vector 2759 */ 2760 static irqreturn_t igc_msix_other(int irq, void *data) 2761 { 2762 struct igc_adapter *adapter = data; 2763 struct igc_hw *hw = &adapter->hw; 2764 u32 icr = rd32(IGC_ICR); 2765 2766 /* reading ICR causes bit 31 of EICR to be cleared */ 2767 if (icr & IGC_ICR_DRSTA) 2768 schedule_work(&adapter->reset_task); 2769 2770 if (icr & IGC_ICR_DOUTSYNC) { 2771 /* HW is reporting DMA is out of sync */ 2772 adapter->stats.doosync++; 2773 } 2774 2775 if (icr & IGC_ICR_LSC) { 2776 hw->mac.get_link_status = 1; 2777 /* guard against interrupt when we're going down */ 2778 if (!test_bit(__IGC_DOWN, &adapter->state)) 2779 mod_timer(&adapter->watchdog_timer, jiffies + 1); 2780 } 2781 2782 wr32(IGC_EIMS, adapter->eims_other); 2783 2784 return IRQ_HANDLED; 2785 } 2786 2787 /** 2788 * igc_write_ivar - configure ivar for given MSI-X vector 2789 * @hw: pointer to the HW structure 2790 * @msix_vector: vector number we are allocating to a given ring 2791 * @index: row index of IVAR register to write within IVAR table 2792 * @offset: column offset of in IVAR, should be multiple of 8 2793 * 2794 * The IVAR table consists of 2 columns, 2795 * each containing an cause allocation for an Rx and Tx ring, and a 2796 * variable number of rows depending on the number of queues supported. 2797 */ 2798 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 2799 int index, int offset) 2800 { 2801 u32 ivar = array_rd32(IGC_IVAR0, index); 2802 2803 /* clear any bits that are currently set */ 2804 ivar &= ~((u32)0xFF << offset); 2805 2806 /* write vector and valid bit */ 2807 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 2808 2809 array_wr32(IGC_IVAR0, index, ivar); 2810 } 2811 2812 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 2813 { 2814 struct igc_adapter *adapter = q_vector->adapter; 2815 struct igc_hw *hw = &adapter->hw; 2816 int rx_queue = IGC_N0_QUEUE; 2817 int tx_queue = IGC_N0_QUEUE; 2818 2819 if (q_vector->rx.ring) 2820 rx_queue = q_vector->rx.ring->reg_idx; 2821 if (q_vector->tx.ring) 2822 tx_queue = q_vector->tx.ring->reg_idx; 2823 2824 switch (hw->mac.type) { 2825 case igc_i225: 2826 if (rx_queue > IGC_N0_QUEUE) 2827 igc_write_ivar(hw, msix_vector, 2828 rx_queue >> 1, 2829 (rx_queue & 0x1) << 4); 2830 if (tx_queue > IGC_N0_QUEUE) 2831 igc_write_ivar(hw, msix_vector, 2832 tx_queue >> 1, 2833 ((tx_queue & 0x1) << 4) + 8); 2834 q_vector->eims_value = BIT(msix_vector); 2835 break; 2836 default: 2837 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 2838 break; 2839 } 2840 2841 /* add q_vector eims value to global eims_enable_mask */ 2842 adapter->eims_enable_mask |= q_vector->eims_value; 2843 2844 /* configure q_vector to set itr on first interrupt */ 2845 q_vector->set_itr = 1; 2846 } 2847 2848 /** 2849 * igc_configure_msix - Configure MSI-X hardware 2850 * @adapter: Pointer to adapter structure 2851 * 2852 * igc_configure_msix sets up the hardware to properly 2853 * generate MSI-X interrupts. 2854 */ 2855 static void igc_configure_msix(struct igc_adapter *adapter) 2856 { 2857 struct igc_hw *hw = &adapter->hw; 2858 int i, vector = 0; 2859 u32 tmp; 2860 2861 adapter->eims_enable_mask = 0; 2862 2863 /* set vector for other causes, i.e. link changes */ 2864 switch (hw->mac.type) { 2865 case igc_i225: 2866 /* Turn on MSI-X capability first, or our settings 2867 * won't stick. And it will take days to debug. 2868 */ 2869 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 2870 IGC_GPIE_PBA | IGC_GPIE_EIAME | 2871 IGC_GPIE_NSICR); 2872 2873 /* enable msix_other interrupt */ 2874 adapter->eims_other = BIT(vector); 2875 tmp = (vector++ | IGC_IVAR_VALID) << 8; 2876 2877 wr32(IGC_IVAR_MISC, tmp); 2878 break; 2879 default: 2880 /* do nothing, since nothing else supports MSI-X */ 2881 break; 2882 } /* switch (hw->mac.type) */ 2883 2884 adapter->eims_enable_mask |= adapter->eims_other; 2885 2886 for (i = 0; i < adapter->num_q_vectors; i++) 2887 igc_assign_vector(adapter->q_vector[i], vector++); 2888 2889 wrfl(); 2890 } 2891 2892 static irqreturn_t igc_msix_ring(int irq, void *data) 2893 { 2894 struct igc_q_vector *q_vector = data; 2895 2896 /* Write the ITR value calculated from the previous interrupt. */ 2897 igc_write_itr(q_vector); 2898 2899 napi_schedule(&q_vector->napi); 2900 2901 return IRQ_HANDLED; 2902 } 2903 2904 /** 2905 * igc_request_msix - Initialize MSI-X interrupts 2906 * @adapter: Pointer to adapter structure 2907 * 2908 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 2909 * kernel. 2910 */ 2911 static int igc_request_msix(struct igc_adapter *adapter) 2912 { 2913 int i = 0, err = 0, vector = 0, free_vector = 0; 2914 struct net_device *netdev = adapter->netdev; 2915 2916 err = request_irq(adapter->msix_entries[vector].vector, 2917 &igc_msix_other, 0, netdev->name, adapter); 2918 if (err) 2919 goto err_out; 2920 2921 for (i = 0; i < adapter->num_q_vectors; i++) { 2922 struct igc_q_vector *q_vector = adapter->q_vector[i]; 2923 2924 vector++; 2925 2926 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 2927 2928 if (q_vector->rx.ring && q_vector->tx.ring) 2929 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 2930 q_vector->rx.ring->queue_index); 2931 else if (q_vector->tx.ring) 2932 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 2933 q_vector->tx.ring->queue_index); 2934 else if (q_vector->rx.ring) 2935 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 2936 q_vector->rx.ring->queue_index); 2937 else 2938 sprintf(q_vector->name, "%s-unused", netdev->name); 2939 2940 err = request_irq(adapter->msix_entries[vector].vector, 2941 igc_msix_ring, 0, q_vector->name, 2942 q_vector); 2943 if (err) 2944 goto err_free; 2945 } 2946 2947 igc_configure_msix(adapter); 2948 return 0; 2949 2950 err_free: 2951 /* free already assigned IRQs */ 2952 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 2953 2954 vector--; 2955 for (i = 0; i < vector; i++) { 2956 free_irq(adapter->msix_entries[free_vector++].vector, 2957 adapter->q_vector[i]); 2958 } 2959 err_out: 2960 return err; 2961 } 2962 2963 /** 2964 * igc_reset_q_vector - Reset config for interrupt vector 2965 * @adapter: board private structure to initialize 2966 * @v_idx: Index of vector to be reset 2967 * 2968 * If NAPI is enabled it will delete any references to the 2969 * NAPI struct. This is preparation for igc_free_q_vector. 2970 */ 2971 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 2972 { 2973 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2974 2975 /* if we're coming from igc_set_interrupt_capability, the vectors are 2976 * not yet allocated 2977 */ 2978 if (!q_vector) 2979 return; 2980 2981 if (q_vector->tx.ring) 2982 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 2983 2984 if (q_vector->rx.ring) 2985 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 2986 2987 netif_napi_del(&q_vector->napi); 2988 } 2989 2990 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 2991 { 2992 int v_idx = adapter->num_q_vectors; 2993 2994 if (adapter->msix_entries) { 2995 pci_disable_msix(adapter->pdev); 2996 kfree(adapter->msix_entries); 2997 adapter->msix_entries = NULL; 2998 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 2999 pci_disable_msi(adapter->pdev); 3000 } 3001 3002 while (v_idx--) 3003 igc_reset_q_vector(adapter, v_idx); 3004 } 3005 3006 /** 3007 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 3008 * @adapter: Pointer to adapter structure 3009 * 3010 * This function resets the device so that it has 0 rx queues, tx queues, and 3011 * MSI-X interrupts allocated. 3012 */ 3013 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 3014 { 3015 igc_free_q_vectors(adapter); 3016 igc_reset_interrupt_capability(adapter); 3017 } 3018 3019 /** 3020 * igc_free_q_vectors - Free memory allocated for interrupt vectors 3021 * @adapter: board private structure to initialize 3022 * 3023 * This function frees the memory allocated to the q_vectors. In addition if 3024 * NAPI is enabled it will delete any references to the NAPI struct prior 3025 * to freeing the q_vector. 3026 */ 3027 static void igc_free_q_vectors(struct igc_adapter *adapter) 3028 { 3029 int v_idx = adapter->num_q_vectors; 3030 3031 adapter->num_tx_queues = 0; 3032 adapter->num_rx_queues = 0; 3033 adapter->num_q_vectors = 0; 3034 3035 while (v_idx--) { 3036 igc_reset_q_vector(adapter, v_idx); 3037 igc_free_q_vector(adapter, v_idx); 3038 } 3039 } 3040 3041 /** 3042 * igc_free_q_vector - Free memory allocated for specific interrupt vector 3043 * @adapter: board private structure to initialize 3044 * @v_idx: Index of vector to be freed 3045 * 3046 * This function frees the memory allocated to the q_vector. 3047 */ 3048 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 3049 { 3050 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 3051 3052 adapter->q_vector[v_idx] = NULL; 3053 3054 /* igc_get_stats64() might access the rings on this vector, 3055 * we must wait a grace period before freeing it. 3056 */ 3057 if (q_vector) 3058 kfree_rcu(q_vector, rcu); 3059 } 3060 3061 /* Need to wait a few seconds after link up to get diagnostic information from 3062 * the phy 3063 */ 3064 static void igc_update_phy_info(struct timer_list *t) 3065 { 3066 struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); 3067 3068 igc_get_phy_info(&adapter->hw); 3069 } 3070 3071 /** 3072 * igc_has_link - check shared code for link and determine up/down 3073 * @adapter: pointer to driver private info 3074 */ 3075 bool igc_has_link(struct igc_adapter *adapter) 3076 { 3077 struct igc_hw *hw = &adapter->hw; 3078 bool link_active = false; 3079 3080 /* get_link_status is set on LSC (link status) interrupt or 3081 * rx sequence error interrupt. get_link_status will stay 3082 * false until the igc_check_for_link establishes link 3083 * for copper adapters ONLY 3084 */ 3085 switch (hw->phy.media_type) { 3086 case igc_media_type_copper: 3087 if (!hw->mac.get_link_status) 3088 return true; 3089 hw->mac.ops.check_for_link(hw); 3090 link_active = !hw->mac.get_link_status; 3091 break; 3092 default: 3093 case igc_media_type_unknown: 3094 break; 3095 } 3096 3097 if (hw->mac.type == igc_i225 && 3098 hw->phy.id == I225_I_PHY_ID) { 3099 if (!netif_carrier_ok(adapter->netdev)) { 3100 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3101 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 3102 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 3103 adapter->link_check_timeout = jiffies; 3104 } 3105 } 3106 3107 return link_active; 3108 } 3109 3110 /** 3111 * igc_watchdog - Timer Call-back 3112 * @data: pointer to adapter cast into an unsigned long 3113 */ 3114 static void igc_watchdog(struct timer_list *t) 3115 { 3116 struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); 3117 /* Do the rest outside of interrupt context */ 3118 schedule_work(&adapter->watchdog_task); 3119 } 3120 3121 static void igc_watchdog_task(struct work_struct *work) 3122 { 3123 struct igc_adapter *adapter = container_of(work, 3124 struct igc_adapter, 3125 watchdog_task); 3126 struct net_device *netdev = adapter->netdev; 3127 struct igc_hw *hw = &adapter->hw; 3128 struct igc_phy_info *phy = &hw->phy; 3129 u16 phy_data, retry_count = 20; 3130 u32 connsw; 3131 u32 link; 3132 int i; 3133 3134 link = igc_has_link(adapter); 3135 3136 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 3137 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 3138 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3139 else 3140 link = false; 3141 } 3142 3143 /* Force link down if we have fiber to swap to */ 3144 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 3145 if (hw->phy.media_type == igc_media_type_copper) { 3146 connsw = rd32(IGC_CONNSW); 3147 if (!(connsw & IGC_CONNSW_AUTOSENSE_EN)) 3148 link = 0; 3149 } 3150 } 3151 if (link) { 3152 if (!netif_carrier_ok(netdev)) { 3153 u32 ctrl; 3154 3155 hw->mac.ops.get_speed_and_duplex(hw, 3156 &adapter->link_speed, 3157 &adapter->link_duplex); 3158 3159 ctrl = rd32(IGC_CTRL); 3160 /* Link status message must follow this format */ 3161 netdev_info(netdev, 3162 "igc: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 3163 netdev->name, 3164 adapter->link_speed, 3165 adapter->link_duplex == FULL_DUPLEX ? 3166 "Full" : "Half", 3167 (ctrl & IGC_CTRL_TFCE) && 3168 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 3169 (ctrl & IGC_CTRL_RFCE) ? "RX" : 3170 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 3171 3172 /* check if SmartSpeed worked */ 3173 igc_check_downshift(hw); 3174 if (phy->speed_downgraded) 3175 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 3176 3177 /* adjust timeout factor according to speed/duplex */ 3178 adapter->tx_timeout_factor = 1; 3179 switch (adapter->link_speed) { 3180 case SPEED_10: 3181 adapter->tx_timeout_factor = 14; 3182 break; 3183 case SPEED_100: 3184 /* maybe add some timeout factor ? */ 3185 break; 3186 } 3187 3188 if (adapter->link_speed != SPEED_1000) 3189 goto no_wait; 3190 3191 /* wait for Remote receiver status OK */ 3192 retry_read_status: 3193 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 3194 &phy_data)) { 3195 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 3196 retry_count) { 3197 msleep(100); 3198 retry_count--; 3199 goto retry_read_status; 3200 } else if (!retry_count) { 3201 dev_err(&adapter->pdev->dev, "exceed max 2 second\n"); 3202 } 3203 } else { 3204 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n"); 3205 } 3206 no_wait: 3207 netif_carrier_on(netdev); 3208 3209 /* link state has changed, schedule phy info update */ 3210 if (!test_bit(__IGC_DOWN, &adapter->state)) 3211 mod_timer(&adapter->phy_info_timer, 3212 round_jiffies(jiffies + 2 * HZ)); 3213 } 3214 } else { 3215 if (netif_carrier_ok(netdev)) { 3216 adapter->link_speed = 0; 3217 adapter->link_duplex = 0; 3218 3219 /* Links status message must follow this format */ 3220 netdev_info(netdev, "igc: %s NIC Link is Down\n", 3221 netdev->name); 3222 netif_carrier_off(netdev); 3223 3224 /* link state has changed, schedule phy info update */ 3225 if (!test_bit(__IGC_DOWN, &adapter->state)) 3226 mod_timer(&adapter->phy_info_timer, 3227 round_jiffies(jiffies + 2 * HZ)); 3228 3229 /* link is down, time to check for alternate media */ 3230 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 3231 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 3232 schedule_work(&adapter->reset_task); 3233 /* return immediately */ 3234 return; 3235 } 3236 } 3237 3238 /* also check for alternate media here */ 3239 } else if (!netif_carrier_ok(netdev) && 3240 (adapter->flags & IGC_FLAG_MAS_ENABLE)) { 3241 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 3242 schedule_work(&adapter->reset_task); 3243 /* return immediately */ 3244 return; 3245 } 3246 } 3247 } 3248 3249 spin_lock(&adapter->stats64_lock); 3250 igc_update_stats(adapter); 3251 spin_unlock(&adapter->stats64_lock); 3252 3253 for (i = 0; i < adapter->num_tx_queues; i++) { 3254 struct igc_ring *tx_ring = adapter->tx_ring[i]; 3255 3256 if (!netif_carrier_ok(netdev)) { 3257 /* We've lost link, so the controller stops DMA, 3258 * but we've got queued Tx work that's never going 3259 * to get done, so reset controller to flush Tx. 3260 * (Do the reset outside of interrupt context). 3261 */ 3262 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 3263 adapter->tx_timeout_count++; 3264 schedule_work(&adapter->reset_task); 3265 /* return immediately since reset is imminent */ 3266 return; 3267 } 3268 } 3269 3270 /* Force detection of hung controller every watchdog period */ 3271 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 3272 } 3273 3274 /* Cause software interrupt to ensure Rx ring is cleaned */ 3275 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 3276 u32 eics = 0; 3277 3278 for (i = 0; i < adapter->num_q_vectors; i++) 3279 eics |= adapter->q_vector[i]->eims_value; 3280 wr32(IGC_EICS, eics); 3281 } else { 3282 wr32(IGC_ICS, IGC_ICS_RXDMT0); 3283 } 3284 3285 /* Reset the timer */ 3286 if (!test_bit(__IGC_DOWN, &adapter->state)) { 3287 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 3288 mod_timer(&adapter->watchdog_timer, 3289 round_jiffies(jiffies + HZ)); 3290 else 3291 mod_timer(&adapter->watchdog_timer, 3292 round_jiffies(jiffies + 2 * HZ)); 3293 } 3294 } 3295 3296 /** 3297 * igc_update_ring_itr - update the dynamic ITR value based on packet size 3298 * @q_vector: pointer to q_vector 3299 * 3300 * Stores a new ITR value based on strictly on packet size. This 3301 * algorithm is less sophisticated than that used in igc_update_itr, 3302 * due to the difficulty of synchronizing statistics across multiple 3303 * receive rings. The divisors and thresholds used by this function 3304 * were determined based on theoretical maximum wire speed and testing 3305 * data, in order to minimize response time while increasing bulk 3306 * throughput. 3307 * NOTE: This function is called only when operating in a multiqueue 3308 * receive environment. 3309 */ 3310 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 3311 { 3312 struct igc_adapter *adapter = q_vector->adapter; 3313 int new_val = q_vector->itr_val; 3314 int avg_wire_size = 0; 3315 unsigned int packets; 3316 3317 /* For non-gigabit speeds, just fix the interrupt rate at 4000 3318 * ints/sec - ITR timer value of 120 ticks. 3319 */ 3320 switch (adapter->link_speed) { 3321 case SPEED_10: 3322 case SPEED_100: 3323 new_val = IGC_4K_ITR; 3324 goto set_itr_val; 3325 default: 3326 break; 3327 } 3328 3329 packets = q_vector->rx.total_packets; 3330 if (packets) 3331 avg_wire_size = q_vector->rx.total_bytes / packets; 3332 3333 packets = q_vector->tx.total_packets; 3334 if (packets) 3335 avg_wire_size = max_t(u32, avg_wire_size, 3336 q_vector->tx.total_bytes / packets); 3337 3338 /* if avg_wire_size isn't set no work was done */ 3339 if (!avg_wire_size) 3340 goto clear_counts; 3341 3342 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 3343 avg_wire_size += 24; 3344 3345 /* Don't starve jumbo frames */ 3346 avg_wire_size = min(avg_wire_size, 3000); 3347 3348 /* Give a little boost to mid-size frames */ 3349 if (avg_wire_size > 300 && avg_wire_size < 1200) 3350 new_val = avg_wire_size / 3; 3351 else 3352 new_val = avg_wire_size / 2; 3353 3354 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3355 if (new_val < IGC_20K_ITR && 3356 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3357 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3358 new_val = IGC_20K_ITR; 3359 3360 set_itr_val: 3361 if (new_val != q_vector->itr_val) { 3362 q_vector->itr_val = new_val; 3363 q_vector->set_itr = 1; 3364 } 3365 clear_counts: 3366 q_vector->rx.total_bytes = 0; 3367 q_vector->rx.total_packets = 0; 3368 q_vector->tx.total_bytes = 0; 3369 q_vector->tx.total_packets = 0; 3370 } 3371 3372 /** 3373 * igc_update_itr - update the dynamic ITR value based on statistics 3374 * @q_vector: pointer to q_vector 3375 * @ring_container: ring info to update the itr for 3376 * 3377 * Stores a new ITR value based on packets and byte 3378 * counts during the last interrupt. The advantage of per interrupt 3379 * computation is faster updates and more accurate ITR for the current 3380 * traffic pattern. Constants in this function were computed 3381 * based on theoretical maximum wire speed and thresholds were set based 3382 * on testing data as well as attempting to minimize response time 3383 * while increasing bulk throughput. 3384 * NOTE: These calculations are only valid when operating in a single- 3385 * queue environment. 3386 */ 3387 static void igc_update_itr(struct igc_q_vector *q_vector, 3388 struct igc_ring_container *ring_container) 3389 { 3390 unsigned int packets = ring_container->total_packets; 3391 unsigned int bytes = ring_container->total_bytes; 3392 u8 itrval = ring_container->itr; 3393 3394 /* no packets, exit with status unchanged */ 3395 if (packets == 0) 3396 return; 3397 3398 switch (itrval) { 3399 case lowest_latency: 3400 /* handle TSO and jumbo frames */ 3401 if (bytes / packets > 8000) 3402 itrval = bulk_latency; 3403 else if ((packets < 5) && (bytes > 512)) 3404 itrval = low_latency; 3405 break; 3406 case low_latency: /* 50 usec aka 20000 ints/s */ 3407 if (bytes > 10000) { 3408 /* this if handles the TSO accounting */ 3409 if (bytes / packets > 8000) 3410 itrval = bulk_latency; 3411 else if ((packets < 10) || ((bytes / packets) > 1200)) 3412 itrval = bulk_latency; 3413 else if ((packets > 35)) 3414 itrval = lowest_latency; 3415 } else if (bytes / packets > 2000) { 3416 itrval = bulk_latency; 3417 } else if (packets <= 2 && bytes < 512) { 3418 itrval = lowest_latency; 3419 } 3420 break; 3421 case bulk_latency: /* 250 usec aka 4000 ints/s */ 3422 if (bytes > 25000) { 3423 if (packets > 35) 3424 itrval = low_latency; 3425 } else if (bytes < 1500) { 3426 itrval = low_latency; 3427 } 3428 break; 3429 } 3430 3431 /* clear work counters since we have the values we need */ 3432 ring_container->total_bytes = 0; 3433 ring_container->total_packets = 0; 3434 3435 /* write updated itr to ring container */ 3436 ring_container->itr = itrval; 3437 } 3438 3439 /** 3440 * igc_intr_msi - Interrupt Handler 3441 * @irq: interrupt number 3442 * @data: pointer to a network interface device structure 3443 */ 3444 static irqreturn_t igc_intr_msi(int irq, void *data) 3445 { 3446 struct igc_adapter *adapter = data; 3447 struct igc_q_vector *q_vector = adapter->q_vector[0]; 3448 struct igc_hw *hw = &adapter->hw; 3449 /* read ICR disables interrupts using IAM */ 3450 u32 icr = rd32(IGC_ICR); 3451 3452 igc_write_itr(q_vector); 3453 3454 if (icr & IGC_ICR_DRSTA) 3455 schedule_work(&adapter->reset_task); 3456 3457 if (icr & IGC_ICR_DOUTSYNC) { 3458 /* HW is reporting DMA is out of sync */ 3459 adapter->stats.doosync++; 3460 } 3461 3462 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 3463 hw->mac.get_link_status = 1; 3464 if (!test_bit(__IGC_DOWN, &adapter->state)) 3465 mod_timer(&adapter->watchdog_timer, jiffies + 1); 3466 } 3467 3468 napi_schedule(&q_vector->napi); 3469 3470 return IRQ_HANDLED; 3471 } 3472 3473 /** 3474 * igc_intr - Legacy Interrupt Handler 3475 * @irq: interrupt number 3476 * @data: pointer to a network interface device structure 3477 */ 3478 static irqreturn_t igc_intr(int irq, void *data) 3479 { 3480 struct igc_adapter *adapter = data; 3481 struct igc_q_vector *q_vector = adapter->q_vector[0]; 3482 struct igc_hw *hw = &adapter->hw; 3483 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 3484 * need for the IMC write 3485 */ 3486 u32 icr = rd32(IGC_ICR); 3487 3488 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 3489 * not set, then the adapter didn't send an interrupt 3490 */ 3491 if (!(icr & IGC_ICR_INT_ASSERTED)) 3492 return IRQ_NONE; 3493 3494 igc_write_itr(q_vector); 3495 3496 if (icr & IGC_ICR_DRSTA) 3497 schedule_work(&adapter->reset_task); 3498 3499 if (icr & IGC_ICR_DOUTSYNC) { 3500 /* HW is reporting DMA is out of sync */ 3501 adapter->stats.doosync++; 3502 } 3503 3504 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 3505 hw->mac.get_link_status = 1; 3506 /* guard against interrupt when we're going down */ 3507 if (!test_bit(__IGC_DOWN, &adapter->state)) 3508 mod_timer(&adapter->watchdog_timer, jiffies + 1); 3509 } 3510 3511 napi_schedule(&q_vector->napi); 3512 3513 return IRQ_HANDLED; 3514 } 3515 3516 static void igc_set_itr(struct igc_q_vector *q_vector) 3517 { 3518 struct igc_adapter *adapter = q_vector->adapter; 3519 u32 new_itr = q_vector->itr_val; 3520 u8 current_itr = 0; 3521 3522 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 3523 switch (adapter->link_speed) { 3524 case SPEED_10: 3525 case SPEED_100: 3526 current_itr = 0; 3527 new_itr = IGC_4K_ITR; 3528 goto set_itr_now; 3529 default: 3530 break; 3531 } 3532 3533 igc_update_itr(q_vector, &q_vector->tx); 3534 igc_update_itr(q_vector, &q_vector->rx); 3535 3536 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 3537 3538 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3539 if (current_itr == lowest_latency && 3540 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3541 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3542 current_itr = low_latency; 3543 3544 switch (current_itr) { 3545 /* counts and packets in update_itr are dependent on these numbers */ 3546 case lowest_latency: 3547 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 3548 break; 3549 case low_latency: 3550 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 3551 break; 3552 case bulk_latency: 3553 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 3554 break; 3555 default: 3556 break; 3557 } 3558 3559 set_itr_now: 3560 if (new_itr != q_vector->itr_val) { 3561 /* this attempts to bias the interrupt rate towards Bulk 3562 * by adding intermediate steps when interrupt rate is 3563 * increasing 3564 */ 3565 new_itr = new_itr > q_vector->itr_val ? 3566 max((new_itr * q_vector->itr_val) / 3567 (new_itr + (q_vector->itr_val >> 2)), 3568 new_itr) : new_itr; 3569 /* Don't write the value here; it resets the adapter's 3570 * internal timer, and causes us to delay far longer than 3571 * we should between interrupts. Instead, we write the ITR 3572 * value at the beginning of the next interrupt so the timing 3573 * ends up being correct. 3574 */ 3575 q_vector->itr_val = new_itr; 3576 q_vector->set_itr = 1; 3577 } 3578 } 3579 3580 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 3581 { 3582 struct igc_adapter *adapter = q_vector->adapter; 3583 struct igc_hw *hw = &adapter->hw; 3584 3585 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 3586 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 3587 if (adapter->num_q_vectors == 1) 3588 igc_set_itr(q_vector); 3589 else 3590 igc_update_ring_itr(q_vector); 3591 } 3592 3593 if (!test_bit(__IGC_DOWN, &adapter->state)) { 3594 if (adapter->msix_entries) 3595 wr32(IGC_EIMS, q_vector->eims_value); 3596 else 3597 igc_irq_enable(adapter); 3598 } 3599 } 3600 3601 /** 3602 * igc_poll - NAPI Rx polling callback 3603 * @napi: napi polling structure 3604 * @budget: count of how many packets we should handle 3605 */ 3606 static int igc_poll(struct napi_struct *napi, int budget) 3607 { 3608 struct igc_q_vector *q_vector = container_of(napi, 3609 struct igc_q_vector, 3610 napi); 3611 bool clean_complete = true; 3612 int work_done = 0; 3613 3614 if (q_vector->tx.ring) 3615 clean_complete = igc_clean_tx_irq(q_vector, budget); 3616 3617 if (q_vector->rx.ring) { 3618 int cleaned = igc_clean_rx_irq(q_vector, budget); 3619 3620 work_done += cleaned; 3621 if (cleaned >= budget) 3622 clean_complete = false; 3623 } 3624 3625 /* If all work not completed, return budget and keep polling */ 3626 if (!clean_complete) 3627 return budget; 3628 3629 /* Exit the polling mode, but don't re-enable interrupts if stack might 3630 * poll us due to busy-polling 3631 */ 3632 if (likely(napi_complete_done(napi, work_done))) 3633 igc_ring_irq_enable(q_vector); 3634 3635 return min(work_done, budget - 1); 3636 } 3637 3638 /** 3639 * igc_set_interrupt_capability - set MSI or MSI-X if supported 3640 * @adapter: Pointer to adapter structure 3641 * 3642 * Attempt to configure interrupts using the best available 3643 * capabilities of the hardware and kernel. 3644 */ 3645 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 3646 bool msix) 3647 { 3648 int numvecs, i; 3649 int err; 3650 3651 if (!msix) 3652 goto msi_only; 3653 adapter->flags |= IGC_FLAG_HAS_MSIX; 3654 3655 /* Number of supported queues. */ 3656 adapter->num_rx_queues = adapter->rss_queues; 3657 3658 adapter->num_tx_queues = adapter->rss_queues; 3659 3660 /* start with one vector for every Rx queue */ 3661 numvecs = adapter->num_rx_queues; 3662 3663 /* if Tx handler is separate add 1 for every Tx queue */ 3664 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 3665 numvecs += adapter->num_tx_queues; 3666 3667 /* store the number of vectors reserved for queues */ 3668 adapter->num_q_vectors = numvecs; 3669 3670 /* add 1 vector for link status interrupts */ 3671 numvecs++; 3672 3673 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), 3674 GFP_KERNEL); 3675 3676 if (!adapter->msix_entries) 3677 return; 3678 3679 /* populate entry values */ 3680 for (i = 0; i < numvecs; i++) 3681 adapter->msix_entries[i].entry = i; 3682 3683 err = pci_enable_msix_range(adapter->pdev, 3684 adapter->msix_entries, 3685 numvecs, 3686 numvecs); 3687 if (err > 0) 3688 return; 3689 3690 kfree(adapter->msix_entries); 3691 adapter->msix_entries = NULL; 3692 3693 igc_reset_interrupt_capability(adapter); 3694 3695 msi_only: 3696 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 3697 3698 adapter->rss_queues = 1; 3699 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 3700 adapter->num_rx_queues = 1; 3701 adapter->num_tx_queues = 1; 3702 adapter->num_q_vectors = 1; 3703 if (!pci_enable_msi(adapter->pdev)) 3704 adapter->flags |= IGC_FLAG_HAS_MSI; 3705 } 3706 3707 static void igc_add_ring(struct igc_ring *ring, 3708 struct igc_ring_container *head) 3709 { 3710 head->ring = ring; 3711 head->count++; 3712 } 3713 3714 /** 3715 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 3716 * @adapter: board private structure to initialize 3717 * @v_count: q_vectors allocated on adapter, used for ring interleaving 3718 * @v_idx: index of vector in adapter struct 3719 * @txr_count: total number of Tx rings to allocate 3720 * @txr_idx: index of first Tx ring to allocate 3721 * @rxr_count: total number of Rx rings to allocate 3722 * @rxr_idx: index of first Rx ring to allocate 3723 * 3724 * We allocate one q_vector. If allocation fails we return -ENOMEM. 3725 */ 3726 static int igc_alloc_q_vector(struct igc_adapter *adapter, 3727 unsigned int v_count, unsigned int v_idx, 3728 unsigned int txr_count, unsigned int txr_idx, 3729 unsigned int rxr_count, unsigned int rxr_idx) 3730 { 3731 struct igc_q_vector *q_vector; 3732 struct igc_ring *ring; 3733 int ring_count; 3734 3735 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 3736 if (txr_count > 1 || rxr_count > 1) 3737 return -ENOMEM; 3738 3739 ring_count = txr_count + rxr_count; 3740 3741 /* allocate q_vector and rings */ 3742 q_vector = adapter->q_vector[v_idx]; 3743 if (!q_vector) 3744 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), 3745 GFP_KERNEL); 3746 else 3747 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 3748 if (!q_vector) 3749 return -ENOMEM; 3750 3751 /* initialize NAPI */ 3752 netif_napi_add(adapter->netdev, &q_vector->napi, 3753 igc_poll, 64); 3754 3755 /* tie q_vector and adapter together */ 3756 adapter->q_vector[v_idx] = q_vector; 3757 q_vector->adapter = adapter; 3758 3759 /* initialize work limits */ 3760 q_vector->tx.work_limit = adapter->tx_work_limit; 3761 3762 /* initialize ITR configuration */ 3763 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 3764 q_vector->itr_val = IGC_START_ITR; 3765 3766 /* initialize pointer to rings */ 3767 ring = q_vector->ring; 3768 3769 /* initialize ITR */ 3770 if (rxr_count) { 3771 /* rx or rx/tx vector */ 3772 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 3773 q_vector->itr_val = adapter->rx_itr_setting; 3774 } else { 3775 /* tx only vector */ 3776 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 3777 q_vector->itr_val = adapter->tx_itr_setting; 3778 } 3779 3780 if (txr_count) { 3781 /* assign generic ring traits */ 3782 ring->dev = &adapter->pdev->dev; 3783 ring->netdev = adapter->netdev; 3784 3785 /* configure backlink on ring */ 3786 ring->q_vector = q_vector; 3787 3788 /* update q_vector Tx values */ 3789 igc_add_ring(ring, &q_vector->tx); 3790 3791 /* apply Tx specific ring traits */ 3792 ring->count = adapter->tx_ring_count; 3793 ring->queue_index = txr_idx; 3794 3795 /* assign ring to adapter */ 3796 adapter->tx_ring[txr_idx] = ring; 3797 3798 /* push pointer to next ring */ 3799 ring++; 3800 } 3801 3802 if (rxr_count) { 3803 /* assign generic ring traits */ 3804 ring->dev = &adapter->pdev->dev; 3805 ring->netdev = adapter->netdev; 3806 3807 /* configure backlink on ring */ 3808 ring->q_vector = q_vector; 3809 3810 /* update q_vector Rx values */ 3811 igc_add_ring(ring, &q_vector->rx); 3812 3813 /* apply Rx specific ring traits */ 3814 ring->count = adapter->rx_ring_count; 3815 ring->queue_index = rxr_idx; 3816 3817 /* assign ring to adapter */ 3818 adapter->rx_ring[rxr_idx] = ring; 3819 } 3820 3821 return 0; 3822 } 3823 3824 /** 3825 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 3826 * @adapter: board private structure to initialize 3827 * 3828 * We allocate one q_vector per queue interrupt. If allocation fails we 3829 * return -ENOMEM. 3830 */ 3831 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 3832 { 3833 int rxr_remaining = adapter->num_rx_queues; 3834 int txr_remaining = adapter->num_tx_queues; 3835 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 3836 int q_vectors = adapter->num_q_vectors; 3837 int err; 3838 3839 if (q_vectors >= (rxr_remaining + txr_remaining)) { 3840 for (; rxr_remaining; v_idx++) { 3841 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3842 0, 0, 1, rxr_idx); 3843 3844 if (err) 3845 goto err_out; 3846 3847 /* update counts and index */ 3848 rxr_remaining--; 3849 rxr_idx++; 3850 } 3851 } 3852 3853 for (; v_idx < q_vectors; v_idx++) { 3854 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 3855 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 3856 3857 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3858 tqpv, txr_idx, rqpv, rxr_idx); 3859 3860 if (err) 3861 goto err_out; 3862 3863 /* update counts and index */ 3864 rxr_remaining -= rqpv; 3865 txr_remaining -= tqpv; 3866 rxr_idx++; 3867 txr_idx++; 3868 } 3869 3870 return 0; 3871 3872 err_out: 3873 adapter->num_tx_queues = 0; 3874 adapter->num_rx_queues = 0; 3875 adapter->num_q_vectors = 0; 3876 3877 while (v_idx--) 3878 igc_free_q_vector(adapter, v_idx); 3879 3880 return -ENOMEM; 3881 } 3882 3883 /** 3884 * igc_cache_ring_register - Descriptor ring to register mapping 3885 * @adapter: board private structure to initialize 3886 * 3887 * Once we know the feature-set enabled for the device, we'll cache 3888 * the register offset the descriptor ring is assigned to. 3889 */ 3890 static void igc_cache_ring_register(struct igc_adapter *adapter) 3891 { 3892 int i = 0, j = 0; 3893 3894 switch (adapter->hw.mac.type) { 3895 case igc_i225: 3896 /* Fall through */ 3897 default: 3898 for (; i < adapter->num_rx_queues; i++) 3899 adapter->rx_ring[i]->reg_idx = i; 3900 for (; j < adapter->num_tx_queues; j++) 3901 adapter->tx_ring[j]->reg_idx = j; 3902 break; 3903 } 3904 } 3905 3906 /** 3907 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 3908 * @adapter: Pointer to adapter structure 3909 * 3910 * This function initializes the interrupts and allocates all of the queues. 3911 */ 3912 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 3913 { 3914 struct pci_dev *pdev = adapter->pdev; 3915 int err = 0; 3916 3917 igc_set_interrupt_capability(adapter, msix); 3918 3919 err = igc_alloc_q_vectors(adapter); 3920 if (err) { 3921 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); 3922 goto err_alloc_q_vectors; 3923 } 3924 3925 igc_cache_ring_register(adapter); 3926 3927 return 0; 3928 3929 err_alloc_q_vectors: 3930 igc_reset_interrupt_capability(adapter); 3931 return err; 3932 } 3933 3934 static void igc_free_irq(struct igc_adapter *adapter) 3935 { 3936 if (adapter->msix_entries) { 3937 int vector = 0, i; 3938 3939 free_irq(adapter->msix_entries[vector++].vector, adapter); 3940 3941 for (i = 0; i < adapter->num_q_vectors; i++) 3942 free_irq(adapter->msix_entries[vector++].vector, 3943 adapter->q_vector[i]); 3944 } else { 3945 free_irq(adapter->pdev->irq, adapter); 3946 } 3947 } 3948 3949 /** 3950 * igc_irq_disable - Mask off interrupt generation on the NIC 3951 * @adapter: board private structure 3952 */ 3953 static void igc_irq_disable(struct igc_adapter *adapter) 3954 { 3955 struct igc_hw *hw = &adapter->hw; 3956 3957 if (adapter->msix_entries) { 3958 u32 regval = rd32(IGC_EIAM); 3959 3960 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 3961 wr32(IGC_EIMC, adapter->eims_enable_mask); 3962 regval = rd32(IGC_EIAC); 3963 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 3964 } 3965 3966 wr32(IGC_IAM, 0); 3967 wr32(IGC_IMC, ~0); 3968 wrfl(); 3969 3970 if (adapter->msix_entries) { 3971 int vector = 0, i; 3972 3973 synchronize_irq(adapter->msix_entries[vector++].vector); 3974 3975 for (i = 0; i < adapter->num_q_vectors; i++) 3976 synchronize_irq(adapter->msix_entries[vector++].vector); 3977 } else { 3978 synchronize_irq(adapter->pdev->irq); 3979 } 3980 } 3981 3982 /** 3983 * igc_irq_enable - Enable default interrupt generation settings 3984 * @adapter: board private structure 3985 */ 3986 static void igc_irq_enable(struct igc_adapter *adapter) 3987 { 3988 struct igc_hw *hw = &adapter->hw; 3989 3990 if (adapter->msix_entries) { 3991 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 3992 u32 regval = rd32(IGC_EIAC); 3993 3994 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 3995 regval = rd32(IGC_EIAM); 3996 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 3997 wr32(IGC_EIMS, adapter->eims_enable_mask); 3998 wr32(IGC_IMS, ims); 3999 } else { 4000 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 4001 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 4002 } 4003 } 4004 4005 /** 4006 * igc_request_irq - initialize interrupts 4007 * @adapter: Pointer to adapter structure 4008 * 4009 * Attempts to configure interrupts using the best available 4010 * capabilities of the hardware and kernel. 4011 */ 4012 static int igc_request_irq(struct igc_adapter *adapter) 4013 { 4014 struct net_device *netdev = adapter->netdev; 4015 struct pci_dev *pdev = adapter->pdev; 4016 int err = 0; 4017 4018 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4019 err = igc_request_msix(adapter); 4020 if (!err) 4021 goto request_done; 4022 /* fall back to MSI */ 4023 igc_free_all_tx_resources(adapter); 4024 igc_free_all_rx_resources(adapter); 4025 4026 igc_clear_interrupt_scheme(adapter); 4027 err = igc_init_interrupt_scheme(adapter, false); 4028 if (err) 4029 goto request_done; 4030 igc_setup_all_tx_resources(adapter); 4031 igc_setup_all_rx_resources(adapter); 4032 igc_configure(adapter); 4033 } 4034 4035 igc_assign_vector(adapter->q_vector[0], 0); 4036 4037 if (adapter->flags & IGC_FLAG_HAS_MSI) { 4038 err = request_irq(pdev->irq, &igc_intr_msi, 0, 4039 netdev->name, adapter); 4040 if (!err) 4041 goto request_done; 4042 4043 /* fall back to legacy interrupts */ 4044 igc_reset_interrupt_capability(adapter); 4045 adapter->flags &= ~IGC_FLAG_HAS_MSI; 4046 } 4047 4048 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 4049 netdev->name, adapter); 4050 4051 if (err) 4052 dev_err(&pdev->dev, "Error %d getting interrupt\n", 4053 err); 4054 4055 request_done: 4056 return err; 4057 } 4058 4059 static void igc_write_itr(struct igc_q_vector *q_vector) 4060 { 4061 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 4062 4063 if (!q_vector->set_itr) 4064 return; 4065 4066 if (!itr_val) 4067 itr_val = IGC_ITR_VAL_MASK; 4068 4069 itr_val |= IGC_EITR_CNT_IGNR; 4070 4071 writel(itr_val, q_vector->itr_register); 4072 q_vector->set_itr = 0; 4073 } 4074 4075 /** 4076 * igc_open - Called when a network interface is made active 4077 * @netdev: network interface device structure 4078 * 4079 * Returns 0 on success, negative value on failure 4080 * 4081 * The open entry point is called when a network interface is made 4082 * active by the system (IFF_UP). At this point all resources needed 4083 * for transmit and receive operations are allocated, the interrupt 4084 * handler is registered with the OS, the watchdog timer is started, 4085 * and the stack is notified that the interface is ready. 4086 */ 4087 static int __igc_open(struct net_device *netdev, bool resuming) 4088 { 4089 struct igc_adapter *adapter = netdev_priv(netdev); 4090 struct igc_hw *hw = &adapter->hw; 4091 int err = 0; 4092 int i = 0; 4093 4094 /* disallow open during test */ 4095 4096 if (test_bit(__IGC_TESTING, &adapter->state)) { 4097 WARN_ON(resuming); 4098 return -EBUSY; 4099 } 4100 4101 netif_carrier_off(netdev); 4102 4103 /* allocate transmit descriptors */ 4104 err = igc_setup_all_tx_resources(adapter); 4105 if (err) 4106 goto err_setup_tx; 4107 4108 /* allocate receive descriptors */ 4109 err = igc_setup_all_rx_resources(adapter); 4110 if (err) 4111 goto err_setup_rx; 4112 4113 igc_power_up_link(adapter); 4114 4115 igc_configure(adapter); 4116 4117 err = igc_request_irq(adapter); 4118 if (err) 4119 goto err_req_irq; 4120 4121 /* Notify the stack of the actual queue counts. */ 4122 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 4123 if (err) 4124 goto err_set_queues; 4125 4126 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 4127 if (err) 4128 goto err_set_queues; 4129 4130 clear_bit(__IGC_DOWN, &adapter->state); 4131 4132 for (i = 0; i < adapter->num_q_vectors; i++) 4133 napi_enable(&adapter->q_vector[i]->napi); 4134 4135 /* Clear any pending interrupts. */ 4136 rd32(IGC_ICR); 4137 igc_irq_enable(adapter); 4138 4139 netif_tx_start_all_queues(netdev); 4140 4141 /* start the watchdog. */ 4142 hw->mac.get_link_status = 1; 4143 schedule_work(&adapter->watchdog_task); 4144 4145 return IGC_SUCCESS; 4146 4147 err_set_queues: 4148 igc_free_irq(adapter); 4149 err_req_irq: 4150 igc_release_hw_control(adapter); 4151 igc_power_down_link(adapter); 4152 igc_free_all_rx_resources(adapter); 4153 err_setup_rx: 4154 igc_free_all_tx_resources(adapter); 4155 err_setup_tx: 4156 igc_reset(adapter); 4157 4158 return err; 4159 } 4160 4161 static int igc_open(struct net_device *netdev) 4162 { 4163 return __igc_open(netdev, false); 4164 } 4165 4166 /** 4167 * igc_close - Disables a network interface 4168 * @netdev: network interface device structure 4169 * 4170 * Returns 0, this is not allowed to fail 4171 * 4172 * The close entry point is called when an interface is de-activated 4173 * by the OS. The hardware is still under the driver's control, but 4174 * needs to be disabled. A global MAC reset is issued to stop the 4175 * hardware, and all transmit and receive resources are freed. 4176 */ 4177 static int __igc_close(struct net_device *netdev, bool suspending) 4178 { 4179 struct igc_adapter *adapter = netdev_priv(netdev); 4180 4181 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 4182 4183 igc_down(adapter); 4184 4185 igc_release_hw_control(adapter); 4186 4187 igc_free_irq(adapter); 4188 4189 igc_free_all_tx_resources(adapter); 4190 igc_free_all_rx_resources(adapter); 4191 4192 return 0; 4193 } 4194 4195 static int igc_close(struct net_device *netdev) 4196 { 4197 if (netif_device_present(netdev) || netdev->dismantle) 4198 return __igc_close(netdev, false); 4199 return 0; 4200 } 4201 4202 static const struct net_device_ops igc_netdev_ops = { 4203 .ndo_open = igc_open, 4204 .ndo_stop = igc_close, 4205 .ndo_start_xmit = igc_xmit_frame, 4206 .ndo_set_rx_mode = igc_set_rx_mode, 4207 .ndo_set_mac_address = igc_set_mac, 4208 .ndo_change_mtu = igc_change_mtu, 4209 .ndo_get_stats = igc_get_stats, 4210 .ndo_fix_features = igc_fix_features, 4211 .ndo_set_features = igc_set_features, 4212 .ndo_features_check = igc_features_check, 4213 }; 4214 4215 /* PCIe configuration access */ 4216 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4217 { 4218 struct igc_adapter *adapter = hw->back; 4219 4220 pci_read_config_word(adapter->pdev, reg, value); 4221 } 4222 4223 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4224 { 4225 struct igc_adapter *adapter = hw->back; 4226 4227 pci_write_config_word(adapter->pdev, reg, *value); 4228 } 4229 4230 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4231 { 4232 struct igc_adapter *adapter = hw->back; 4233 4234 if (!pci_is_pcie(adapter->pdev)) 4235 return -IGC_ERR_CONFIG; 4236 4237 pcie_capability_read_word(adapter->pdev, reg, value); 4238 4239 return IGC_SUCCESS; 4240 } 4241 4242 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4243 { 4244 struct igc_adapter *adapter = hw->back; 4245 4246 if (!pci_is_pcie(adapter->pdev)) 4247 return -IGC_ERR_CONFIG; 4248 4249 pcie_capability_write_word(adapter->pdev, reg, *value); 4250 4251 return IGC_SUCCESS; 4252 } 4253 4254 u32 igc_rd32(struct igc_hw *hw, u32 reg) 4255 { 4256 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 4257 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 4258 u32 value = 0; 4259 4260 if (IGC_REMOVED(hw_addr)) 4261 return ~value; 4262 4263 value = readl(&hw_addr[reg]); 4264 4265 /* reads should not return all F's */ 4266 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 4267 struct net_device *netdev = igc->netdev; 4268 4269 hw->hw_addr = NULL; 4270 netif_device_detach(netdev); 4271 netdev_err(netdev, "PCIe link lost, device now detached\n"); 4272 WARN(pci_device_is_present(igc->pdev), 4273 "igc: Failed to read reg 0x%x!\n", reg); 4274 } 4275 4276 return value; 4277 } 4278 4279 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) 4280 { 4281 struct pci_dev *pdev = adapter->pdev; 4282 struct igc_mac_info *mac = &adapter->hw.mac; 4283 4284 mac->autoneg = 0; 4285 4286 /* Make sure dplx is at most 1 bit and lsb of speed is not set 4287 * for the switch() below to work 4288 */ 4289 if ((spd & 1) || (dplx & ~1)) 4290 goto err_inval; 4291 4292 switch (spd + dplx) { 4293 case SPEED_10 + DUPLEX_HALF: 4294 mac->forced_speed_duplex = ADVERTISE_10_HALF; 4295 break; 4296 case SPEED_10 + DUPLEX_FULL: 4297 mac->forced_speed_duplex = ADVERTISE_10_FULL; 4298 break; 4299 case SPEED_100 + DUPLEX_HALF: 4300 mac->forced_speed_duplex = ADVERTISE_100_HALF; 4301 break; 4302 case SPEED_100 + DUPLEX_FULL: 4303 mac->forced_speed_duplex = ADVERTISE_100_FULL; 4304 break; 4305 case SPEED_1000 + DUPLEX_FULL: 4306 mac->autoneg = 1; 4307 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 4308 break; 4309 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 4310 goto err_inval; 4311 case SPEED_2500 + DUPLEX_FULL: 4312 mac->autoneg = 1; 4313 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 4314 break; 4315 case SPEED_2500 + DUPLEX_HALF: /* not supported */ 4316 default: 4317 goto err_inval; 4318 } 4319 4320 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 4321 adapter->hw.phy.mdix = AUTO_ALL_MODES; 4322 4323 return 0; 4324 4325 err_inval: 4326 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n"); 4327 return -EINVAL; 4328 } 4329 4330 /** 4331 * igc_probe - Device Initialization Routine 4332 * @pdev: PCI device information struct 4333 * @ent: entry in igc_pci_tbl 4334 * 4335 * Returns 0 on success, negative on failure 4336 * 4337 * igc_probe initializes an adapter identified by a pci_dev structure. 4338 * The OS initialization, configuring the adapter private structure, 4339 * and a hardware reset occur. 4340 */ 4341 static int igc_probe(struct pci_dev *pdev, 4342 const struct pci_device_id *ent) 4343 { 4344 struct igc_adapter *adapter; 4345 struct net_device *netdev; 4346 struct igc_hw *hw; 4347 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 4348 int err; 4349 4350 err = pci_enable_device_mem(pdev); 4351 if (err) 4352 return err; 4353 4354 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 4355 if (!err) { 4356 err = dma_set_coherent_mask(&pdev->dev, 4357 DMA_BIT_MASK(64)); 4358 } else { 4359 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 4360 if (err) { 4361 err = dma_set_coherent_mask(&pdev->dev, 4362 DMA_BIT_MASK(32)); 4363 if (err) { 4364 dev_err(&pdev->dev, "igc: Wrong DMA config\n"); 4365 goto err_dma; 4366 } 4367 } 4368 } 4369 4370 err = pci_request_selected_regions(pdev, 4371 pci_select_bars(pdev, 4372 IORESOURCE_MEM), 4373 igc_driver_name); 4374 if (err) 4375 goto err_pci_reg; 4376 4377 pci_enable_pcie_error_reporting(pdev); 4378 4379 pci_set_master(pdev); 4380 4381 err = -ENOMEM; 4382 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 4383 IGC_MAX_TX_QUEUES); 4384 4385 if (!netdev) 4386 goto err_alloc_etherdev; 4387 4388 SET_NETDEV_DEV(netdev, &pdev->dev); 4389 4390 pci_set_drvdata(pdev, netdev); 4391 adapter = netdev_priv(netdev); 4392 adapter->netdev = netdev; 4393 adapter->pdev = pdev; 4394 hw = &adapter->hw; 4395 hw->back = adapter; 4396 adapter->port_num = hw->bus.func; 4397 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 4398 4399 err = pci_save_state(pdev); 4400 if (err) 4401 goto err_ioremap; 4402 4403 err = -EIO; 4404 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 4405 pci_resource_len(pdev, 0)); 4406 if (!adapter->io_addr) 4407 goto err_ioremap; 4408 4409 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 4410 hw->hw_addr = adapter->io_addr; 4411 4412 netdev->netdev_ops = &igc_netdev_ops; 4413 igc_set_ethtool_ops(netdev); 4414 netdev->watchdog_timeo = 5 * HZ; 4415 4416 netdev->mem_start = pci_resource_start(pdev, 0); 4417 netdev->mem_end = pci_resource_end(pdev, 0); 4418 4419 /* PCI config space info */ 4420 hw->vendor_id = pdev->vendor; 4421 hw->device_id = pdev->device; 4422 hw->revision_id = pdev->revision; 4423 hw->subsystem_vendor_id = pdev->subsystem_vendor; 4424 hw->subsystem_device_id = pdev->subsystem_device; 4425 4426 /* Copy the default MAC and PHY function pointers */ 4427 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 4428 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 4429 4430 /* Initialize skew-specific constants */ 4431 err = ei->get_invariants(hw); 4432 if (err) 4433 goto err_sw_init; 4434 4435 /* Add supported features to the features list*/ 4436 netdev->features |= NETIF_F_RXCSUM; 4437 netdev->features |= NETIF_F_HW_CSUM; 4438 netdev->features |= NETIF_F_SCTP_CRC; 4439 4440 /* setup the private structure */ 4441 err = igc_sw_init(adapter); 4442 if (err) 4443 goto err_sw_init; 4444 4445 /* copy netdev features into list of user selectable features */ 4446 netdev->hw_features |= NETIF_F_NTUPLE; 4447 netdev->hw_features |= netdev->features; 4448 4449 /* MTU range: 68 - 9216 */ 4450 netdev->min_mtu = ETH_MIN_MTU; 4451 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 4452 4453 /* before reading the NVM, reset the controller to put the device in a 4454 * known good starting state 4455 */ 4456 hw->mac.ops.reset_hw(hw); 4457 4458 if (igc_get_flash_presence_i225(hw)) { 4459 if (hw->nvm.ops.validate(hw) < 0) { 4460 dev_err(&pdev->dev, 4461 "The NVM Checksum Is Not Valid\n"); 4462 err = -EIO; 4463 goto err_eeprom; 4464 } 4465 } 4466 4467 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 4468 /* copy the MAC address out of the NVM */ 4469 if (hw->mac.ops.read_mac_addr(hw)) 4470 dev_err(&pdev->dev, "NVM Read Error\n"); 4471 } 4472 4473 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 4474 4475 if (!is_valid_ether_addr(netdev->dev_addr)) { 4476 dev_err(&pdev->dev, "Invalid MAC Address\n"); 4477 err = -EIO; 4478 goto err_eeprom; 4479 } 4480 4481 /* configure RXPBSIZE and TXPBSIZE */ 4482 wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT); 4483 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); 4484 4485 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 4486 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 4487 4488 INIT_WORK(&adapter->reset_task, igc_reset_task); 4489 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 4490 4491 /* Initialize link properties that are user-changeable */ 4492 adapter->fc_autoneg = true; 4493 hw->mac.autoneg = true; 4494 hw->phy.autoneg_advertised = 0xaf; 4495 4496 hw->fc.requested_mode = igc_fc_default; 4497 hw->fc.current_mode = igc_fc_default; 4498 4499 /* reset the hardware with the new settings */ 4500 igc_reset(adapter); 4501 4502 /* let the f/w know that the h/w is now under the control of the 4503 * driver. 4504 */ 4505 igc_get_hw_control(adapter); 4506 4507 strncpy(netdev->name, "eth%d", IFNAMSIZ); 4508 err = register_netdev(netdev); 4509 if (err) 4510 goto err_register; 4511 4512 /* carrier off reporting is important to ethtool even BEFORE open */ 4513 netif_carrier_off(netdev); 4514 4515 /* Check if Media Autosense is enabled */ 4516 adapter->ei = *ei; 4517 4518 /* print pcie link status and MAC address */ 4519 pcie_print_link_status(pdev); 4520 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 4521 4522 return 0; 4523 4524 err_register: 4525 igc_release_hw_control(adapter); 4526 err_eeprom: 4527 if (!igc_check_reset_block(hw)) 4528 igc_reset_phy(hw); 4529 err_sw_init: 4530 igc_clear_interrupt_scheme(adapter); 4531 iounmap(adapter->io_addr); 4532 err_ioremap: 4533 free_netdev(netdev); 4534 err_alloc_etherdev: 4535 pci_release_selected_regions(pdev, 4536 pci_select_bars(pdev, IORESOURCE_MEM)); 4537 err_pci_reg: 4538 err_dma: 4539 pci_disable_device(pdev); 4540 return err; 4541 } 4542 4543 /** 4544 * igc_remove - Device Removal Routine 4545 * @pdev: PCI device information struct 4546 * 4547 * igc_remove is called by the PCI subsystem to alert the driver 4548 * that it should release a PCI device. This could be caused by a 4549 * Hot-Plug event, or because the driver is going to be removed from 4550 * memory. 4551 */ 4552 static void igc_remove(struct pci_dev *pdev) 4553 { 4554 struct net_device *netdev = pci_get_drvdata(pdev); 4555 struct igc_adapter *adapter = netdev_priv(netdev); 4556 4557 set_bit(__IGC_DOWN, &adapter->state); 4558 4559 del_timer_sync(&adapter->watchdog_timer); 4560 del_timer_sync(&adapter->phy_info_timer); 4561 4562 cancel_work_sync(&adapter->reset_task); 4563 cancel_work_sync(&adapter->watchdog_task); 4564 4565 /* Release control of h/w to f/w. If f/w is AMT enabled, this 4566 * would have already happened in close and is redundant. 4567 */ 4568 igc_release_hw_control(adapter); 4569 unregister_netdev(netdev); 4570 4571 igc_clear_interrupt_scheme(adapter); 4572 pci_iounmap(pdev, adapter->io_addr); 4573 pci_release_mem_regions(pdev); 4574 4575 kfree(adapter->mac_table); 4576 free_netdev(netdev); 4577 4578 pci_disable_pcie_error_reporting(pdev); 4579 4580 pci_disable_device(pdev); 4581 } 4582 4583 static struct pci_driver igc_driver = { 4584 .name = igc_driver_name, 4585 .id_table = igc_pci_tbl, 4586 .probe = igc_probe, 4587 .remove = igc_remove, 4588 }; 4589 4590 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 4591 const u32 max_rss_queues) 4592 { 4593 /* Determine if we need to pair queues. */ 4594 /* If rss_queues > half of max_rss_queues, pair the queues in 4595 * order to conserve interrupts due to limited supply. 4596 */ 4597 if (adapter->rss_queues > (max_rss_queues / 2)) 4598 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 4599 else 4600 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 4601 } 4602 4603 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 4604 { 4605 unsigned int max_rss_queues; 4606 4607 /* Determine the maximum number of RSS queues supported. */ 4608 max_rss_queues = IGC_MAX_RX_QUEUES; 4609 4610 return max_rss_queues; 4611 } 4612 4613 static void igc_init_queue_configuration(struct igc_adapter *adapter) 4614 { 4615 u32 max_rss_queues; 4616 4617 max_rss_queues = igc_get_max_rss_queues(adapter); 4618 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 4619 4620 igc_set_flag_queue_pairs(adapter, max_rss_queues); 4621 } 4622 4623 /** 4624 * igc_sw_init - Initialize general software structures (struct igc_adapter) 4625 * @adapter: board private structure to initialize 4626 * 4627 * igc_sw_init initializes the Adapter private data structure. 4628 * Fields are initialized based on PCI device information and 4629 * OS network device settings (MTU size). 4630 */ 4631 static int igc_sw_init(struct igc_adapter *adapter) 4632 { 4633 struct net_device *netdev = adapter->netdev; 4634 struct pci_dev *pdev = adapter->pdev; 4635 struct igc_hw *hw = &adapter->hw; 4636 4637 int size = sizeof(struct igc_mac_addr) * hw->mac.rar_entry_count; 4638 4639 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 4640 4641 /* set default ring sizes */ 4642 adapter->tx_ring_count = IGC_DEFAULT_TXD; 4643 adapter->rx_ring_count = IGC_DEFAULT_RXD; 4644 4645 /* set default ITR values */ 4646 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 4647 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 4648 4649 /* set default work limits */ 4650 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 4651 4652 /* adjust max frame to be at least the size of a standard frame */ 4653 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 4654 VLAN_HLEN; 4655 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 4656 4657 spin_lock_init(&adapter->nfc_lock); 4658 spin_lock_init(&adapter->stats64_lock); 4659 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 4660 adapter->flags |= IGC_FLAG_HAS_MSIX; 4661 4662 adapter->mac_table = kzalloc(size, GFP_ATOMIC); 4663 if (!adapter->mac_table) 4664 return -ENOMEM; 4665 4666 igc_init_queue_configuration(adapter); 4667 4668 /* This call may decrease the number of queues */ 4669 if (igc_init_interrupt_scheme(adapter, true)) { 4670 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 4671 return -ENOMEM; 4672 } 4673 4674 /* Explicitly disable IRQ since the NIC can be in any state. */ 4675 igc_irq_disable(adapter); 4676 4677 set_bit(__IGC_DOWN, &adapter->state); 4678 4679 return 0; 4680 } 4681 4682 /** 4683 * igc_reinit_queues - return error 4684 * @adapter: pointer to adapter structure 4685 */ 4686 int igc_reinit_queues(struct igc_adapter *adapter) 4687 { 4688 struct net_device *netdev = adapter->netdev; 4689 struct pci_dev *pdev = adapter->pdev; 4690 int err = 0; 4691 4692 if (netif_running(netdev)) 4693 igc_close(netdev); 4694 4695 igc_reset_interrupt_capability(adapter); 4696 4697 if (igc_init_interrupt_scheme(adapter, true)) { 4698 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 4699 return -ENOMEM; 4700 } 4701 4702 if (netif_running(netdev)) 4703 err = igc_open(netdev); 4704 4705 return err; 4706 } 4707 4708 /** 4709 * igc_get_hw_dev - return device 4710 * @hw: pointer to hardware structure 4711 * 4712 * used by hardware layer to print debugging information 4713 */ 4714 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 4715 { 4716 struct igc_adapter *adapter = hw->back; 4717 4718 return adapter->netdev; 4719 } 4720 4721 /** 4722 * igc_init_module - Driver Registration Routine 4723 * 4724 * igc_init_module is the first routine called when the driver is 4725 * loaded. All it does is register with the PCI subsystem. 4726 */ 4727 static int __init igc_init_module(void) 4728 { 4729 int ret; 4730 4731 pr_info("%s - version %s\n", 4732 igc_driver_string, igc_driver_version); 4733 4734 pr_info("%s\n", igc_copyright); 4735 4736 ret = pci_register_driver(&igc_driver); 4737 return ret; 4738 } 4739 4740 module_init(igc_init_module); 4741 4742 /** 4743 * igc_exit_module - Driver Exit Cleanup Routine 4744 * 4745 * igc_exit_module is called just before the driver is removed 4746 * from memory. 4747 */ 4748 static void __exit igc_exit_module(void) 4749 { 4750 pci_unregister_driver(&igc_driver); 4751 } 4752 4753 module_exit(igc_exit_module); 4754 /* igc_main.c */ 4755