1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/aer.h> 8 #include <linux/tcp.h> 9 #include <linux/udp.h> 10 #include <linux/ip.h> 11 #include <linux/pm_runtime.h> 12 #include <net/pkt_sched.h> 13 14 #include <net/ipv6.h> 15 16 #include "igc.h" 17 #include "igc_hw.h" 18 #include "igc_tsn.h" 19 20 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 21 22 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 23 24 static int debug = -1; 25 26 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 27 MODULE_DESCRIPTION(DRV_SUMMARY); 28 MODULE_LICENSE("GPL v2"); 29 module_param(debug, int, 0); 30 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 31 32 char igc_driver_name[] = "igc"; 33 static const char igc_driver_string[] = DRV_SUMMARY; 34 static const char igc_copyright[] = 35 "Copyright(c) 2018 Intel Corporation."; 36 37 static const struct igc_info *igc_info_tbl[] = { 38 [board_base] = &igc_base_info, 39 }; 40 41 static const struct pci_device_id igc_pci_tbl[] = { 42 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base }, 43 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base }, 44 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, 45 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, 46 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, 47 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base }, 48 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), board_base }, 49 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base }, 50 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base }, 51 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base }, 52 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base }, 53 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base }, 54 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base }, 55 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base }, 56 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, 57 /* required last entry */ 58 {0, } 59 }; 60 61 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 62 63 enum latency_range { 64 lowest_latency = 0, 65 low_latency = 1, 66 bulk_latency = 2, 67 latency_invalid = 255 68 }; 69 70 void igc_reset(struct igc_adapter *adapter) 71 { 72 struct net_device *dev = adapter->netdev; 73 struct igc_hw *hw = &adapter->hw; 74 struct igc_fc_info *fc = &hw->fc; 75 u32 pba, hwm; 76 77 /* Repartition PBA for greater than 9k MTU if required */ 78 pba = IGC_PBA_34K; 79 80 /* flow control settings 81 * The high water mark must be low enough to fit one full frame 82 * after transmitting the pause frame. As such we must have enough 83 * space to allow for us to complete our current transmit and then 84 * receive the frame that is in progress from the link partner. 85 * Set it to: 86 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 87 */ 88 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 89 90 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 91 fc->low_water = fc->high_water - 16; 92 fc->pause_time = 0xFFFF; 93 fc->send_xon = 1; 94 fc->current_mode = fc->requested_mode; 95 96 hw->mac.ops.reset_hw(hw); 97 98 if (hw->mac.ops.init_hw(hw)) 99 netdev_err(dev, "Error on hardware initialization\n"); 100 101 /* Re-establish EEE setting */ 102 igc_set_eee_i225(hw, true, true, true); 103 104 if (!netif_running(adapter->netdev)) 105 igc_power_down_phy_copper_base(&adapter->hw); 106 107 /* Re-enable PTP, where applicable. */ 108 igc_ptp_reset(adapter); 109 110 /* Re-enable TSN offloading, where applicable. */ 111 igc_tsn_offload_apply(adapter); 112 113 igc_get_phy_info(hw); 114 } 115 116 /** 117 * igc_power_up_link - Power up the phy link 118 * @adapter: address of board private structure 119 */ 120 static void igc_power_up_link(struct igc_adapter *adapter) 121 { 122 igc_reset_phy(&adapter->hw); 123 124 igc_power_up_phy_copper(&adapter->hw); 125 126 igc_setup_link(&adapter->hw); 127 } 128 129 /** 130 * igc_release_hw_control - release control of the h/w to f/w 131 * @adapter: address of board private structure 132 * 133 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 134 * For ASF and Pass Through versions of f/w this means that the 135 * driver is no longer loaded. 136 */ 137 static void igc_release_hw_control(struct igc_adapter *adapter) 138 { 139 struct igc_hw *hw = &adapter->hw; 140 u32 ctrl_ext; 141 142 /* Let firmware take over control of h/w */ 143 ctrl_ext = rd32(IGC_CTRL_EXT); 144 wr32(IGC_CTRL_EXT, 145 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 146 } 147 148 /** 149 * igc_get_hw_control - get control of the h/w from f/w 150 * @adapter: address of board private structure 151 * 152 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 153 * For ASF and Pass Through versions of f/w this means that 154 * the driver is loaded. 155 */ 156 static void igc_get_hw_control(struct igc_adapter *adapter) 157 { 158 struct igc_hw *hw = &adapter->hw; 159 u32 ctrl_ext; 160 161 /* Let firmware know the driver has taken over */ 162 ctrl_ext = rd32(IGC_CTRL_EXT); 163 wr32(IGC_CTRL_EXT, 164 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 165 } 166 167 /** 168 * igc_clean_tx_ring - Free Tx Buffers 169 * @tx_ring: ring to be cleaned 170 */ 171 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 172 { 173 u16 i = tx_ring->next_to_clean; 174 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 175 176 while (i != tx_ring->next_to_use) { 177 union igc_adv_tx_desc *eop_desc, *tx_desc; 178 179 /* Free all the Tx ring sk_buffs */ 180 dev_kfree_skb_any(tx_buffer->skb); 181 182 /* unmap skb header data */ 183 dma_unmap_single(tx_ring->dev, 184 dma_unmap_addr(tx_buffer, dma), 185 dma_unmap_len(tx_buffer, len), 186 DMA_TO_DEVICE); 187 188 /* check for eop_desc to determine the end of the packet */ 189 eop_desc = tx_buffer->next_to_watch; 190 tx_desc = IGC_TX_DESC(tx_ring, i); 191 192 /* unmap remaining buffers */ 193 while (tx_desc != eop_desc) { 194 tx_buffer++; 195 tx_desc++; 196 i++; 197 if (unlikely(i == tx_ring->count)) { 198 i = 0; 199 tx_buffer = tx_ring->tx_buffer_info; 200 tx_desc = IGC_TX_DESC(tx_ring, 0); 201 } 202 203 /* unmap any remaining paged data */ 204 if (dma_unmap_len(tx_buffer, len)) 205 dma_unmap_page(tx_ring->dev, 206 dma_unmap_addr(tx_buffer, dma), 207 dma_unmap_len(tx_buffer, len), 208 DMA_TO_DEVICE); 209 } 210 211 /* move us one more past the eop_desc for start of next pkt */ 212 tx_buffer++; 213 i++; 214 if (unlikely(i == tx_ring->count)) { 215 i = 0; 216 tx_buffer = tx_ring->tx_buffer_info; 217 } 218 } 219 220 /* reset BQL for queue */ 221 netdev_tx_reset_queue(txring_txq(tx_ring)); 222 223 /* reset next_to_use and next_to_clean */ 224 tx_ring->next_to_use = 0; 225 tx_ring->next_to_clean = 0; 226 } 227 228 /** 229 * igc_free_tx_resources - Free Tx Resources per Queue 230 * @tx_ring: Tx descriptor ring for a specific queue 231 * 232 * Free all transmit software resources 233 */ 234 void igc_free_tx_resources(struct igc_ring *tx_ring) 235 { 236 igc_clean_tx_ring(tx_ring); 237 238 vfree(tx_ring->tx_buffer_info); 239 tx_ring->tx_buffer_info = NULL; 240 241 /* if not set, then don't free */ 242 if (!tx_ring->desc) 243 return; 244 245 dma_free_coherent(tx_ring->dev, tx_ring->size, 246 tx_ring->desc, tx_ring->dma); 247 248 tx_ring->desc = NULL; 249 } 250 251 /** 252 * igc_free_all_tx_resources - Free Tx Resources for All Queues 253 * @adapter: board private structure 254 * 255 * Free all transmit software resources 256 */ 257 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 258 { 259 int i; 260 261 for (i = 0; i < adapter->num_tx_queues; i++) 262 igc_free_tx_resources(adapter->tx_ring[i]); 263 } 264 265 /** 266 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 267 * @adapter: board private structure 268 */ 269 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 270 { 271 int i; 272 273 for (i = 0; i < adapter->num_tx_queues; i++) 274 if (adapter->tx_ring[i]) 275 igc_clean_tx_ring(adapter->tx_ring[i]); 276 } 277 278 /** 279 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 280 * @tx_ring: tx descriptor ring (for a specific queue) to setup 281 * 282 * Return 0 on success, negative on failure 283 */ 284 int igc_setup_tx_resources(struct igc_ring *tx_ring) 285 { 286 struct net_device *ndev = tx_ring->netdev; 287 struct device *dev = tx_ring->dev; 288 int size = 0; 289 290 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 291 tx_ring->tx_buffer_info = vzalloc(size); 292 if (!tx_ring->tx_buffer_info) 293 goto err; 294 295 /* round up to nearest 4K */ 296 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 297 tx_ring->size = ALIGN(tx_ring->size, 4096); 298 299 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 300 &tx_ring->dma, GFP_KERNEL); 301 302 if (!tx_ring->desc) 303 goto err; 304 305 tx_ring->next_to_use = 0; 306 tx_ring->next_to_clean = 0; 307 308 return 0; 309 310 err: 311 vfree(tx_ring->tx_buffer_info); 312 netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n"); 313 return -ENOMEM; 314 } 315 316 /** 317 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 318 * @adapter: board private structure 319 * 320 * Return 0 on success, negative on failure 321 */ 322 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 323 { 324 struct net_device *dev = adapter->netdev; 325 int i, err = 0; 326 327 for (i = 0; i < adapter->num_tx_queues; i++) { 328 err = igc_setup_tx_resources(adapter->tx_ring[i]); 329 if (err) { 330 netdev_err(dev, "Error on Tx queue %u setup\n", i); 331 for (i--; i >= 0; i--) 332 igc_free_tx_resources(adapter->tx_ring[i]); 333 break; 334 } 335 } 336 337 return err; 338 } 339 340 /** 341 * igc_clean_rx_ring - Free Rx Buffers per Queue 342 * @rx_ring: ring to free buffers from 343 */ 344 static void igc_clean_rx_ring(struct igc_ring *rx_ring) 345 { 346 u16 i = rx_ring->next_to_clean; 347 348 dev_kfree_skb(rx_ring->skb); 349 rx_ring->skb = NULL; 350 351 /* Free all the Rx ring sk_buffs */ 352 while (i != rx_ring->next_to_alloc) { 353 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 354 355 /* Invalidate cache lines that may have been written to by 356 * device so that we avoid corrupting memory. 357 */ 358 dma_sync_single_range_for_cpu(rx_ring->dev, 359 buffer_info->dma, 360 buffer_info->page_offset, 361 igc_rx_bufsz(rx_ring), 362 DMA_FROM_DEVICE); 363 364 /* free resources associated with mapping */ 365 dma_unmap_page_attrs(rx_ring->dev, 366 buffer_info->dma, 367 igc_rx_pg_size(rx_ring), 368 DMA_FROM_DEVICE, 369 IGC_RX_DMA_ATTR); 370 __page_frag_cache_drain(buffer_info->page, 371 buffer_info->pagecnt_bias); 372 373 i++; 374 if (i == rx_ring->count) 375 i = 0; 376 } 377 378 rx_ring->next_to_alloc = 0; 379 rx_ring->next_to_clean = 0; 380 rx_ring->next_to_use = 0; 381 } 382 383 /** 384 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 385 * @adapter: board private structure 386 */ 387 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 388 { 389 int i; 390 391 for (i = 0; i < adapter->num_rx_queues; i++) 392 if (adapter->rx_ring[i]) 393 igc_clean_rx_ring(adapter->rx_ring[i]); 394 } 395 396 /** 397 * igc_free_rx_resources - Free Rx Resources 398 * @rx_ring: ring to clean the resources from 399 * 400 * Free all receive software resources 401 */ 402 void igc_free_rx_resources(struct igc_ring *rx_ring) 403 { 404 igc_clean_rx_ring(rx_ring); 405 406 vfree(rx_ring->rx_buffer_info); 407 rx_ring->rx_buffer_info = NULL; 408 409 /* if not set, then don't free */ 410 if (!rx_ring->desc) 411 return; 412 413 dma_free_coherent(rx_ring->dev, rx_ring->size, 414 rx_ring->desc, rx_ring->dma); 415 416 rx_ring->desc = NULL; 417 } 418 419 /** 420 * igc_free_all_rx_resources - Free Rx Resources for All Queues 421 * @adapter: board private structure 422 * 423 * Free all receive software resources 424 */ 425 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 426 { 427 int i; 428 429 for (i = 0; i < adapter->num_rx_queues; i++) 430 igc_free_rx_resources(adapter->rx_ring[i]); 431 } 432 433 /** 434 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 435 * @rx_ring: rx descriptor ring (for a specific queue) to setup 436 * 437 * Returns 0 on success, negative on failure 438 */ 439 int igc_setup_rx_resources(struct igc_ring *rx_ring) 440 { 441 struct net_device *ndev = rx_ring->netdev; 442 struct device *dev = rx_ring->dev; 443 int size, desc_len; 444 445 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 446 rx_ring->rx_buffer_info = vzalloc(size); 447 if (!rx_ring->rx_buffer_info) 448 goto err; 449 450 desc_len = sizeof(union igc_adv_rx_desc); 451 452 /* Round up to nearest 4K */ 453 rx_ring->size = rx_ring->count * desc_len; 454 rx_ring->size = ALIGN(rx_ring->size, 4096); 455 456 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 457 &rx_ring->dma, GFP_KERNEL); 458 459 if (!rx_ring->desc) 460 goto err; 461 462 rx_ring->next_to_alloc = 0; 463 rx_ring->next_to_clean = 0; 464 rx_ring->next_to_use = 0; 465 466 return 0; 467 468 err: 469 vfree(rx_ring->rx_buffer_info); 470 rx_ring->rx_buffer_info = NULL; 471 netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n"); 472 return -ENOMEM; 473 } 474 475 /** 476 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 477 * (Descriptors) for all queues 478 * @adapter: board private structure 479 * 480 * Return 0 on success, negative on failure 481 */ 482 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 483 { 484 struct net_device *dev = adapter->netdev; 485 int i, err = 0; 486 487 for (i = 0; i < adapter->num_rx_queues; i++) { 488 err = igc_setup_rx_resources(adapter->rx_ring[i]); 489 if (err) { 490 netdev_err(dev, "Error on Rx queue %u setup\n", i); 491 for (i--; i >= 0; i--) 492 igc_free_rx_resources(adapter->rx_ring[i]); 493 break; 494 } 495 } 496 497 return err; 498 } 499 500 /** 501 * igc_configure_rx_ring - Configure a receive ring after Reset 502 * @adapter: board private structure 503 * @ring: receive ring to be configured 504 * 505 * Configure the Rx unit of the MAC after a reset. 506 */ 507 static void igc_configure_rx_ring(struct igc_adapter *adapter, 508 struct igc_ring *ring) 509 { 510 struct igc_hw *hw = &adapter->hw; 511 union igc_adv_rx_desc *rx_desc; 512 int reg_idx = ring->reg_idx; 513 u32 srrctl = 0, rxdctl = 0; 514 u64 rdba = ring->dma; 515 516 /* disable the queue */ 517 wr32(IGC_RXDCTL(reg_idx), 0); 518 519 /* Set DMA base address registers */ 520 wr32(IGC_RDBAL(reg_idx), 521 rdba & 0x00000000ffffffffULL); 522 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 523 wr32(IGC_RDLEN(reg_idx), 524 ring->count * sizeof(union igc_adv_rx_desc)); 525 526 /* initialize head and tail */ 527 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 528 wr32(IGC_RDH(reg_idx), 0); 529 writel(0, ring->tail); 530 531 /* reset next-to- use/clean to place SW in sync with hardware */ 532 ring->next_to_clean = 0; 533 ring->next_to_use = 0; 534 535 /* set descriptor configuration */ 536 srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; 537 if (ring_uses_large_buffer(ring)) 538 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 539 else 540 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 541 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 542 543 wr32(IGC_SRRCTL(reg_idx), srrctl); 544 545 rxdctl |= IGC_RX_PTHRESH; 546 rxdctl |= IGC_RX_HTHRESH << 8; 547 rxdctl |= IGC_RX_WTHRESH << 16; 548 549 /* initialize rx_buffer_info */ 550 memset(ring->rx_buffer_info, 0, 551 sizeof(struct igc_rx_buffer) * ring->count); 552 553 /* initialize Rx descriptor 0 */ 554 rx_desc = IGC_RX_DESC(ring, 0); 555 rx_desc->wb.upper.length = 0; 556 557 /* enable receive descriptor fetching */ 558 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 559 560 wr32(IGC_RXDCTL(reg_idx), rxdctl); 561 } 562 563 /** 564 * igc_configure_rx - Configure receive Unit after Reset 565 * @adapter: board private structure 566 * 567 * Configure the Rx unit of the MAC after a reset. 568 */ 569 static void igc_configure_rx(struct igc_adapter *adapter) 570 { 571 int i; 572 573 /* Setup the HW Rx Head and Tail Descriptor Pointers and 574 * the Base and Length of the Rx Descriptor Ring 575 */ 576 for (i = 0; i < adapter->num_rx_queues; i++) 577 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 578 } 579 580 /** 581 * igc_configure_tx_ring - Configure transmit ring after Reset 582 * @adapter: board private structure 583 * @ring: tx ring to configure 584 * 585 * Configure a transmit ring after a reset. 586 */ 587 static void igc_configure_tx_ring(struct igc_adapter *adapter, 588 struct igc_ring *ring) 589 { 590 struct igc_hw *hw = &adapter->hw; 591 int reg_idx = ring->reg_idx; 592 u64 tdba = ring->dma; 593 u32 txdctl = 0; 594 595 /* disable the queue */ 596 wr32(IGC_TXDCTL(reg_idx), 0); 597 wrfl(); 598 mdelay(10); 599 600 wr32(IGC_TDLEN(reg_idx), 601 ring->count * sizeof(union igc_adv_tx_desc)); 602 wr32(IGC_TDBAL(reg_idx), 603 tdba & 0x00000000ffffffffULL); 604 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 605 606 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 607 wr32(IGC_TDH(reg_idx), 0); 608 writel(0, ring->tail); 609 610 txdctl |= IGC_TX_PTHRESH; 611 txdctl |= IGC_TX_HTHRESH << 8; 612 txdctl |= IGC_TX_WTHRESH << 16; 613 614 txdctl |= IGC_TXDCTL_QUEUE_ENABLE; 615 wr32(IGC_TXDCTL(reg_idx), txdctl); 616 } 617 618 /** 619 * igc_configure_tx - Configure transmit Unit after Reset 620 * @adapter: board private structure 621 * 622 * Configure the Tx unit of the MAC after a reset. 623 */ 624 static void igc_configure_tx(struct igc_adapter *adapter) 625 { 626 int i; 627 628 for (i = 0; i < adapter->num_tx_queues; i++) 629 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 630 } 631 632 /** 633 * igc_setup_mrqc - configure the multiple receive queue control registers 634 * @adapter: Board private structure 635 */ 636 static void igc_setup_mrqc(struct igc_adapter *adapter) 637 { 638 struct igc_hw *hw = &adapter->hw; 639 u32 j, num_rx_queues; 640 u32 mrqc, rxcsum; 641 u32 rss_key[10]; 642 643 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 644 for (j = 0; j < 10; j++) 645 wr32(IGC_RSSRK(j), rss_key[j]); 646 647 num_rx_queues = adapter->rss_queues; 648 649 if (adapter->rss_indir_tbl_init != num_rx_queues) { 650 for (j = 0; j < IGC_RETA_SIZE; j++) 651 adapter->rss_indir_tbl[j] = 652 (j * num_rx_queues) / IGC_RETA_SIZE; 653 adapter->rss_indir_tbl_init = num_rx_queues; 654 } 655 igc_write_rss_indir_tbl(adapter); 656 657 /* Disable raw packet checksumming so that RSS hash is placed in 658 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 659 * offloads as they are enabled by default 660 */ 661 rxcsum = rd32(IGC_RXCSUM); 662 rxcsum |= IGC_RXCSUM_PCSD; 663 664 /* Enable Receive Checksum Offload for SCTP */ 665 rxcsum |= IGC_RXCSUM_CRCOFL; 666 667 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 668 wr32(IGC_RXCSUM, rxcsum); 669 670 /* Generate RSS hash based on packet types, TCP/UDP 671 * port numbers and/or IPv4/v6 src and dst addresses 672 */ 673 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 674 IGC_MRQC_RSS_FIELD_IPV4_TCP | 675 IGC_MRQC_RSS_FIELD_IPV6 | 676 IGC_MRQC_RSS_FIELD_IPV6_TCP | 677 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 678 679 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 680 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 681 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 682 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 683 684 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 685 686 wr32(IGC_MRQC, mrqc); 687 } 688 689 /** 690 * igc_setup_rctl - configure the receive control registers 691 * @adapter: Board private structure 692 */ 693 static void igc_setup_rctl(struct igc_adapter *adapter) 694 { 695 struct igc_hw *hw = &adapter->hw; 696 u32 rctl; 697 698 rctl = rd32(IGC_RCTL); 699 700 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 701 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 702 703 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 704 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 705 706 /* enable stripping of CRC. Newer features require 707 * that the HW strips the CRC. 708 */ 709 rctl |= IGC_RCTL_SECRC; 710 711 /* disable store bad packets and clear size bits. */ 712 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 713 714 /* enable LPE to allow for reception of jumbo frames */ 715 rctl |= IGC_RCTL_LPE; 716 717 /* disable queue 0 to prevent tail write w/o re-config */ 718 wr32(IGC_RXDCTL(0), 0); 719 720 /* This is useful for sniffing bad packets. */ 721 if (adapter->netdev->features & NETIF_F_RXALL) { 722 /* UPE and MPE will be handled by normal PROMISC logic 723 * in set_rx_mode 724 */ 725 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 726 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 727 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 728 729 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 730 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 731 } 732 733 wr32(IGC_RCTL, rctl); 734 } 735 736 /** 737 * igc_setup_tctl - configure the transmit control registers 738 * @adapter: Board private structure 739 */ 740 static void igc_setup_tctl(struct igc_adapter *adapter) 741 { 742 struct igc_hw *hw = &adapter->hw; 743 u32 tctl; 744 745 /* disable queue 0 which icould be enabled by default */ 746 wr32(IGC_TXDCTL(0), 0); 747 748 /* Program the Transmit Control Register */ 749 tctl = rd32(IGC_TCTL); 750 tctl &= ~IGC_TCTL_CT; 751 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 752 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 753 754 /* Enable transmits */ 755 tctl |= IGC_TCTL_EN; 756 757 wr32(IGC_TCTL, tctl); 758 } 759 760 /** 761 * igc_set_mac_filter_hw() - Set MAC address filter in hardware 762 * @adapter: Pointer to adapter where the filter should be set 763 * @index: Filter index 764 * @type: MAC address filter type (source or destination) 765 * @addr: MAC address 766 * @queue: If non-negative, queue assignment feature is enabled and frames 767 * matching the filter are enqueued onto 'queue'. Otherwise, queue 768 * assignment is disabled. 769 */ 770 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, 771 enum igc_mac_filter_type type, 772 const u8 *addr, int queue) 773 { 774 struct net_device *dev = adapter->netdev; 775 struct igc_hw *hw = &adapter->hw; 776 u32 ral, rah; 777 778 if (WARN_ON(index >= hw->mac.rar_entry_count)) 779 return; 780 781 ral = le32_to_cpup((__le32 *)(addr)); 782 rah = le16_to_cpup((__le16 *)(addr + 4)); 783 784 if (type == IGC_MAC_FILTER_TYPE_SRC) { 785 rah &= ~IGC_RAH_ASEL_MASK; 786 rah |= IGC_RAH_ASEL_SRC_ADDR; 787 } 788 789 if (queue >= 0) { 790 rah &= ~IGC_RAH_QSEL_MASK; 791 rah |= (queue << IGC_RAH_QSEL_SHIFT); 792 rah |= IGC_RAH_QSEL_ENABLE; 793 } 794 795 rah |= IGC_RAH_AV; 796 797 wr32(IGC_RAL(index), ral); 798 wr32(IGC_RAH(index), rah); 799 800 netdev_dbg(dev, "MAC address filter set in HW: index %d", index); 801 } 802 803 /** 804 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware 805 * @adapter: Pointer to adapter where the filter should be cleared 806 * @index: Filter index 807 */ 808 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) 809 { 810 struct net_device *dev = adapter->netdev; 811 struct igc_hw *hw = &adapter->hw; 812 813 if (WARN_ON(index >= hw->mac.rar_entry_count)) 814 return; 815 816 wr32(IGC_RAL(index), 0); 817 wr32(IGC_RAH(index), 0); 818 819 netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index); 820 } 821 822 /* Set default MAC address for the PF in the first RAR entry */ 823 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 824 { 825 struct net_device *dev = adapter->netdev; 826 u8 *addr = adapter->hw.mac.addr; 827 828 netdev_dbg(dev, "Set default MAC address filter: address %pM", addr); 829 830 igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1); 831 } 832 833 /** 834 * igc_set_mac - Change the Ethernet Address of the NIC 835 * @netdev: network interface device structure 836 * @p: pointer to an address structure 837 * 838 * Returns 0 on success, negative on failure 839 */ 840 static int igc_set_mac(struct net_device *netdev, void *p) 841 { 842 struct igc_adapter *adapter = netdev_priv(netdev); 843 struct igc_hw *hw = &adapter->hw; 844 struct sockaddr *addr = p; 845 846 if (!is_valid_ether_addr(addr->sa_data)) 847 return -EADDRNOTAVAIL; 848 849 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 850 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 851 852 /* set the correct pool for the new PF MAC address in entry 0 */ 853 igc_set_default_mac_filter(adapter); 854 855 return 0; 856 } 857 858 /** 859 * igc_write_mc_addr_list - write multicast addresses to MTA 860 * @netdev: network interface device structure 861 * 862 * Writes multicast address list to the MTA hash table. 863 * Returns: -ENOMEM on failure 864 * 0 on no addresses written 865 * X on writing X addresses to MTA 866 **/ 867 static int igc_write_mc_addr_list(struct net_device *netdev) 868 { 869 struct igc_adapter *adapter = netdev_priv(netdev); 870 struct igc_hw *hw = &adapter->hw; 871 struct netdev_hw_addr *ha; 872 u8 *mta_list; 873 int i; 874 875 if (netdev_mc_empty(netdev)) { 876 /* nothing to program, so clear mc list */ 877 igc_update_mc_addr_list(hw, NULL, 0); 878 return 0; 879 } 880 881 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 882 if (!mta_list) 883 return -ENOMEM; 884 885 /* The shared function expects a packed array of only addresses. */ 886 i = 0; 887 netdev_for_each_mc_addr(ha, netdev) 888 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 889 890 igc_update_mc_addr_list(hw, mta_list, i); 891 kfree(mta_list); 892 893 return netdev_mc_count(netdev); 894 } 895 896 static __le32 igc_tx_launchtime(struct igc_adapter *adapter, ktime_t txtime) 897 { 898 ktime_t cycle_time = adapter->cycle_time; 899 ktime_t base_time = adapter->base_time; 900 u32 launchtime; 901 902 /* FIXME: when using ETF together with taprio, we may have a 903 * case where 'delta' is larger than the cycle_time, this may 904 * cause problems if we don't read the current value of 905 * IGC_BASET, as the value writen into the launchtime 906 * descriptor field may be misinterpreted. 907 */ 908 div_s64_rem(ktime_sub_ns(txtime, base_time), cycle_time, &launchtime); 909 910 return cpu_to_le32(launchtime); 911 } 912 913 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, 914 struct igc_tx_buffer *first, 915 u32 vlan_macip_lens, u32 type_tucmd, 916 u32 mss_l4len_idx) 917 { 918 struct igc_adv_tx_context_desc *context_desc; 919 u16 i = tx_ring->next_to_use; 920 921 context_desc = IGC_TX_CTXTDESC(tx_ring, i); 922 923 i++; 924 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 925 926 /* set bits to identify this as an advanced context descriptor */ 927 type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 928 929 /* For i225, context index must be unique per ring. */ 930 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 931 mss_l4len_idx |= tx_ring->reg_idx << 4; 932 933 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 934 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 935 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 936 937 /* We assume there is always a valid Tx time available. Invalid times 938 * should have been handled by the upper layers. 939 */ 940 if (tx_ring->launchtime_enable) { 941 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 942 ktime_t txtime = first->skb->tstamp; 943 944 first->skb->tstamp = ktime_set(0, 0); 945 context_desc->launch_time = igc_tx_launchtime(adapter, 946 txtime); 947 } else { 948 context_desc->launch_time = 0; 949 } 950 } 951 952 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first) 953 { 954 struct sk_buff *skb = first->skb; 955 u32 vlan_macip_lens = 0; 956 u32 type_tucmd = 0; 957 958 if (skb->ip_summed != CHECKSUM_PARTIAL) { 959 csum_failed: 960 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && 961 !tx_ring->launchtime_enable) 962 return; 963 goto no_csum; 964 } 965 966 switch (skb->csum_offset) { 967 case offsetof(struct tcphdr, check): 968 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 969 fallthrough; 970 case offsetof(struct udphdr, check): 971 break; 972 case offsetof(struct sctphdr, checksum): 973 /* validate that this is actually an SCTP request */ 974 if (skb_csum_is_sctp(skb)) { 975 type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; 976 break; 977 } 978 fallthrough; 979 default: 980 skb_checksum_help(skb); 981 goto csum_failed; 982 } 983 984 /* update TX checksum flag */ 985 first->tx_flags |= IGC_TX_FLAGS_CSUM; 986 vlan_macip_lens = skb_checksum_start_offset(skb) - 987 skb_network_offset(skb); 988 no_csum: 989 vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; 990 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 991 992 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0); 993 } 994 995 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 996 { 997 struct net_device *netdev = tx_ring->netdev; 998 999 netif_stop_subqueue(netdev, tx_ring->queue_index); 1000 1001 /* memory barriier comment */ 1002 smp_mb(); 1003 1004 /* We need to check again in a case another CPU has just 1005 * made room available. 1006 */ 1007 if (igc_desc_unused(tx_ring) < size) 1008 return -EBUSY; 1009 1010 /* A reprieve! */ 1011 netif_wake_subqueue(netdev, tx_ring->queue_index); 1012 1013 u64_stats_update_begin(&tx_ring->tx_syncp2); 1014 tx_ring->tx_stats.restart_queue2++; 1015 u64_stats_update_end(&tx_ring->tx_syncp2); 1016 1017 return 0; 1018 } 1019 1020 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1021 { 1022 if (igc_desc_unused(tx_ring) >= size) 1023 return 0; 1024 return __igc_maybe_stop_tx(tx_ring, size); 1025 } 1026 1027 #define IGC_SET_FLAG(_input, _flag, _result) \ 1028 (((_flag) <= (_result)) ? \ 1029 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ 1030 ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) 1031 1032 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 1033 { 1034 /* set type for advanced descriptor with frame checksum insertion */ 1035 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 1036 IGC_ADVTXD_DCMD_DEXT | 1037 IGC_ADVTXD_DCMD_IFCS; 1038 1039 /* set segmentation bits for TSO */ 1040 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, 1041 (IGC_ADVTXD_DCMD_TSE)); 1042 1043 /* set timestamp bit if present */ 1044 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, 1045 (IGC_ADVTXD_MAC_TSTAMP)); 1046 1047 return cmd_type; 1048 } 1049 1050 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 1051 union igc_adv_tx_desc *tx_desc, 1052 u32 tx_flags, unsigned int paylen) 1053 { 1054 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 1055 1056 /* insert L4 checksum */ 1057 olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) * 1058 ((IGC_TXD_POPTS_TXSM << 8) / 1059 IGC_TX_FLAGS_CSUM); 1060 1061 /* insert IPv4 checksum */ 1062 olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) * 1063 (((IGC_TXD_POPTS_IXSM << 8)) / 1064 IGC_TX_FLAGS_IPV4); 1065 1066 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 1067 } 1068 1069 static int igc_tx_map(struct igc_ring *tx_ring, 1070 struct igc_tx_buffer *first, 1071 const u8 hdr_len) 1072 { 1073 struct sk_buff *skb = first->skb; 1074 struct igc_tx_buffer *tx_buffer; 1075 union igc_adv_tx_desc *tx_desc; 1076 u32 tx_flags = first->tx_flags; 1077 skb_frag_t *frag; 1078 u16 i = tx_ring->next_to_use; 1079 unsigned int data_len, size; 1080 dma_addr_t dma; 1081 u32 cmd_type = igc_tx_cmd_type(skb, tx_flags); 1082 1083 tx_desc = IGC_TX_DESC(tx_ring, i); 1084 1085 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 1086 1087 size = skb_headlen(skb); 1088 data_len = skb->data_len; 1089 1090 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1091 1092 tx_buffer = first; 1093 1094 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1095 if (dma_mapping_error(tx_ring->dev, dma)) 1096 goto dma_error; 1097 1098 /* record length, and DMA address */ 1099 dma_unmap_len_set(tx_buffer, len, size); 1100 dma_unmap_addr_set(tx_buffer, dma, dma); 1101 1102 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1103 1104 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 1105 tx_desc->read.cmd_type_len = 1106 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 1107 1108 i++; 1109 tx_desc++; 1110 if (i == tx_ring->count) { 1111 tx_desc = IGC_TX_DESC(tx_ring, 0); 1112 i = 0; 1113 } 1114 tx_desc->read.olinfo_status = 0; 1115 1116 dma += IGC_MAX_DATA_PER_TXD; 1117 size -= IGC_MAX_DATA_PER_TXD; 1118 1119 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1120 } 1121 1122 if (likely(!data_len)) 1123 break; 1124 1125 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 1126 1127 i++; 1128 tx_desc++; 1129 if (i == tx_ring->count) { 1130 tx_desc = IGC_TX_DESC(tx_ring, 0); 1131 i = 0; 1132 } 1133 tx_desc->read.olinfo_status = 0; 1134 1135 size = skb_frag_size(frag); 1136 data_len -= size; 1137 1138 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 1139 size, DMA_TO_DEVICE); 1140 1141 tx_buffer = &tx_ring->tx_buffer_info[i]; 1142 } 1143 1144 /* write last descriptor with RS and EOP bits */ 1145 cmd_type |= size | IGC_TXD_DCMD; 1146 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1147 1148 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1149 1150 /* set the timestamp */ 1151 first->time_stamp = jiffies; 1152 1153 skb_tx_timestamp(skb); 1154 1155 /* Force memory writes to complete before letting h/w know there 1156 * are new descriptors to fetch. (Only applicable for weak-ordered 1157 * memory model archs, such as IA-64). 1158 * 1159 * We also need this memory barrier to make certain all of the 1160 * status bits have been updated before next_to_watch is written. 1161 */ 1162 wmb(); 1163 1164 /* set next_to_watch value indicating a packet is present */ 1165 first->next_to_watch = tx_desc; 1166 1167 i++; 1168 if (i == tx_ring->count) 1169 i = 0; 1170 1171 tx_ring->next_to_use = i; 1172 1173 /* Make sure there is space in the ring for the next send. */ 1174 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 1175 1176 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1177 writel(i, tx_ring->tail); 1178 } 1179 1180 return 0; 1181 dma_error: 1182 netdev_err(tx_ring->netdev, "TX DMA map failed\n"); 1183 tx_buffer = &tx_ring->tx_buffer_info[i]; 1184 1185 /* clear dma mappings for failed tx_buffer_info map */ 1186 while (tx_buffer != first) { 1187 if (dma_unmap_len(tx_buffer, len)) 1188 dma_unmap_page(tx_ring->dev, 1189 dma_unmap_addr(tx_buffer, dma), 1190 dma_unmap_len(tx_buffer, len), 1191 DMA_TO_DEVICE); 1192 dma_unmap_len_set(tx_buffer, len, 0); 1193 1194 if (i-- == 0) 1195 i += tx_ring->count; 1196 tx_buffer = &tx_ring->tx_buffer_info[i]; 1197 } 1198 1199 if (dma_unmap_len(tx_buffer, len)) 1200 dma_unmap_single(tx_ring->dev, 1201 dma_unmap_addr(tx_buffer, dma), 1202 dma_unmap_len(tx_buffer, len), 1203 DMA_TO_DEVICE); 1204 dma_unmap_len_set(tx_buffer, len, 0); 1205 1206 dev_kfree_skb_any(tx_buffer->skb); 1207 tx_buffer->skb = NULL; 1208 1209 tx_ring->next_to_use = i; 1210 1211 return -1; 1212 } 1213 1214 static int igc_tso(struct igc_ring *tx_ring, 1215 struct igc_tx_buffer *first, 1216 u8 *hdr_len) 1217 { 1218 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 1219 struct sk_buff *skb = first->skb; 1220 union { 1221 struct iphdr *v4; 1222 struct ipv6hdr *v6; 1223 unsigned char *hdr; 1224 } ip; 1225 union { 1226 struct tcphdr *tcp; 1227 struct udphdr *udp; 1228 unsigned char *hdr; 1229 } l4; 1230 u32 paylen, l4_offset; 1231 int err; 1232 1233 if (skb->ip_summed != CHECKSUM_PARTIAL) 1234 return 0; 1235 1236 if (!skb_is_gso(skb)) 1237 return 0; 1238 1239 err = skb_cow_head(skb, 0); 1240 if (err < 0) 1241 return err; 1242 1243 ip.hdr = skb_network_header(skb); 1244 l4.hdr = skb_checksum_start(skb); 1245 1246 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 1247 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 1248 1249 /* initialize outer IP header fields */ 1250 if (ip.v4->version == 4) { 1251 unsigned char *csum_start = skb_checksum_start(skb); 1252 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 1253 1254 /* IP header will have to cancel out any data that 1255 * is not a part of the outer IP header 1256 */ 1257 ip.v4->check = csum_fold(csum_partial(trans_start, 1258 csum_start - trans_start, 1259 0)); 1260 type_tucmd |= IGC_ADVTXD_TUCMD_IPV4; 1261 1262 ip.v4->tot_len = 0; 1263 first->tx_flags |= IGC_TX_FLAGS_TSO | 1264 IGC_TX_FLAGS_CSUM | 1265 IGC_TX_FLAGS_IPV4; 1266 } else { 1267 ip.v6->payload_len = 0; 1268 first->tx_flags |= IGC_TX_FLAGS_TSO | 1269 IGC_TX_FLAGS_CSUM; 1270 } 1271 1272 /* determine offset of inner transport header */ 1273 l4_offset = l4.hdr - skb->data; 1274 1275 /* remove payload length from inner checksum */ 1276 paylen = skb->len - l4_offset; 1277 if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) { 1278 /* compute length of segmentation header */ 1279 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 1280 csum_replace_by_diff(&l4.tcp->check, 1281 (__force __wsum)htonl(paylen)); 1282 } else { 1283 /* compute length of segmentation header */ 1284 *hdr_len = sizeof(*l4.udp) + l4_offset; 1285 csum_replace_by_diff(&l4.udp->check, 1286 (__force __wsum)htonl(paylen)); 1287 } 1288 1289 /* update gso size and bytecount with header size */ 1290 first->gso_segs = skb_shinfo(skb)->gso_segs; 1291 first->bytecount += (first->gso_segs - 1) * *hdr_len; 1292 1293 /* MSS L4LEN IDX */ 1294 mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT; 1295 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT; 1296 1297 /* VLAN MACLEN IPLEN */ 1298 vlan_macip_lens = l4.hdr - ip.hdr; 1299 vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT; 1300 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1301 1302 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, 1303 type_tucmd, mss_l4len_idx); 1304 1305 return 1; 1306 } 1307 1308 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1309 struct igc_ring *tx_ring) 1310 { 1311 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1312 __be16 protocol = vlan_get_protocol(skb); 1313 struct igc_tx_buffer *first; 1314 u32 tx_flags = 0; 1315 unsigned short f; 1316 u8 hdr_len = 0; 1317 int tso = 0; 1318 1319 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1320 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1321 * + 2 desc gap to keep tail from touching head, 1322 * + 1 desc for context descriptor, 1323 * otherwise try next time 1324 */ 1325 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1326 count += TXD_USE_COUNT(skb_frag_size( 1327 &skb_shinfo(skb)->frags[f])); 1328 1329 if (igc_maybe_stop_tx(tx_ring, count + 3)) { 1330 /* this is a hard error */ 1331 return NETDEV_TX_BUSY; 1332 } 1333 1334 /* record the location of the first descriptor for this packet */ 1335 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1336 first->skb = skb; 1337 first->bytecount = skb->len; 1338 first->gso_segs = 1; 1339 1340 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 1341 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 1342 1343 /* FIXME: add support for retrieving timestamps from 1344 * the other timer registers before skipping the 1345 * timestamping request. 1346 */ 1347 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && 1348 !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS, 1349 &adapter->state)) { 1350 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1351 tx_flags |= IGC_TX_FLAGS_TSTAMP; 1352 1353 adapter->ptp_tx_skb = skb_get(skb); 1354 adapter->ptp_tx_start = jiffies; 1355 } else { 1356 adapter->tx_hwtstamp_skipped++; 1357 } 1358 } 1359 1360 /* record initial flags and protocol */ 1361 first->tx_flags = tx_flags; 1362 first->protocol = protocol; 1363 1364 tso = igc_tso(tx_ring, first, &hdr_len); 1365 if (tso < 0) 1366 goto out_drop; 1367 else if (!tso) 1368 igc_tx_csum(tx_ring, first); 1369 1370 igc_tx_map(tx_ring, first, hdr_len); 1371 1372 return NETDEV_TX_OK; 1373 1374 out_drop: 1375 dev_kfree_skb_any(first->skb); 1376 first->skb = NULL; 1377 1378 return NETDEV_TX_OK; 1379 } 1380 1381 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1382 struct sk_buff *skb) 1383 { 1384 unsigned int r_idx = skb->queue_mapping; 1385 1386 if (r_idx >= adapter->num_tx_queues) 1387 r_idx = r_idx % adapter->num_tx_queues; 1388 1389 return adapter->tx_ring[r_idx]; 1390 } 1391 1392 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1393 struct net_device *netdev) 1394 { 1395 struct igc_adapter *adapter = netdev_priv(netdev); 1396 1397 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1398 * in order to meet this minimum size requirement. 1399 */ 1400 if (skb->len < 17) { 1401 if (skb_padto(skb, 17)) 1402 return NETDEV_TX_OK; 1403 skb->len = 17; 1404 } 1405 1406 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1407 } 1408 1409 static void igc_rx_checksum(struct igc_ring *ring, 1410 union igc_adv_rx_desc *rx_desc, 1411 struct sk_buff *skb) 1412 { 1413 skb_checksum_none_assert(skb); 1414 1415 /* Ignore Checksum bit is set */ 1416 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) 1417 return; 1418 1419 /* Rx checksum disabled via ethtool */ 1420 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1421 return; 1422 1423 /* TCP/UDP checksum error bit is set */ 1424 if (igc_test_staterr(rx_desc, 1425 IGC_RXDEXT_STATERR_L4E | 1426 IGC_RXDEXT_STATERR_IPE)) { 1427 /* work around errata with sctp packets where the TCPE aka 1428 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 1429 * packets (aka let the stack check the crc32c) 1430 */ 1431 if (!(skb->len == 60 && 1432 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 1433 u64_stats_update_begin(&ring->rx_syncp); 1434 ring->rx_stats.csum_err++; 1435 u64_stats_update_end(&ring->rx_syncp); 1436 } 1437 /* let the stack verify checksum errors */ 1438 return; 1439 } 1440 /* It must be a TCP or UDP packet with a valid checksum */ 1441 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | 1442 IGC_RXD_STAT_UDPCS)) 1443 skb->ip_summed = CHECKSUM_UNNECESSARY; 1444 1445 netdev_dbg(ring->netdev, "cksum success: bits %08X\n", 1446 le32_to_cpu(rx_desc->wb.upper.status_error)); 1447 } 1448 1449 static inline void igc_rx_hash(struct igc_ring *ring, 1450 union igc_adv_rx_desc *rx_desc, 1451 struct sk_buff *skb) 1452 { 1453 if (ring->netdev->features & NETIF_F_RXHASH) 1454 skb_set_hash(skb, 1455 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 1456 PKT_HASH_TYPE_L3); 1457 } 1458 1459 /** 1460 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1461 * @rx_ring: rx descriptor ring packet is being transacted on 1462 * @rx_desc: pointer to the EOP Rx descriptor 1463 * @skb: pointer to current skb being populated 1464 * 1465 * This function checks the ring, descriptor, and packet information in order 1466 * to populate the hash, checksum, VLAN, protocol, and other fields within the 1467 * skb. 1468 */ 1469 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1470 union igc_adv_rx_desc *rx_desc, 1471 struct sk_buff *skb) 1472 { 1473 igc_rx_hash(rx_ring, rx_desc, skb); 1474 1475 igc_rx_checksum(rx_ring, rx_desc, skb); 1476 1477 skb_record_rx_queue(skb, rx_ring->queue_index); 1478 1479 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1480 } 1481 1482 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1483 const unsigned int size) 1484 { 1485 struct igc_rx_buffer *rx_buffer; 1486 1487 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1488 prefetchw(rx_buffer->page); 1489 1490 /* we are reusing so sync this buffer for CPU use */ 1491 dma_sync_single_range_for_cpu(rx_ring->dev, 1492 rx_buffer->dma, 1493 rx_buffer->page_offset, 1494 size, 1495 DMA_FROM_DEVICE); 1496 1497 rx_buffer->pagecnt_bias--; 1498 1499 return rx_buffer; 1500 } 1501 1502 /** 1503 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1504 * @rx_ring: rx descriptor ring to transact packets on 1505 * @rx_buffer: buffer containing page to add 1506 * @skb: sk_buff to place the data into 1507 * @size: size of buffer to be added 1508 * 1509 * This function will add the data contained in rx_buffer->page to the skb. 1510 */ 1511 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1512 struct igc_rx_buffer *rx_buffer, 1513 struct sk_buff *skb, 1514 unsigned int size) 1515 { 1516 #if (PAGE_SIZE < 8192) 1517 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1518 1519 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1520 rx_buffer->page_offset, size, truesize); 1521 rx_buffer->page_offset ^= truesize; 1522 #else 1523 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1524 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1525 SKB_DATA_ALIGN(size); 1526 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1527 rx_buffer->page_offset, size, truesize); 1528 rx_buffer->page_offset += truesize; 1529 #endif 1530 } 1531 1532 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1533 struct igc_rx_buffer *rx_buffer, 1534 union igc_adv_rx_desc *rx_desc, 1535 unsigned int size) 1536 { 1537 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1538 #if (PAGE_SIZE < 8192) 1539 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1540 #else 1541 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1542 SKB_DATA_ALIGN(IGC_SKB_PAD + size); 1543 #endif 1544 struct sk_buff *skb; 1545 1546 /* prefetch first cache line of first page */ 1547 net_prefetch(va); 1548 1549 /* build an skb around the page buffer */ 1550 skb = build_skb(va - IGC_SKB_PAD, truesize); 1551 if (unlikely(!skb)) 1552 return NULL; 1553 1554 /* update pointers within the skb to store the data */ 1555 skb_reserve(skb, IGC_SKB_PAD); 1556 __skb_put(skb, size); 1557 1558 /* update buffer offset */ 1559 #if (PAGE_SIZE < 8192) 1560 rx_buffer->page_offset ^= truesize; 1561 #else 1562 rx_buffer->page_offset += truesize; 1563 #endif 1564 1565 return skb; 1566 } 1567 1568 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1569 struct igc_rx_buffer *rx_buffer, 1570 union igc_adv_rx_desc *rx_desc, 1571 unsigned int size) 1572 { 1573 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1574 #if (PAGE_SIZE < 8192) 1575 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1576 #else 1577 unsigned int truesize = SKB_DATA_ALIGN(size); 1578 #endif 1579 unsigned int headlen; 1580 struct sk_buff *skb; 1581 1582 /* prefetch first cache line of first page */ 1583 net_prefetch(va); 1584 1585 /* allocate a skb to store the frags */ 1586 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); 1587 if (unlikely(!skb)) 1588 return NULL; 1589 1590 if (unlikely(igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP))) { 1591 igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb); 1592 va += IGC_TS_HDR_LEN; 1593 size -= IGC_TS_HDR_LEN; 1594 } 1595 1596 /* Determine available headroom for copy */ 1597 headlen = size; 1598 if (headlen > IGC_RX_HDR_LEN) 1599 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 1600 1601 /* align pull length to size of long to optimize memcpy performance */ 1602 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 1603 1604 /* update all of the pointers */ 1605 size -= headlen; 1606 if (size) { 1607 skb_add_rx_frag(skb, 0, rx_buffer->page, 1608 (va + headlen) - page_address(rx_buffer->page), 1609 size, truesize); 1610 #if (PAGE_SIZE < 8192) 1611 rx_buffer->page_offset ^= truesize; 1612 #else 1613 rx_buffer->page_offset += truesize; 1614 #endif 1615 } else { 1616 rx_buffer->pagecnt_bias++; 1617 } 1618 1619 return skb; 1620 } 1621 1622 /** 1623 * igc_reuse_rx_page - page flip buffer and store it back on the ring 1624 * @rx_ring: rx descriptor ring to store buffers on 1625 * @old_buff: donor buffer to have page reused 1626 * 1627 * Synchronizes page for reuse by the adapter 1628 */ 1629 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 1630 struct igc_rx_buffer *old_buff) 1631 { 1632 u16 nta = rx_ring->next_to_alloc; 1633 struct igc_rx_buffer *new_buff; 1634 1635 new_buff = &rx_ring->rx_buffer_info[nta]; 1636 1637 /* update, and store next to alloc */ 1638 nta++; 1639 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1640 1641 /* Transfer page from old buffer to new buffer. 1642 * Move each member individually to avoid possible store 1643 * forwarding stalls. 1644 */ 1645 new_buff->dma = old_buff->dma; 1646 new_buff->page = old_buff->page; 1647 new_buff->page_offset = old_buff->page_offset; 1648 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1649 } 1650 1651 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) 1652 { 1653 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1654 struct page *page = rx_buffer->page; 1655 1656 /* avoid re-using remote and pfmemalloc pages */ 1657 if (!dev_page_is_reusable(page)) 1658 return false; 1659 1660 #if (PAGE_SIZE < 8192) 1661 /* if we are only owner of page we can reuse it */ 1662 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 1663 return false; 1664 #else 1665 #define IGC_LAST_OFFSET \ 1666 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 1667 1668 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 1669 return false; 1670 #endif 1671 1672 /* If we have drained the page fragment pool we need to update 1673 * the pagecnt_bias and page count so that we fully restock the 1674 * number of references the driver holds. 1675 */ 1676 if (unlikely(!pagecnt_bias)) { 1677 page_ref_add(page, USHRT_MAX); 1678 rx_buffer->pagecnt_bias = USHRT_MAX; 1679 } 1680 1681 return true; 1682 } 1683 1684 /** 1685 * igc_is_non_eop - process handling of non-EOP buffers 1686 * @rx_ring: Rx ring being processed 1687 * @rx_desc: Rx descriptor for current buffer 1688 * 1689 * This function updates next to clean. If the buffer is an EOP buffer 1690 * this function exits returning false, otherwise it will place the 1691 * sk_buff in the next buffer to be chained and return true indicating 1692 * that this is in fact a non-EOP buffer. 1693 */ 1694 static bool igc_is_non_eop(struct igc_ring *rx_ring, 1695 union igc_adv_rx_desc *rx_desc) 1696 { 1697 u32 ntc = rx_ring->next_to_clean + 1; 1698 1699 /* fetch, update, and store next to clean */ 1700 ntc = (ntc < rx_ring->count) ? ntc : 0; 1701 rx_ring->next_to_clean = ntc; 1702 1703 prefetch(IGC_RX_DESC(rx_ring, ntc)); 1704 1705 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 1706 return false; 1707 1708 return true; 1709 } 1710 1711 /** 1712 * igc_cleanup_headers - Correct corrupted or empty headers 1713 * @rx_ring: rx descriptor ring packet is being transacted on 1714 * @rx_desc: pointer to the EOP Rx descriptor 1715 * @skb: pointer to current skb being fixed 1716 * 1717 * Address the case where we are pulling data in on pages only 1718 * and as such no data is present in the skb header. 1719 * 1720 * In addition if skb is not at least 60 bytes we need to pad it so that 1721 * it is large enough to qualify as a valid Ethernet frame. 1722 * 1723 * Returns true if an error was encountered and skb was freed. 1724 */ 1725 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 1726 union igc_adv_rx_desc *rx_desc, 1727 struct sk_buff *skb) 1728 { 1729 if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) { 1730 struct net_device *netdev = rx_ring->netdev; 1731 1732 if (!(netdev->features & NETIF_F_RXALL)) { 1733 dev_kfree_skb_any(skb); 1734 return true; 1735 } 1736 } 1737 1738 /* if eth_skb_pad returns an error the skb was freed */ 1739 if (eth_skb_pad(skb)) 1740 return true; 1741 1742 return false; 1743 } 1744 1745 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 1746 struct igc_rx_buffer *rx_buffer) 1747 { 1748 if (igc_can_reuse_rx_page(rx_buffer)) { 1749 /* hand second half of page back to the ring */ 1750 igc_reuse_rx_page(rx_ring, rx_buffer); 1751 } else { 1752 /* We are not reusing the buffer so unmap it and free 1753 * any references we are holding to it 1754 */ 1755 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1756 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 1757 IGC_RX_DMA_ATTR); 1758 __page_frag_cache_drain(rx_buffer->page, 1759 rx_buffer->pagecnt_bias); 1760 } 1761 1762 /* clear contents of rx_buffer */ 1763 rx_buffer->page = NULL; 1764 } 1765 1766 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 1767 { 1768 return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; 1769 } 1770 1771 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 1772 struct igc_rx_buffer *bi) 1773 { 1774 struct page *page = bi->page; 1775 dma_addr_t dma; 1776 1777 /* since we are recycling buffers we should seldom need to alloc */ 1778 if (likely(page)) 1779 return true; 1780 1781 /* alloc new page for storage */ 1782 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 1783 if (unlikely(!page)) { 1784 rx_ring->rx_stats.alloc_failed++; 1785 return false; 1786 } 1787 1788 /* map page for use */ 1789 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1790 igc_rx_pg_size(rx_ring), 1791 DMA_FROM_DEVICE, 1792 IGC_RX_DMA_ATTR); 1793 1794 /* if mapping failed free memory back to system since 1795 * there isn't much point in holding memory we can't use 1796 */ 1797 if (dma_mapping_error(rx_ring->dev, dma)) { 1798 __free_page(page); 1799 1800 rx_ring->rx_stats.alloc_failed++; 1801 return false; 1802 } 1803 1804 bi->dma = dma; 1805 bi->page = page; 1806 bi->page_offset = igc_rx_offset(rx_ring); 1807 bi->pagecnt_bias = 1; 1808 1809 return true; 1810 } 1811 1812 /** 1813 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 1814 * @rx_ring: rx descriptor ring 1815 * @cleaned_count: number of buffers to clean 1816 */ 1817 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 1818 { 1819 union igc_adv_rx_desc *rx_desc; 1820 u16 i = rx_ring->next_to_use; 1821 struct igc_rx_buffer *bi; 1822 u16 bufsz; 1823 1824 /* nothing to do */ 1825 if (!cleaned_count) 1826 return; 1827 1828 rx_desc = IGC_RX_DESC(rx_ring, i); 1829 bi = &rx_ring->rx_buffer_info[i]; 1830 i -= rx_ring->count; 1831 1832 bufsz = igc_rx_bufsz(rx_ring); 1833 1834 do { 1835 if (!igc_alloc_mapped_page(rx_ring, bi)) 1836 break; 1837 1838 /* sync the buffer for use by the device */ 1839 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1840 bi->page_offset, bufsz, 1841 DMA_FROM_DEVICE); 1842 1843 /* Refresh the desc even if buffer_addrs didn't change 1844 * because each write-back erases this info. 1845 */ 1846 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1847 1848 rx_desc++; 1849 bi++; 1850 i++; 1851 if (unlikely(!i)) { 1852 rx_desc = IGC_RX_DESC(rx_ring, 0); 1853 bi = rx_ring->rx_buffer_info; 1854 i -= rx_ring->count; 1855 } 1856 1857 /* clear the length for the next_to_use descriptor */ 1858 rx_desc->wb.upper.length = 0; 1859 1860 cleaned_count--; 1861 } while (cleaned_count); 1862 1863 i += rx_ring->count; 1864 1865 if (rx_ring->next_to_use != i) { 1866 /* record the next descriptor to use */ 1867 rx_ring->next_to_use = i; 1868 1869 /* update next to alloc since we have filled the ring */ 1870 rx_ring->next_to_alloc = i; 1871 1872 /* Force memory writes to complete before letting h/w 1873 * know there are new descriptors to fetch. (Only 1874 * applicable for weak-ordered memory model archs, 1875 * such as IA-64). 1876 */ 1877 wmb(); 1878 writel(i, rx_ring->tail); 1879 } 1880 } 1881 1882 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 1883 { 1884 unsigned int total_bytes = 0, total_packets = 0; 1885 struct igc_ring *rx_ring = q_vector->rx.ring; 1886 struct sk_buff *skb = rx_ring->skb; 1887 u16 cleaned_count = igc_desc_unused(rx_ring); 1888 1889 while (likely(total_packets < budget)) { 1890 union igc_adv_rx_desc *rx_desc; 1891 struct igc_rx_buffer *rx_buffer; 1892 unsigned int size; 1893 1894 /* return some buffers to hardware, one at a time is too slow */ 1895 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 1896 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1897 cleaned_count = 0; 1898 } 1899 1900 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 1901 size = le16_to_cpu(rx_desc->wb.upper.length); 1902 if (!size) 1903 break; 1904 1905 /* This memory barrier is needed to keep us from reading 1906 * any other fields out of the rx_desc until we know the 1907 * descriptor has been written back 1908 */ 1909 dma_rmb(); 1910 1911 rx_buffer = igc_get_rx_buffer(rx_ring, size); 1912 1913 /* retrieve a buffer from the ring */ 1914 if (skb) 1915 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 1916 else if (ring_uses_build_skb(rx_ring)) 1917 skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); 1918 else 1919 skb = igc_construct_skb(rx_ring, rx_buffer, 1920 rx_desc, size); 1921 1922 /* exit if we failed to retrieve a buffer */ 1923 if (!skb) { 1924 rx_ring->rx_stats.alloc_failed++; 1925 rx_buffer->pagecnt_bias++; 1926 break; 1927 } 1928 1929 igc_put_rx_buffer(rx_ring, rx_buffer); 1930 cleaned_count++; 1931 1932 /* fetch next buffer in frame if non-eop */ 1933 if (igc_is_non_eop(rx_ring, rx_desc)) 1934 continue; 1935 1936 /* verify the packet layout is correct */ 1937 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) { 1938 skb = NULL; 1939 continue; 1940 } 1941 1942 /* probably a little skewed due to removing CRC */ 1943 total_bytes += skb->len; 1944 1945 /* populate checksum, VLAN, and protocol */ 1946 igc_process_skb_fields(rx_ring, rx_desc, skb); 1947 1948 napi_gro_receive(&q_vector->napi, skb); 1949 1950 /* reset skb pointer */ 1951 skb = NULL; 1952 1953 /* update budget accounting */ 1954 total_packets++; 1955 } 1956 1957 /* place incomplete frames back on ring for completion */ 1958 rx_ring->skb = skb; 1959 1960 u64_stats_update_begin(&rx_ring->rx_syncp); 1961 rx_ring->rx_stats.packets += total_packets; 1962 rx_ring->rx_stats.bytes += total_bytes; 1963 u64_stats_update_end(&rx_ring->rx_syncp); 1964 q_vector->rx.total_packets += total_packets; 1965 q_vector->rx.total_bytes += total_bytes; 1966 1967 if (cleaned_count) 1968 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1969 1970 return total_packets; 1971 } 1972 1973 /** 1974 * igc_clean_tx_irq - Reclaim resources after transmit completes 1975 * @q_vector: pointer to q_vector containing needed info 1976 * @napi_budget: Used to determine if we are in netpoll 1977 * 1978 * returns true if ring is completely cleaned 1979 */ 1980 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 1981 { 1982 struct igc_adapter *adapter = q_vector->adapter; 1983 unsigned int total_bytes = 0, total_packets = 0; 1984 unsigned int budget = q_vector->tx.work_limit; 1985 struct igc_ring *tx_ring = q_vector->tx.ring; 1986 unsigned int i = tx_ring->next_to_clean; 1987 struct igc_tx_buffer *tx_buffer; 1988 union igc_adv_tx_desc *tx_desc; 1989 1990 if (test_bit(__IGC_DOWN, &adapter->state)) 1991 return true; 1992 1993 tx_buffer = &tx_ring->tx_buffer_info[i]; 1994 tx_desc = IGC_TX_DESC(tx_ring, i); 1995 i -= tx_ring->count; 1996 1997 do { 1998 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 1999 2000 /* if next_to_watch is not set then there is no work pending */ 2001 if (!eop_desc) 2002 break; 2003 2004 /* prevent any other reads prior to eop_desc */ 2005 smp_rmb(); 2006 2007 /* if DD is not set pending work has not been completed */ 2008 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 2009 break; 2010 2011 /* clear next_to_watch to prevent false hangs */ 2012 tx_buffer->next_to_watch = NULL; 2013 2014 /* update the statistics for this packet */ 2015 total_bytes += tx_buffer->bytecount; 2016 total_packets += tx_buffer->gso_segs; 2017 2018 /* free the skb */ 2019 napi_consume_skb(tx_buffer->skb, napi_budget); 2020 2021 /* unmap skb header data */ 2022 dma_unmap_single(tx_ring->dev, 2023 dma_unmap_addr(tx_buffer, dma), 2024 dma_unmap_len(tx_buffer, len), 2025 DMA_TO_DEVICE); 2026 2027 /* clear tx_buffer data */ 2028 dma_unmap_len_set(tx_buffer, len, 0); 2029 2030 /* clear last DMA location and unmap remaining buffers */ 2031 while (tx_desc != eop_desc) { 2032 tx_buffer++; 2033 tx_desc++; 2034 i++; 2035 if (unlikely(!i)) { 2036 i -= tx_ring->count; 2037 tx_buffer = tx_ring->tx_buffer_info; 2038 tx_desc = IGC_TX_DESC(tx_ring, 0); 2039 } 2040 2041 /* unmap any remaining paged data */ 2042 if (dma_unmap_len(tx_buffer, len)) { 2043 dma_unmap_page(tx_ring->dev, 2044 dma_unmap_addr(tx_buffer, dma), 2045 dma_unmap_len(tx_buffer, len), 2046 DMA_TO_DEVICE); 2047 dma_unmap_len_set(tx_buffer, len, 0); 2048 } 2049 } 2050 2051 /* move us one more past the eop_desc for start of next pkt */ 2052 tx_buffer++; 2053 tx_desc++; 2054 i++; 2055 if (unlikely(!i)) { 2056 i -= tx_ring->count; 2057 tx_buffer = tx_ring->tx_buffer_info; 2058 tx_desc = IGC_TX_DESC(tx_ring, 0); 2059 } 2060 2061 /* issue prefetch for next Tx descriptor */ 2062 prefetch(tx_desc); 2063 2064 /* update budget accounting */ 2065 budget--; 2066 } while (likely(budget)); 2067 2068 netdev_tx_completed_queue(txring_txq(tx_ring), 2069 total_packets, total_bytes); 2070 2071 i += tx_ring->count; 2072 tx_ring->next_to_clean = i; 2073 u64_stats_update_begin(&tx_ring->tx_syncp); 2074 tx_ring->tx_stats.bytes += total_bytes; 2075 tx_ring->tx_stats.packets += total_packets; 2076 u64_stats_update_end(&tx_ring->tx_syncp); 2077 q_vector->tx.total_bytes += total_bytes; 2078 q_vector->tx.total_packets += total_packets; 2079 2080 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 2081 struct igc_hw *hw = &adapter->hw; 2082 2083 /* Detect a transmit hang in hardware, this serializes the 2084 * check with the clearing of time_stamp and movement of i 2085 */ 2086 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 2087 if (tx_buffer->next_to_watch && 2088 time_after(jiffies, tx_buffer->time_stamp + 2089 (adapter->tx_timeout_factor * HZ)) && 2090 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { 2091 /* detected Tx unit hang */ 2092 netdev_err(tx_ring->netdev, 2093 "Detected Tx Unit Hang\n" 2094 " Tx Queue <%d>\n" 2095 " TDH <%x>\n" 2096 " TDT <%x>\n" 2097 " next_to_use <%x>\n" 2098 " next_to_clean <%x>\n" 2099 "buffer_info[next_to_clean]\n" 2100 " time_stamp <%lx>\n" 2101 " next_to_watch <%p>\n" 2102 " jiffies <%lx>\n" 2103 " desc.status <%x>\n", 2104 tx_ring->queue_index, 2105 rd32(IGC_TDH(tx_ring->reg_idx)), 2106 readl(tx_ring->tail), 2107 tx_ring->next_to_use, 2108 tx_ring->next_to_clean, 2109 tx_buffer->time_stamp, 2110 tx_buffer->next_to_watch, 2111 jiffies, 2112 tx_buffer->next_to_watch->wb.status); 2113 netif_stop_subqueue(tx_ring->netdev, 2114 tx_ring->queue_index); 2115 2116 /* we are about to reset, no point in enabling stuff */ 2117 return true; 2118 } 2119 } 2120 2121 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 2122 if (unlikely(total_packets && 2123 netif_carrier_ok(tx_ring->netdev) && 2124 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 2125 /* Make sure that anybody stopping the queue after this 2126 * sees the new next_to_clean. 2127 */ 2128 smp_mb(); 2129 if (__netif_subqueue_stopped(tx_ring->netdev, 2130 tx_ring->queue_index) && 2131 !(test_bit(__IGC_DOWN, &adapter->state))) { 2132 netif_wake_subqueue(tx_ring->netdev, 2133 tx_ring->queue_index); 2134 2135 u64_stats_update_begin(&tx_ring->tx_syncp); 2136 tx_ring->tx_stats.restart_queue++; 2137 u64_stats_update_end(&tx_ring->tx_syncp); 2138 } 2139 } 2140 2141 return !!budget; 2142 } 2143 2144 static int igc_find_mac_filter(struct igc_adapter *adapter, 2145 enum igc_mac_filter_type type, const u8 *addr) 2146 { 2147 struct igc_hw *hw = &adapter->hw; 2148 int max_entries = hw->mac.rar_entry_count; 2149 u32 ral, rah; 2150 int i; 2151 2152 for (i = 0; i < max_entries; i++) { 2153 ral = rd32(IGC_RAL(i)); 2154 rah = rd32(IGC_RAH(i)); 2155 2156 if (!(rah & IGC_RAH_AV)) 2157 continue; 2158 if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type) 2159 continue; 2160 if ((rah & IGC_RAH_RAH_MASK) != 2161 le16_to_cpup((__le16 *)(addr + 4))) 2162 continue; 2163 if (ral != le32_to_cpup((__le32 *)(addr))) 2164 continue; 2165 2166 return i; 2167 } 2168 2169 return -1; 2170 } 2171 2172 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) 2173 { 2174 struct igc_hw *hw = &adapter->hw; 2175 int max_entries = hw->mac.rar_entry_count; 2176 u32 rah; 2177 int i; 2178 2179 for (i = 0; i < max_entries; i++) { 2180 rah = rd32(IGC_RAH(i)); 2181 2182 if (!(rah & IGC_RAH_AV)) 2183 return i; 2184 } 2185 2186 return -1; 2187 } 2188 2189 /** 2190 * igc_add_mac_filter() - Add MAC address filter 2191 * @adapter: Pointer to adapter where the filter should be added 2192 * @type: MAC address filter type (source or destination) 2193 * @addr: MAC address 2194 * @queue: If non-negative, queue assignment feature is enabled and frames 2195 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2196 * assignment is disabled. 2197 * 2198 * Return: 0 in case of success, negative errno code otherwise. 2199 */ 2200 static int igc_add_mac_filter(struct igc_adapter *adapter, 2201 enum igc_mac_filter_type type, const u8 *addr, 2202 int queue) 2203 { 2204 struct net_device *dev = adapter->netdev; 2205 int index; 2206 2207 index = igc_find_mac_filter(adapter, type, addr); 2208 if (index >= 0) 2209 goto update_filter; 2210 2211 index = igc_get_avail_mac_filter_slot(adapter); 2212 if (index < 0) 2213 return -ENOSPC; 2214 2215 netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n", 2216 index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2217 addr, queue); 2218 2219 update_filter: 2220 igc_set_mac_filter_hw(adapter, index, type, addr, queue); 2221 return 0; 2222 } 2223 2224 /** 2225 * igc_del_mac_filter() - Delete MAC address filter 2226 * @adapter: Pointer to adapter where the filter should be deleted from 2227 * @type: MAC address filter type (source or destination) 2228 * @addr: MAC address 2229 */ 2230 static void igc_del_mac_filter(struct igc_adapter *adapter, 2231 enum igc_mac_filter_type type, const u8 *addr) 2232 { 2233 struct net_device *dev = adapter->netdev; 2234 int index; 2235 2236 index = igc_find_mac_filter(adapter, type, addr); 2237 if (index < 0) 2238 return; 2239 2240 if (index == 0) { 2241 /* If this is the default filter, we don't actually delete it. 2242 * We just reset to its default value i.e. disable queue 2243 * assignment. 2244 */ 2245 netdev_dbg(dev, "Disable default MAC filter queue assignment"); 2246 2247 igc_set_mac_filter_hw(adapter, 0, type, addr, -1); 2248 } else { 2249 netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n", 2250 index, 2251 type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2252 addr); 2253 2254 igc_clear_mac_filter_hw(adapter, index); 2255 } 2256 } 2257 2258 /** 2259 * igc_add_vlan_prio_filter() - Add VLAN priority filter 2260 * @adapter: Pointer to adapter where the filter should be added 2261 * @prio: VLAN priority value 2262 * @queue: Queue number which matching frames are assigned to 2263 * 2264 * Return: 0 in case of success, negative errno code otherwise. 2265 */ 2266 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, 2267 int queue) 2268 { 2269 struct net_device *dev = adapter->netdev; 2270 struct igc_hw *hw = &adapter->hw; 2271 u32 vlanpqf; 2272 2273 vlanpqf = rd32(IGC_VLANPQF); 2274 2275 if (vlanpqf & IGC_VLANPQF_VALID(prio)) { 2276 netdev_dbg(dev, "VLAN priority filter already in use\n"); 2277 return -EEXIST; 2278 } 2279 2280 vlanpqf |= IGC_VLANPQF_QSEL(prio, queue); 2281 vlanpqf |= IGC_VLANPQF_VALID(prio); 2282 2283 wr32(IGC_VLANPQF, vlanpqf); 2284 2285 netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n", 2286 prio, queue); 2287 return 0; 2288 } 2289 2290 /** 2291 * igc_del_vlan_prio_filter() - Delete VLAN priority filter 2292 * @adapter: Pointer to adapter where the filter should be deleted from 2293 * @prio: VLAN priority value 2294 */ 2295 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) 2296 { 2297 struct igc_hw *hw = &adapter->hw; 2298 u32 vlanpqf; 2299 2300 vlanpqf = rd32(IGC_VLANPQF); 2301 2302 vlanpqf &= ~IGC_VLANPQF_VALID(prio); 2303 vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK); 2304 2305 wr32(IGC_VLANPQF, vlanpqf); 2306 2307 netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n", 2308 prio); 2309 } 2310 2311 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) 2312 { 2313 struct igc_hw *hw = &adapter->hw; 2314 int i; 2315 2316 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2317 u32 etqf = rd32(IGC_ETQF(i)); 2318 2319 if (!(etqf & IGC_ETQF_FILTER_ENABLE)) 2320 return i; 2321 } 2322 2323 return -1; 2324 } 2325 2326 /** 2327 * igc_add_etype_filter() - Add ethertype filter 2328 * @adapter: Pointer to adapter where the filter should be added 2329 * @etype: Ethertype value 2330 * @queue: If non-negative, queue assignment feature is enabled and frames 2331 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2332 * assignment is disabled. 2333 * 2334 * Return: 0 in case of success, negative errno code otherwise. 2335 */ 2336 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, 2337 int queue) 2338 { 2339 struct igc_hw *hw = &adapter->hw; 2340 int index; 2341 u32 etqf; 2342 2343 index = igc_get_avail_etype_filter_slot(adapter); 2344 if (index < 0) 2345 return -ENOSPC; 2346 2347 etqf = rd32(IGC_ETQF(index)); 2348 2349 etqf &= ~IGC_ETQF_ETYPE_MASK; 2350 etqf |= etype; 2351 2352 if (queue >= 0) { 2353 etqf &= ~IGC_ETQF_QUEUE_MASK; 2354 etqf |= (queue << IGC_ETQF_QUEUE_SHIFT); 2355 etqf |= IGC_ETQF_QUEUE_ENABLE; 2356 } 2357 2358 etqf |= IGC_ETQF_FILTER_ENABLE; 2359 2360 wr32(IGC_ETQF(index), etqf); 2361 2362 netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n", 2363 etype, queue); 2364 return 0; 2365 } 2366 2367 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) 2368 { 2369 struct igc_hw *hw = &adapter->hw; 2370 int i; 2371 2372 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2373 u32 etqf = rd32(IGC_ETQF(i)); 2374 2375 if ((etqf & IGC_ETQF_ETYPE_MASK) == etype) 2376 return i; 2377 } 2378 2379 return -1; 2380 } 2381 2382 /** 2383 * igc_del_etype_filter() - Delete ethertype filter 2384 * @adapter: Pointer to adapter where the filter should be deleted from 2385 * @etype: Ethertype value 2386 */ 2387 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) 2388 { 2389 struct igc_hw *hw = &adapter->hw; 2390 int index; 2391 2392 index = igc_find_etype_filter(adapter, etype); 2393 if (index < 0) 2394 return; 2395 2396 wr32(IGC_ETQF(index), 0); 2397 2398 netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n", 2399 etype); 2400 } 2401 2402 static int igc_enable_nfc_rule(struct igc_adapter *adapter, 2403 const struct igc_nfc_rule *rule) 2404 { 2405 int err; 2406 2407 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { 2408 err = igc_add_etype_filter(adapter, rule->filter.etype, 2409 rule->action); 2410 if (err) 2411 return err; 2412 } 2413 2414 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { 2415 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2416 rule->filter.src_addr, rule->action); 2417 if (err) 2418 return err; 2419 } 2420 2421 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { 2422 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2423 rule->filter.dst_addr, rule->action); 2424 if (err) 2425 return err; 2426 } 2427 2428 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2429 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2430 VLAN_PRIO_SHIFT; 2431 2432 err = igc_add_vlan_prio_filter(adapter, prio, rule->action); 2433 if (err) 2434 return err; 2435 } 2436 2437 return 0; 2438 } 2439 2440 static void igc_disable_nfc_rule(struct igc_adapter *adapter, 2441 const struct igc_nfc_rule *rule) 2442 { 2443 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) 2444 igc_del_etype_filter(adapter, rule->filter.etype); 2445 2446 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2447 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2448 VLAN_PRIO_SHIFT; 2449 2450 igc_del_vlan_prio_filter(adapter, prio); 2451 } 2452 2453 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) 2454 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2455 rule->filter.src_addr); 2456 2457 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) 2458 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2459 rule->filter.dst_addr); 2460 } 2461 2462 /** 2463 * igc_get_nfc_rule() - Get NFC rule 2464 * @adapter: Pointer to adapter 2465 * @location: Rule location 2466 * 2467 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2468 * 2469 * Return: Pointer to NFC rule at @location. If not found, NULL. 2470 */ 2471 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, 2472 u32 location) 2473 { 2474 struct igc_nfc_rule *rule; 2475 2476 list_for_each_entry(rule, &adapter->nfc_rule_list, list) { 2477 if (rule->location == location) 2478 return rule; 2479 if (rule->location > location) 2480 break; 2481 } 2482 2483 return NULL; 2484 } 2485 2486 /** 2487 * igc_del_nfc_rule() - Delete NFC rule 2488 * @adapter: Pointer to adapter 2489 * @rule: Pointer to rule to be deleted 2490 * 2491 * Disable NFC rule in hardware and delete it from adapter. 2492 * 2493 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2494 */ 2495 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2496 { 2497 igc_disable_nfc_rule(adapter, rule); 2498 2499 list_del(&rule->list); 2500 adapter->nfc_rule_count--; 2501 2502 kfree(rule); 2503 } 2504 2505 static void igc_flush_nfc_rules(struct igc_adapter *adapter) 2506 { 2507 struct igc_nfc_rule *rule, *tmp; 2508 2509 mutex_lock(&adapter->nfc_rule_lock); 2510 2511 list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list) 2512 igc_del_nfc_rule(adapter, rule); 2513 2514 mutex_unlock(&adapter->nfc_rule_lock); 2515 } 2516 2517 /** 2518 * igc_add_nfc_rule() - Add NFC rule 2519 * @adapter: Pointer to adapter 2520 * @rule: Pointer to rule to be added 2521 * 2522 * Enable NFC rule in hardware and add it to adapter. 2523 * 2524 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2525 * 2526 * Return: 0 on success, negative errno on failure. 2527 */ 2528 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2529 { 2530 struct igc_nfc_rule *pred, *cur; 2531 int err; 2532 2533 err = igc_enable_nfc_rule(adapter, rule); 2534 if (err) 2535 return err; 2536 2537 pred = NULL; 2538 list_for_each_entry(cur, &adapter->nfc_rule_list, list) { 2539 if (cur->location >= rule->location) 2540 break; 2541 pred = cur; 2542 } 2543 2544 list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list); 2545 adapter->nfc_rule_count++; 2546 return 0; 2547 } 2548 2549 static void igc_restore_nfc_rules(struct igc_adapter *adapter) 2550 { 2551 struct igc_nfc_rule *rule; 2552 2553 mutex_lock(&adapter->nfc_rule_lock); 2554 2555 list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list) 2556 igc_enable_nfc_rule(adapter, rule); 2557 2558 mutex_unlock(&adapter->nfc_rule_lock); 2559 } 2560 2561 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) 2562 { 2563 struct igc_adapter *adapter = netdev_priv(netdev); 2564 2565 return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1); 2566 } 2567 2568 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) 2569 { 2570 struct igc_adapter *adapter = netdev_priv(netdev); 2571 2572 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr); 2573 return 0; 2574 } 2575 2576 /** 2577 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 2578 * @netdev: network interface device structure 2579 * 2580 * The set_rx_mode entry point is called whenever the unicast or multicast 2581 * address lists or the network interface flags are updated. This routine is 2582 * responsible for configuring the hardware for proper unicast, multicast, 2583 * promiscuous mode, and all-multi behavior. 2584 */ 2585 static void igc_set_rx_mode(struct net_device *netdev) 2586 { 2587 struct igc_adapter *adapter = netdev_priv(netdev); 2588 struct igc_hw *hw = &adapter->hw; 2589 u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 2590 int count; 2591 2592 /* Check for Promiscuous and All Multicast modes */ 2593 if (netdev->flags & IFF_PROMISC) { 2594 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; 2595 } else { 2596 if (netdev->flags & IFF_ALLMULTI) { 2597 rctl |= IGC_RCTL_MPE; 2598 } else { 2599 /* Write addresses to the MTA, if the attempt fails 2600 * then we should just turn on promiscuous mode so 2601 * that we can at least receive multicast traffic 2602 */ 2603 count = igc_write_mc_addr_list(netdev); 2604 if (count < 0) 2605 rctl |= IGC_RCTL_MPE; 2606 } 2607 } 2608 2609 /* Write addresses to available RAR registers, if there is not 2610 * sufficient space to store all the addresses then enable 2611 * unicast promiscuous mode 2612 */ 2613 if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) 2614 rctl |= IGC_RCTL_UPE; 2615 2616 /* update state of unicast and multicast */ 2617 rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 2618 wr32(IGC_RCTL, rctl); 2619 2620 #if (PAGE_SIZE < 8192) 2621 if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) 2622 rlpml = IGC_MAX_FRAME_BUILD_SKB; 2623 #endif 2624 wr32(IGC_RLPML, rlpml); 2625 } 2626 2627 /** 2628 * igc_configure - configure the hardware for RX and TX 2629 * @adapter: private board structure 2630 */ 2631 static void igc_configure(struct igc_adapter *adapter) 2632 { 2633 struct net_device *netdev = adapter->netdev; 2634 int i = 0; 2635 2636 igc_get_hw_control(adapter); 2637 igc_set_rx_mode(netdev); 2638 2639 igc_setup_tctl(adapter); 2640 igc_setup_mrqc(adapter); 2641 igc_setup_rctl(adapter); 2642 2643 igc_set_default_mac_filter(adapter); 2644 igc_restore_nfc_rules(adapter); 2645 2646 igc_configure_tx(adapter); 2647 igc_configure_rx(adapter); 2648 2649 igc_rx_fifo_flush_base(&adapter->hw); 2650 2651 /* call igc_desc_unused which always leaves 2652 * at least 1 descriptor unused to make sure 2653 * next_to_use != next_to_clean 2654 */ 2655 for (i = 0; i < adapter->num_rx_queues; i++) { 2656 struct igc_ring *ring = adapter->rx_ring[i]; 2657 2658 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 2659 } 2660 } 2661 2662 /** 2663 * igc_write_ivar - configure ivar for given MSI-X vector 2664 * @hw: pointer to the HW structure 2665 * @msix_vector: vector number we are allocating to a given ring 2666 * @index: row index of IVAR register to write within IVAR table 2667 * @offset: column offset of in IVAR, should be multiple of 8 2668 * 2669 * The IVAR table consists of 2 columns, 2670 * each containing an cause allocation for an Rx and Tx ring, and a 2671 * variable number of rows depending on the number of queues supported. 2672 */ 2673 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 2674 int index, int offset) 2675 { 2676 u32 ivar = array_rd32(IGC_IVAR0, index); 2677 2678 /* clear any bits that are currently set */ 2679 ivar &= ~((u32)0xFF << offset); 2680 2681 /* write vector and valid bit */ 2682 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 2683 2684 array_wr32(IGC_IVAR0, index, ivar); 2685 } 2686 2687 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 2688 { 2689 struct igc_adapter *adapter = q_vector->adapter; 2690 struct igc_hw *hw = &adapter->hw; 2691 int rx_queue = IGC_N0_QUEUE; 2692 int tx_queue = IGC_N0_QUEUE; 2693 2694 if (q_vector->rx.ring) 2695 rx_queue = q_vector->rx.ring->reg_idx; 2696 if (q_vector->tx.ring) 2697 tx_queue = q_vector->tx.ring->reg_idx; 2698 2699 switch (hw->mac.type) { 2700 case igc_i225: 2701 if (rx_queue > IGC_N0_QUEUE) 2702 igc_write_ivar(hw, msix_vector, 2703 rx_queue >> 1, 2704 (rx_queue & 0x1) << 4); 2705 if (tx_queue > IGC_N0_QUEUE) 2706 igc_write_ivar(hw, msix_vector, 2707 tx_queue >> 1, 2708 ((tx_queue & 0x1) << 4) + 8); 2709 q_vector->eims_value = BIT(msix_vector); 2710 break; 2711 default: 2712 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 2713 break; 2714 } 2715 2716 /* add q_vector eims value to global eims_enable_mask */ 2717 adapter->eims_enable_mask |= q_vector->eims_value; 2718 2719 /* configure q_vector to set itr on first interrupt */ 2720 q_vector->set_itr = 1; 2721 } 2722 2723 /** 2724 * igc_configure_msix - Configure MSI-X hardware 2725 * @adapter: Pointer to adapter structure 2726 * 2727 * igc_configure_msix sets up the hardware to properly 2728 * generate MSI-X interrupts. 2729 */ 2730 static void igc_configure_msix(struct igc_adapter *adapter) 2731 { 2732 struct igc_hw *hw = &adapter->hw; 2733 int i, vector = 0; 2734 u32 tmp; 2735 2736 adapter->eims_enable_mask = 0; 2737 2738 /* set vector for other causes, i.e. link changes */ 2739 switch (hw->mac.type) { 2740 case igc_i225: 2741 /* Turn on MSI-X capability first, or our settings 2742 * won't stick. And it will take days to debug. 2743 */ 2744 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 2745 IGC_GPIE_PBA | IGC_GPIE_EIAME | 2746 IGC_GPIE_NSICR); 2747 2748 /* enable msix_other interrupt */ 2749 adapter->eims_other = BIT(vector); 2750 tmp = (vector++ | IGC_IVAR_VALID) << 8; 2751 2752 wr32(IGC_IVAR_MISC, tmp); 2753 break; 2754 default: 2755 /* do nothing, since nothing else supports MSI-X */ 2756 break; 2757 } /* switch (hw->mac.type) */ 2758 2759 adapter->eims_enable_mask |= adapter->eims_other; 2760 2761 for (i = 0; i < adapter->num_q_vectors; i++) 2762 igc_assign_vector(adapter->q_vector[i], vector++); 2763 2764 wrfl(); 2765 } 2766 2767 /** 2768 * igc_irq_enable - Enable default interrupt generation settings 2769 * @adapter: board private structure 2770 */ 2771 static void igc_irq_enable(struct igc_adapter *adapter) 2772 { 2773 struct igc_hw *hw = &adapter->hw; 2774 2775 if (adapter->msix_entries) { 2776 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 2777 u32 regval = rd32(IGC_EIAC); 2778 2779 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 2780 regval = rd32(IGC_EIAM); 2781 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 2782 wr32(IGC_EIMS, adapter->eims_enable_mask); 2783 wr32(IGC_IMS, ims); 2784 } else { 2785 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2786 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2787 } 2788 } 2789 2790 /** 2791 * igc_irq_disable - Mask off interrupt generation on the NIC 2792 * @adapter: board private structure 2793 */ 2794 static void igc_irq_disable(struct igc_adapter *adapter) 2795 { 2796 struct igc_hw *hw = &adapter->hw; 2797 2798 if (adapter->msix_entries) { 2799 u32 regval = rd32(IGC_EIAM); 2800 2801 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 2802 wr32(IGC_EIMC, adapter->eims_enable_mask); 2803 regval = rd32(IGC_EIAC); 2804 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 2805 } 2806 2807 wr32(IGC_IAM, 0); 2808 wr32(IGC_IMC, ~0); 2809 wrfl(); 2810 2811 if (adapter->msix_entries) { 2812 int vector = 0, i; 2813 2814 synchronize_irq(adapter->msix_entries[vector++].vector); 2815 2816 for (i = 0; i < adapter->num_q_vectors; i++) 2817 synchronize_irq(adapter->msix_entries[vector++].vector); 2818 } else { 2819 synchronize_irq(adapter->pdev->irq); 2820 } 2821 } 2822 2823 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 2824 const u32 max_rss_queues) 2825 { 2826 /* Determine if we need to pair queues. */ 2827 /* If rss_queues > half of max_rss_queues, pair the queues in 2828 * order to conserve interrupts due to limited supply. 2829 */ 2830 if (adapter->rss_queues > (max_rss_queues / 2)) 2831 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 2832 else 2833 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 2834 } 2835 2836 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 2837 { 2838 return IGC_MAX_RX_QUEUES; 2839 } 2840 2841 static void igc_init_queue_configuration(struct igc_adapter *adapter) 2842 { 2843 u32 max_rss_queues; 2844 2845 max_rss_queues = igc_get_max_rss_queues(adapter); 2846 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 2847 2848 igc_set_flag_queue_pairs(adapter, max_rss_queues); 2849 } 2850 2851 /** 2852 * igc_reset_q_vector - Reset config for interrupt vector 2853 * @adapter: board private structure to initialize 2854 * @v_idx: Index of vector to be reset 2855 * 2856 * If NAPI is enabled it will delete any references to the 2857 * NAPI struct. This is preparation for igc_free_q_vector. 2858 */ 2859 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 2860 { 2861 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2862 2863 /* if we're coming from igc_set_interrupt_capability, the vectors are 2864 * not yet allocated 2865 */ 2866 if (!q_vector) 2867 return; 2868 2869 if (q_vector->tx.ring) 2870 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 2871 2872 if (q_vector->rx.ring) 2873 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 2874 2875 netif_napi_del(&q_vector->napi); 2876 } 2877 2878 /** 2879 * igc_free_q_vector - Free memory allocated for specific interrupt vector 2880 * @adapter: board private structure to initialize 2881 * @v_idx: Index of vector to be freed 2882 * 2883 * This function frees the memory allocated to the q_vector. 2884 */ 2885 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 2886 { 2887 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2888 2889 adapter->q_vector[v_idx] = NULL; 2890 2891 /* igc_get_stats64() might access the rings on this vector, 2892 * we must wait a grace period before freeing it. 2893 */ 2894 if (q_vector) 2895 kfree_rcu(q_vector, rcu); 2896 } 2897 2898 /** 2899 * igc_free_q_vectors - Free memory allocated for interrupt vectors 2900 * @adapter: board private structure to initialize 2901 * 2902 * This function frees the memory allocated to the q_vectors. In addition if 2903 * NAPI is enabled it will delete any references to the NAPI struct prior 2904 * to freeing the q_vector. 2905 */ 2906 static void igc_free_q_vectors(struct igc_adapter *adapter) 2907 { 2908 int v_idx = adapter->num_q_vectors; 2909 2910 adapter->num_tx_queues = 0; 2911 adapter->num_rx_queues = 0; 2912 adapter->num_q_vectors = 0; 2913 2914 while (v_idx--) { 2915 igc_reset_q_vector(adapter, v_idx); 2916 igc_free_q_vector(adapter, v_idx); 2917 } 2918 } 2919 2920 /** 2921 * igc_update_itr - update the dynamic ITR value based on statistics 2922 * @q_vector: pointer to q_vector 2923 * @ring_container: ring info to update the itr for 2924 * 2925 * Stores a new ITR value based on packets and byte 2926 * counts during the last interrupt. The advantage of per interrupt 2927 * computation is faster updates and more accurate ITR for the current 2928 * traffic pattern. Constants in this function were computed 2929 * based on theoretical maximum wire speed and thresholds were set based 2930 * on testing data as well as attempting to minimize response time 2931 * while increasing bulk throughput. 2932 * NOTE: These calculations are only valid when operating in a single- 2933 * queue environment. 2934 */ 2935 static void igc_update_itr(struct igc_q_vector *q_vector, 2936 struct igc_ring_container *ring_container) 2937 { 2938 unsigned int packets = ring_container->total_packets; 2939 unsigned int bytes = ring_container->total_bytes; 2940 u8 itrval = ring_container->itr; 2941 2942 /* no packets, exit with status unchanged */ 2943 if (packets == 0) 2944 return; 2945 2946 switch (itrval) { 2947 case lowest_latency: 2948 /* handle TSO and jumbo frames */ 2949 if (bytes / packets > 8000) 2950 itrval = bulk_latency; 2951 else if ((packets < 5) && (bytes > 512)) 2952 itrval = low_latency; 2953 break; 2954 case low_latency: /* 50 usec aka 20000 ints/s */ 2955 if (bytes > 10000) { 2956 /* this if handles the TSO accounting */ 2957 if (bytes / packets > 8000) 2958 itrval = bulk_latency; 2959 else if ((packets < 10) || ((bytes / packets) > 1200)) 2960 itrval = bulk_latency; 2961 else if ((packets > 35)) 2962 itrval = lowest_latency; 2963 } else if (bytes / packets > 2000) { 2964 itrval = bulk_latency; 2965 } else if (packets <= 2 && bytes < 512) { 2966 itrval = lowest_latency; 2967 } 2968 break; 2969 case bulk_latency: /* 250 usec aka 4000 ints/s */ 2970 if (bytes > 25000) { 2971 if (packets > 35) 2972 itrval = low_latency; 2973 } else if (bytes < 1500) { 2974 itrval = low_latency; 2975 } 2976 break; 2977 } 2978 2979 /* clear work counters since we have the values we need */ 2980 ring_container->total_bytes = 0; 2981 ring_container->total_packets = 0; 2982 2983 /* write updated itr to ring container */ 2984 ring_container->itr = itrval; 2985 } 2986 2987 static void igc_set_itr(struct igc_q_vector *q_vector) 2988 { 2989 struct igc_adapter *adapter = q_vector->adapter; 2990 u32 new_itr = q_vector->itr_val; 2991 u8 current_itr = 0; 2992 2993 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 2994 switch (adapter->link_speed) { 2995 case SPEED_10: 2996 case SPEED_100: 2997 current_itr = 0; 2998 new_itr = IGC_4K_ITR; 2999 goto set_itr_now; 3000 default: 3001 break; 3002 } 3003 3004 igc_update_itr(q_vector, &q_vector->tx); 3005 igc_update_itr(q_vector, &q_vector->rx); 3006 3007 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 3008 3009 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3010 if (current_itr == lowest_latency && 3011 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3012 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3013 current_itr = low_latency; 3014 3015 switch (current_itr) { 3016 /* counts and packets in update_itr are dependent on these numbers */ 3017 case lowest_latency: 3018 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 3019 break; 3020 case low_latency: 3021 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 3022 break; 3023 case bulk_latency: 3024 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 3025 break; 3026 default: 3027 break; 3028 } 3029 3030 set_itr_now: 3031 if (new_itr != q_vector->itr_val) { 3032 /* this attempts to bias the interrupt rate towards Bulk 3033 * by adding intermediate steps when interrupt rate is 3034 * increasing 3035 */ 3036 new_itr = new_itr > q_vector->itr_val ? 3037 max((new_itr * q_vector->itr_val) / 3038 (new_itr + (q_vector->itr_val >> 2)), 3039 new_itr) : new_itr; 3040 /* Don't write the value here; it resets the adapter's 3041 * internal timer, and causes us to delay far longer than 3042 * we should between interrupts. Instead, we write the ITR 3043 * value at the beginning of the next interrupt so the timing 3044 * ends up being correct. 3045 */ 3046 q_vector->itr_val = new_itr; 3047 q_vector->set_itr = 1; 3048 } 3049 } 3050 3051 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 3052 { 3053 int v_idx = adapter->num_q_vectors; 3054 3055 if (adapter->msix_entries) { 3056 pci_disable_msix(adapter->pdev); 3057 kfree(adapter->msix_entries); 3058 adapter->msix_entries = NULL; 3059 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 3060 pci_disable_msi(adapter->pdev); 3061 } 3062 3063 while (v_idx--) 3064 igc_reset_q_vector(adapter, v_idx); 3065 } 3066 3067 /** 3068 * igc_set_interrupt_capability - set MSI or MSI-X if supported 3069 * @adapter: Pointer to adapter structure 3070 * @msix: boolean value for MSI-X capability 3071 * 3072 * Attempt to configure interrupts using the best available 3073 * capabilities of the hardware and kernel. 3074 */ 3075 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 3076 bool msix) 3077 { 3078 int numvecs, i; 3079 int err; 3080 3081 if (!msix) 3082 goto msi_only; 3083 adapter->flags |= IGC_FLAG_HAS_MSIX; 3084 3085 /* Number of supported queues. */ 3086 adapter->num_rx_queues = adapter->rss_queues; 3087 3088 adapter->num_tx_queues = adapter->rss_queues; 3089 3090 /* start with one vector for every Rx queue */ 3091 numvecs = adapter->num_rx_queues; 3092 3093 /* if Tx handler is separate add 1 for every Tx queue */ 3094 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 3095 numvecs += adapter->num_tx_queues; 3096 3097 /* store the number of vectors reserved for queues */ 3098 adapter->num_q_vectors = numvecs; 3099 3100 /* add 1 vector for link status interrupts */ 3101 numvecs++; 3102 3103 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), 3104 GFP_KERNEL); 3105 3106 if (!adapter->msix_entries) 3107 return; 3108 3109 /* populate entry values */ 3110 for (i = 0; i < numvecs; i++) 3111 adapter->msix_entries[i].entry = i; 3112 3113 err = pci_enable_msix_range(adapter->pdev, 3114 adapter->msix_entries, 3115 numvecs, 3116 numvecs); 3117 if (err > 0) 3118 return; 3119 3120 kfree(adapter->msix_entries); 3121 adapter->msix_entries = NULL; 3122 3123 igc_reset_interrupt_capability(adapter); 3124 3125 msi_only: 3126 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 3127 3128 adapter->rss_queues = 1; 3129 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 3130 adapter->num_rx_queues = 1; 3131 adapter->num_tx_queues = 1; 3132 adapter->num_q_vectors = 1; 3133 if (!pci_enable_msi(adapter->pdev)) 3134 adapter->flags |= IGC_FLAG_HAS_MSI; 3135 } 3136 3137 /** 3138 * igc_update_ring_itr - update the dynamic ITR value based on packet size 3139 * @q_vector: pointer to q_vector 3140 * 3141 * Stores a new ITR value based on strictly on packet size. This 3142 * algorithm is less sophisticated than that used in igc_update_itr, 3143 * due to the difficulty of synchronizing statistics across multiple 3144 * receive rings. The divisors and thresholds used by this function 3145 * were determined based on theoretical maximum wire speed and testing 3146 * data, in order to minimize response time while increasing bulk 3147 * throughput. 3148 * NOTE: This function is called only when operating in a multiqueue 3149 * receive environment. 3150 */ 3151 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 3152 { 3153 struct igc_adapter *adapter = q_vector->adapter; 3154 int new_val = q_vector->itr_val; 3155 int avg_wire_size = 0; 3156 unsigned int packets; 3157 3158 /* For non-gigabit speeds, just fix the interrupt rate at 4000 3159 * ints/sec - ITR timer value of 120 ticks. 3160 */ 3161 switch (adapter->link_speed) { 3162 case SPEED_10: 3163 case SPEED_100: 3164 new_val = IGC_4K_ITR; 3165 goto set_itr_val; 3166 default: 3167 break; 3168 } 3169 3170 packets = q_vector->rx.total_packets; 3171 if (packets) 3172 avg_wire_size = q_vector->rx.total_bytes / packets; 3173 3174 packets = q_vector->tx.total_packets; 3175 if (packets) 3176 avg_wire_size = max_t(u32, avg_wire_size, 3177 q_vector->tx.total_bytes / packets); 3178 3179 /* if avg_wire_size isn't set no work was done */ 3180 if (!avg_wire_size) 3181 goto clear_counts; 3182 3183 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 3184 avg_wire_size += 24; 3185 3186 /* Don't starve jumbo frames */ 3187 avg_wire_size = min(avg_wire_size, 3000); 3188 3189 /* Give a little boost to mid-size frames */ 3190 if (avg_wire_size > 300 && avg_wire_size < 1200) 3191 new_val = avg_wire_size / 3; 3192 else 3193 new_val = avg_wire_size / 2; 3194 3195 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3196 if (new_val < IGC_20K_ITR && 3197 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3198 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3199 new_val = IGC_20K_ITR; 3200 3201 set_itr_val: 3202 if (new_val != q_vector->itr_val) { 3203 q_vector->itr_val = new_val; 3204 q_vector->set_itr = 1; 3205 } 3206 clear_counts: 3207 q_vector->rx.total_bytes = 0; 3208 q_vector->rx.total_packets = 0; 3209 q_vector->tx.total_bytes = 0; 3210 q_vector->tx.total_packets = 0; 3211 } 3212 3213 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 3214 { 3215 struct igc_adapter *adapter = q_vector->adapter; 3216 struct igc_hw *hw = &adapter->hw; 3217 3218 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 3219 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 3220 if (adapter->num_q_vectors == 1) 3221 igc_set_itr(q_vector); 3222 else 3223 igc_update_ring_itr(q_vector); 3224 } 3225 3226 if (!test_bit(__IGC_DOWN, &adapter->state)) { 3227 if (adapter->msix_entries) 3228 wr32(IGC_EIMS, q_vector->eims_value); 3229 else 3230 igc_irq_enable(adapter); 3231 } 3232 } 3233 3234 static void igc_add_ring(struct igc_ring *ring, 3235 struct igc_ring_container *head) 3236 { 3237 head->ring = ring; 3238 head->count++; 3239 } 3240 3241 /** 3242 * igc_cache_ring_register - Descriptor ring to register mapping 3243 * @adapter: board private structure to initialize 3244 * 3245 * Once we know the feature-set enabled for the device, we'll cache 3246 * the register offset the descriptor ring is assigned to. 3247 */ 3248 static void igc_cache_ring_register(struct igc_adapter *adapter) 3249 { 3250 int i = 0, j = 0; 3251 3252 switch (adapter->hw.mac.type) { 3253 case igc_i225: 3254 default: 3255 for (; i < adapter->num_rx_queues; i++) 3256 adapter->rx_ring[i]->reg_idx = i; 3257 for (; j < adapter->num_tx_queues; j++) 3258 adapter->tx_ring[j]->reg_idx = j; 3259 break; 3260 } 3261 } 3262 3263 /** 3264 * igc_poll - NAPI Rx polling callback 3265 * @napi: napi polling structure 3266 * @budget: count of how many packets we should handle 3267 */ 3268 static int igc_poll(struct napi_struct *napi, int budget) 3269 { 3270 struct igc_q_vector *q_vector = container_of(napi, 3271 struct igc_q_vector, 3272 napi); 3273 bool clean_complete = true; 3274 int work_done = 0; 3275 3276 if (q_vector->tx.ring) 3277 clean_complete = igc_clean_tx_irq(q_vector, budget); 3278 3279 if (q_vector->rx.ring) { 3280 int cleaned = igc_clean_rx_irq(q_vector, budget); 3281 3282 work_done += cleaned; 3283 if (cleaned >= budget) 3284 clean_complete = false; 3285 } 3286 3287 /* If all work not completed, return budget and keep polling */ 3288 if (!clean_complete) 3289 return budget; 3290 3291 /* Exit the polling mode, but don't re-enable interrupts if stack might 3292 * poll us due to busy-polling 3293 */ 3294 if (likely(napi_complete_done(napi, work_done))) 3295 igc_ring_irq_enable(q_vector); 3296 3297 return min(work_done, budget - 1); 3298 } 3299 3300 /** 3301 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 3302 * @adapter: board private structure to initialize 3303 * @v_count: q_vectors allocated on adapter, used for ring interleaving 3304 * @v_idx: index of vector in adapter struct 3305 * @txr_count: total number of Tx rings to allocate 3306 * @txr_idx: index of first Tx ring to allocate 3307 * @rxr_count: total number of Rx rings to allocate 3308 * @rxr_idx: index of first Rx ring to allocate 3309 * 3310 * We allocate one q_vector. If allocation fails we return -ENOMEM. 3311 */ 3312 static int igc_alloc_q_vector(struct igc_adapter *adapter, 3313 unsigned int v_count, unsigned int v_idx, 3314 unsigned int txr_count, unsigned int txr_idx, 3315 unsigned int rxr_count, unsigned int rxr_idx) 3316 { 3317 struct igc_q_vector *q_vector; 3318 struct igc_ring *ring; 3319 int ring_count; 3320 3321 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 3322 if (txr_count > 1 || rxr_count > 1) 3323 return -ENOMEM; 3324 3325 ring_count = txr_count + rxr_count; 3326 3327 /* allocate q_vector and rings */ 3328 q_vector = adapter->q_vector[v_idx]; 3329 if (!q_vector) 3330 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), 3331 GFP_KERNEL); 3332 else 3333 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 3334 if (!q_vector) 3335 return -ENOMEM; 3336 3337 /* initialize NAPI */ 3338 netif_napi_add(adapter->netdev, &q_vector->napi, 3339 igc_poll, 64); 3340 3341 /* tie q_vector and adapter together */ 3342 adapter->q_vector[v_idx] = q_vector; 3343 q_vector->adapter = adapter; 3344 3345 /* initialize work limits */ 3346 q_vector->tx.work_limit = adapter->tx_work_limit; 3347 3348 /* initialize ITR configuration */ 3349 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 3350 q_vector->itr_val = IGC_START_ITR; 3351 3352 /* initialize pointer to rings */ 3353 ring = q_vector->ring; 3354 3355 /* initialize ITR */ 3356 if (rxr_count) { 3357 /* rx or rx/tx vector */ 3358 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 3359 q_vector->itr_val = adapter->rx_itr_setting; 3360 } else { 3361 /* tx only vector */ 3362 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 3363 q_vector->itr_val = adapter->tx_itr_setting; 3364 } 3365 3366 if (txr_count) { 3367 /* assign generic ring traits */ 3368 ring->dev = &adapter->pdev->dev; 3369 ring->netdev = adapter->netdev; 3370 3371 /* configure backlink on ring */ 3372 ring->q_vector = q_vector; 3373 3374 /* update q_vector Tx values */ 3375 igc_add_ring(ring, &q_vector->tx); 3376 3377 /* apply Tx specific ring traits */ 3378 ring->count = adapter->tx_ring_count; 3379 ring->queue_index = txr_idx; 3380 3381 /* assign ring to adapter */ 3382 adapter->tx_ring[txr_idx] = ring; 3383 3384 /* push pointer to next ring */ 3385 ring++; 3386 } 3387 3388 if (rxr_count) { 3389 /* assign generic ring traits */ 3390 ring->dev = &adapter->pdev->dev; 3391 ring->netdev = adapter->netdev; 3392 3393 /* configure backlink on ring */ 3394 ring->q_vector = q_vector; 3395 3396 /* update q_vector Rx values */ 3397 igc_add_ring(ring, &q_vector->rx); 3398 3399 /* apply Rx specific ring traits */ 3400 ring->count = adapter->rx_ring_count; 3401 ring->queue_index = rxr_idx; 3402 3403 /* assign ring to adapter */ 3404 adapter->rx_ring[rxr_idx] = ring; 3405 } 3406 3407 return 0; 3408 } 3409 3410 /** 3411 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 3412 * @adapter: board private structure to initialize 3413 * 3414 * We allocate one q_vector per queue interrupt. If allocation fails we 3415 * return -ENOMEM. 3416 */ 3417 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 3418 { 3419 int rxr_remaining = adapter->num_rx_queues; 3420 int txr_remaining = adapter->num_tx_queues; 3421 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 3422 int q_vectors = adapter->num_q_vectors; 3423 int err; 3424 3425 if (q_vectors >= (rxr_remaining + txr_remaining)) { 3426 for (; rxr_remaining; v_idx++) { 3427 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3428 0, 0, 1, rxr_idx); 3429 3430 if (err) 3431 goto err_out; 3432 3433 /* update counts and index */ 3434 rxr_remaining--; 3435 rxr_idx++; 3436 } 3437 } 3438 3439 for (; v_idx < q_vectors; v_idx++) { 3440 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 3441 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 3442 3443 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3444 tqpv, txr_idx, rqpv, rxr_idx); 3445 3446 if (err) 3447 goto err_out; 3448 3449 /* update counts and index */ 3450 rxr_remaining -= rqpv; 3451 txr_remaining -= tqpv; 3452 rxr_idx++; 3453 txr_idx++; 3454 } 3455 3456 return 0; 3457 3458 err_out: 3459 adapter->num_tx_queues = 0; 3460 adapter->num_rx_queues = 0; 3461 adapter->num_q_vectors = 0; 3462 3463 while (v_idx--) 3464 igc_free_q_vector(adapter, v_idx); 3465 3466 return -ENOMEM; 3467 } 3468 3469 /** 3470 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 3471 * @adapter: Pointer to adapter structure 3472 * @msix: boolean for MSI-X capability 3473 * 3474 * This function initializes the interrupts and allocates all of the queues. 3475 */ 3476 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 3477 { 3478 struct net_device *dev = adapter->netdev; 3479 int err = 0; 3480 3481 igc_set_interrupt_capability(adapter, msix); 3482 3483 err = igc_alloc_q_vectors(adapter); 3484 if (err) { 3485 netdev_err(dev, "Unable to allocate memory for vectors\n"); 3486 goto err_alloc_q_vectors; 3487 } 3488 3489 igc_cache_ring_register(adapter); 3490 3491 return 0; 3492 3493 err_alloc_q_vectors: 3494 igc_reset_interrupt_capability(adapter); 3495 return err; 3496 } 3497 3498 /** 3499 * igc_sw_init - Initialize general software structures (struct igc_adapter) 3500 * @adapter: board private structure to initialize 3501 * 3502 * igc_sw_init initializes the Adapter private data structure. 3503 * Fields are initialized based on PCI device information and 3504 * OS network device settings (MTU size). 3505 */ 3506 static int igc_sw_init(struct igc_adapter *adapter) 3507 { 3508 struct net_device *netdev = adapter->netdev; 3509 struct pci_dev *pdev = adapter->pdev; 3510 struct igc_hw *hw = &adapter->hw; 3511 3512 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 3513 3514 /* set default ring sizes */ 3515 adapter->tx_ring_count = IGC_DEFAULT_TXD; 3516 adapter->rx_ring_count = IGC_DEFAULT_RXD; 3517 3518 /* set default ITR values */ 3519 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 3520 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 3521 3522 /* set default work limits */ 3523 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 3524 3525 /* adjust max frame to be at least the size of a standard frame */ 3526 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 3527 VLAN_HLEN; 3528 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 3529 3530 mutex_init(&adapter->nfc_rule_lock); 3531 INIT_LIST_HEAD(&adapter->nfc_rule_list); 3532 adapter->nfc_rule_count = 0; 3533 3534 spin_lock_init(&adapter->stats64_lock); 3535 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 3536 adapter->flags |= IGC_FLAG_HAS_MSIX; 3537 3538 igc_init_queue_configuration(adapter); 3539 3540 /* This call may decrease the number of queues */ 3541 if (igc_init_interrupt_scheme(adapter, true)) { 3542 netdev_err(netdev, "Unable to allocate memory for queues\n"); 3543 return -ENOMEM; 3544 } 3545 3546 /* Explicitly disable IRQ since the NIC can be in any state. */ 3547 igc_irq_disable(adapter); 3548 3549 set_bit(__IGC_DOWN, &adapter->state); 3550 3551 return 0; 3552 } 3553 3554 /** 3555 * igc_up - Open the interface and prepare it to handle traffic 3556 * @adapter: board private structure 3557 */ 3558 void igc_up(struct igc_adapter *adapter) 3559 { 3560 struct igc_hw *hw = &adapter->hw; 3561 int i = 0; 3562 3563 /* hardware has been reset, we need to reload some things */ 3564 igc_configure(adapter); 3565 3566 clear_bit(__IGC_DOWN, &adapter->state); 3567 3568 for (i = 0; i < adapter->num_q_vectors; i++) 3569 napi_enable(&adapter->q_vector[i]->napi); 3570 3571 if (adapter->msix_entries) 3572 igc_configure_msix(adapter); 3573 else 3574 igc_assign_vector(adapter->q_vector[0], 0); 3575 3576 /* Clear any pending interrupts. */ 3577 rd32(IGC_ICR); 3578 igc_irq_enable(adapter); 3579 3580 netif_tx_start_all_queues(adapter->netdev); 3581 3582 /* start the watchdog. */ 3583 hw->mac.get_link_status = 1; 3584 schedule_work(&adapter->watchdog_task); 3585 } 3586 3587 /** 3588 * igc_update_stats - Update the board statistics counters 3589 * @adapter: board private structure 3590 */ 3591 void igc_update_stats(struct igc_adapter *adapter) 3592 { 3593 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 3594 struct pci_dev *pdev = adapter->pdev; 3595 struct igc_hw *hw = &adapter->hw; 3596 u64 _bytes, _packets; 3597 u64 bytes, packets; 3598 unsigned int start; 3599 u32 mpc; 3600 int i; 3601 3602 /* Prevent stats update while adapter is being reset, or if the pci 3603 * connection is down. 3604 */ 3605 if (adapter->link_speed == 0) 3606 return; 3607 if (pci_channel_offline(pdev)) 3608 return; 3609 3610 packets = 0; 3611 bytes = 0; 3612 3613 rcu_read_lock(); 3614 for (i = 0; i < adapter->num_rx_queues; i++) { 3615 struct igc_ring *ring = adapter->rx_ring[i]; 3616 u32 rqdpc = rd32(IGC_RQDPC(i)); 3617 3618 if (hw->mac.type >= igc_i225) 3619 wr32(IGC_RQDPC(i), 0); 3620 3621 if (rqdpc) { 3622 ring->rx_stats.drops += rqdpc; 3623 net_stats->rx_fifo_errors += rqdpc; 3624 } 3625 3626 do { 3627 start = u64_stats_fetch_begin_irq(&ring->rx_syncp); 3628 _bytes = ring->rx_stats.bytes; 3629 _packets = ring->rx_stats.packets; 3630 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); 3631 bytes += _bytes; 3632 packets += _packets; 3633 } 3634 3635 net_stats->rx_bytes = bytes; 3636 net_stats->rx_packets = packets; 3637 3638 packets = 0; 3639 bytes = 0; 3640 for (i = 0; i < adapter->num_tx_queues; i++) { 3641 struct igc_ring *ring = adapter->tx_ring[i]; 3642 3643 do { 3644 start = u64_stats_fetch_begin_irq(&ring->tx_syncp); 3645 _bytes = ring->tx_stats.bytes; 3646 _packets = ring->tx_stats.packets; 3647 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); 3648 bytes += _bytes; 3649 packets += _packets; 3650 } 3651 net_stats->tx_bytes = bytes; 3652 net_stats->tx_packets = packets; 3653 rcu_read_unlock(); 3654 3655 /* read stats registers */ 3656 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 3657 adapter->stats.gprc += rd32(IGC_GPRC); 3658 adapter->stats.gorc += rd32(IGC_GORCL); 3659 rd32(IGC_GORCH); /* clear GORCL */ 3660 adapter->stats.bprc += rd32(IGC_BPRC); 3661 adapter->stats.mprc += rd32(IGC_MPRC); 3662 adapter->stats.roc += rd32(IGC_ROC); 3663 3664 adapter->stats.prc64 += rd32(IGC_PRC64); 3665 adapter->stats.prc127 += rd32(IGC_PRC127); 3666 adapter->stats.prc255 += rd32(IGC_PRC255); 3667 adapter->stats.prc511 += rd32(IGC_PRC511); 3668 adapter->stats.prc1023 += rd32(IGC_PRC1023); 3669 adapter->stats.prc1522 += rd32(IGC_PRC1522); 3670 adapter->stats.tlpic += rd32(IGC_TLPIC); 3671 adapter->stats.rlpic += rd32(IGC_RLPIC); 3672 adapter->stats.hgptc += rd32(IGC_HGPTC); 3673 3674 mpc = rd32(IGC_MPC); 3675 adapter->stats.mpc += mpc; 3676 net_stats->rx_fifo_errors += mpc; 3677 adapter->stats.scc += rd32(IGC_SCC); 3678 adapter->stats.ecol += rd32(IGC_ECOL); 3679 adapter->stats.mcc += rd32(IGC_MCC); 3680 adapter->stats.latecol += rd32(IGC_LATECOL); 3681 adapter->stats.dc += rd32(IGC_DC); 3682 adapter->stats.rlec += rd32(IGC_RLEC); 3683 adapter->stats.xonrxc += rd32(IGC_XONRXC); 3684 adapter->stats.xontxc += rd32(IGC_XONTXC); 3685 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 3686 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 3687 adapter->stats.fcruc += rd32(IGC_FCRUC); 3688 adapter->stats.gptc += rd32(IGC_GPTC); 3689 adapter->stats.gotc += rd32(IGC_GOTCL); 3690 rd32(IGC_GOTCH); /* clear GOTCL */ 3691 adapter->stats.rnbc += rd32(IGC_RNBC); 3692 adapter->stats.ruc += rd32(IGC_RUC); 3693 adapter->stats.rfc += rd32(IGC_RFC); 3694 adapter->stats.rjc += rd32(IGC_RJC); 3695 adapter->stats.tor += rd32(IGC_TORH); 3696 adapter->stats.tot += rd32(IGC_TOTH); 3697 adapter->stats.tpr += rd32(IGC_TPR); 3698 3699 adapter->stats.ptc64 += rd32(IGC_PTC64); 3700 adapter->stats.ptc127 += rd32(IGC_PTC127); 3701 adapter->stats.ptc255 += rd32(IGC_PTC255); 3702 adapter->stats.ptc511 += rd32(IGC_PTC511); 3703 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 3704 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 3705 3706 adapter->stats.mptc += rd32(IGC_MPTC); 3707 adapter->stats.bptc += rd32(IGC_BPTC); 3708 3709 adapter->stats.tpt += rd32(IGC_TPT); 3710 adapter->stats.colc += rd32(IGC_COLC); 3711 adapter->stats.colc += rd32(IGC_RERC); 3712 3713 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 3714 3715 adapter->stats.tsctc += rd32(IGC_TSCTC); 3716 3717 adapter->stats.iac += rd32(IGC_IAC); 3718 3719 /* Fill out the OS statistics structure */ 3720 net_stats->multicast = adapter->stats.mprc; 3721 net_stats->collisions = adapter->stats.colc; 3722 3723 /* Rx Errors */ 3724 3725 /* RLEC on some newer hardware can be incorrect so build 3726 * our own version based on RUC and ROC 3727 */ 3728 net_stats->rx_errors = adapter->stats.rxerrc + 3729 adapter->stats.crcerrs + adapter->stats.algnerrc + 3730 adapter->stats.ruc + adapter->stats.roc + 3731 adapter->stats.cexterr; 3732 net_stats->rx_length_errors = adapter->stats.ruc + 3733 adapter->stats.roc; 3734 net_stats->rx_crc_errors = adapter->stats.crcerrs; 3735 net_stats->rx_frame_errors = adapter->stats.algnerrc; 3736 net_stats->rx_missed_errors = adapter->stats.mpc; 3737 3738 /* Tx Errors */ 3739 net_stats->tx_errors = adapter->stats.ecol + 3740 adapter->stats.latecol; 3741 net_stats->tx_aborted_errors = adapter->stats.ecol; 3742 net_stats->tx_window_errors = adapter->stats.latecol; 3743 net_stats->tx_carrier_errors = adapter->stats.tncrs; 3744 3745 /* Tx Dropped needs to be maintained elsewhere */ 3746 3747 /* Management Stats */ 3748 adapter->stats.mgptc += rd32(IGC_MGTPTC); 3749 adapter->stats.mgprc += rd32(IGC_MGTPRC); 3750 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 3751 } 3752 3753 /** 3754 * igc_down - Close the interface 3755 * @adapter: board private structure 3756 */ 3757 void igc_down(struct igc_adapter *adapter) 3758 { 3759 struct net_device *netdev = adapter->netdev; 3760 struct igc_hw *hw = &adapter->hw; 3761 u32 tctl, rctl; 3762 int i = 0; 3763 3764 set_bit(__IGC_DOWN, &adapter->state); 3765 3766 igc_ptp_suspend(adapter); 3767 3768 /* disable receives in the hardware */ 3769 rctl = rd32(IGC_RCTL); 3770 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 3771 /* flush and sleep below */ 3772 3773 /* set trans_start so we don't get spurious watchdogs during reset */ 3774 netif_trans_update(netdev); 3775 3776 netif_carrier_off(netdev); 3777 netif_tx_stop_all_queues(netdev); 3778 3779 /* disable transmits in the hardware */ 3780 tctl = rd32(IGC_TCTL); 3781 tctl &= ~IGC_TCTL_EN; 3782 wr32(IGC_TCTL, tctl); 3783 /* flush both disables and wait for them to finish */ 3784 wrfl(); 3785 usleep_range(10000, 20000); 3786 3787 igc_irq_disable(adapter); 3788 3789 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3790 3791 for (i = 0; i < adapter->num_q_vectors; i++) { 3792 if (adapter->q_vector[i]) { 3793 napi_synchronize(&adapter->q_vector[i]->napi); 3794 napi_disable(&adapter->q_vector[i]->napi); 3795 } 3796 } 3797 3798 del_timer_sync(&adapter->watchdog_timer); 3799 del_timer_sync(&adapter->phy_info_timer); 3800 3801 /* record the stats before reset*/ 3802 spin_lock(&adapter->stats64_lock); 3803 igc_update_stats(adapter); 3804 spin_unlock(&adapter->stats64_lock); 3805 3806 adapter->link_speed = 0; 3807 adapter->link_duplex = 0; 3808 3809 if (!pci_channel_offline(adapter->pdev)) 3810 igc_reset(adapter); 3811 3812 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 3813 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 3814 3815 igc_clean_all_tx_rings(adapter); 3816 igc_clean_all_rx_rings(adapter); 3817 } 3818 3819 void igc_reinit_locked(struct igc_adapter *adapter) 3820 { 3821 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3822 usleep_range(1000, 2000); 3823 igc_down(adapter); 3824 igc_up(adapter); 3825 clear_bit(__IGC_RESETTING, &adapter->state); 3826 } 3827 3828 static void igc_reset_task(struct work_struct *work) 3829 { 3830 struct igc_adapter *adapter; 3831 3832 adapter = container_of(work, struct igc_adapter, reset_task); 3833 3834 rtnl_lock(); 3835 /* If we're already down or resetting, just bail */ 3836 if (test_bit(__IGC_DOWN, &adapter->state) || 3837 test_bit(__IGC_RESETTING, &adapter->state)) { 3838 rtnl_unlock(); 3839 return; 3840 } 3841 3842 igc_rings_dump(adapter); 3843 igc_regs_dump(adapter); 3844 netdev_err(adapter->netdev, "Reset adapter\n"); 3845 igc_reinit_locked(adapter); 3846 rtnl_unlock(); 3847 } 3848 3849 /** 3850 * igc_change_mtu - Change the Maximum Transfer Unit 3851 * @netdev: network interface device structure 3852 * @new_mtu: new value for maximum frame size 3853 * 3854 * Returns 0 on success, negative on failure 3855 */ 3856 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 3857 { 3858 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 3859 struct igc_adapter *adapter = netdev_priv(netdev); 3860 3861 /* adjust max frame to be at least the size of a standard frame */ 3862 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 3863 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 3864 3865 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3866 usleep_range(1000, 2000); 3867 3868 /* igc_down has a dependency on max_frame_size */ 3869 adapter->max_frame_size = max_frame; 3870 3871 if (netif_running(netdev)) 3872 igc_down(adapter); 3873 3874 netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 3875 netdev->mtu = new_mtu; 3876 3877 if (netif_running(netdev)) 3878 igc_up(adapter); 3879 else 3880 igc_reset(adapter); 3881 3882 clear_bit(__IGC_RESETTING, &adapter->state); 3883 3884 return 0; 3885 } 3886 3887 /** 3888 * igc_get_stats64 - Get System Network Statistics 3889 * @netdev: network interface device structure 3890 * @stats: rtnl_link_stats64 pointer 3891 * 3892 * Returns the address of the device statistics structure. 3893 * The statistics are updated here and also from the timer callback. 3894 */ 3895 static void igc_get_stats64(struct net_device *netdev, 3896 struct rtnl_link_stats64 *stats) 3897 { 3898 struct igc_adapter *adapter = netdev_priv(netdev); 3899 3900 spin_lock(&adapter->stats64_lock); 3901 if (!test_bit(__IGC_RESETTING, &adapter->state)) 3902 igc_update_stats(adapter); 3903 memcpy(stats, &adapter->stats64, sizeof(*stats)); 3904 spin_unlock(&adapter->stats64_lock); 3905 } 3906 3907 static netdev_features_t igc_fix_features(struct net_device *netdev, 3908 netdev_features_t features) 3909 { 3910 /* Since there is no support for separate Rx/Tx vlan accel 3911 * enable/disable make sure Tx flag is always in same state as Rx. 3912 */ 3913 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3914 features |= NETIF_F_HW_VLAN_CTAG_TX; 3915 else 3916 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 3917 3918 return features; 3919 } 3920 3921 static int igc_set_features(struct net_device *netdev, 3922 netdev_features_t features) 3923 { 3924 netdev_features_t changed = netdev->features ^ features; 3925 struct igc_adapter *adapter = netdev_priv(netdev); 3926 3927 /* Add VLAN support */ 3928 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 3929 return 0; 3930 3931 if (!(features & NETIF_F_NTUPLE)) 3932 igc_flush_nfc_rules(adapter); 3933 3934 netdev->features = features; 3935 3936 if (netif_running(netdev)) 3937 igc_reinit_locked(adapter); 3938 else 3939 igc_reset(adapter); 3940 3941 return 1; 3942 } 3943 3944 static netdev_features_t 3945 igc_features_check(struct sk_buff *skb, struct net_device *dev, 3946 netdev_features_t features) 3947 { 3948 unsigned int network_hdr_len, mac_hdr_len; 3949 3950 /* Make certain the headers can be described by a context descriptor */ 3951 mac_hdr_len = skb_network_header(skb) - skb->data; 3952 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 3953 return features & ~(NETIF_F_HW_CSUM | 3954 NETIF_F_SCTP_CRC | 3955 NETIF_F_HW_VLAN_CTAG_TX | 3956 NETIF_F_TSO | 3957 NETIF_F_TSO6); 3958 3959 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 3960 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 3961 return features & ~(NETIF_F_HW_CSUM | 3962 NETIF_F_SCTP_CRC | 3963 NETIF_F_TSO | 3964 NETIF_F_TSO6); 3965 3966 /* We can only support IPv4 TSO in tunnels if we can mangle the 3967 * inner IP ID field, so strip TSO if MANGLEID is not supported. 3968 */ 3969 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 3970 features &= ~NETIF_F_TSO; 3971 3972 return features; 3973 } 3974 3975 static void igc_tsync_interrupt(struct igc_adapter *adapter) 3976 { 3977 struct igc_hw *hw = &adapter->hw; 3978 u32 tsicr = rd32(IGC_TSICR); 3979 u32 ack = 0; 3980 3981 if (tsicr & IGC_TSICR_TXTS) { 3982 /* retrieve hardware timestamp */ 3983 schedule_work(&adapter->ptp_tx_work); 3984 ack |= IGC_TSICR_TXTS; 3985 } 3986 3987 /* acknowledge the interrupts */ 3988 wr32(IGC_TSICR, ack); 3989 } 3990 3991 /** 3992 * igc_msix_other - msix other interrupt handler 3993 * @irq: interrupt number 3994 * @data: pointer to a q_vector 3995 */ 3996 static irqreturn_t igc_msix_other(int irq, void *data) 3997 { 3998 struct igc_adapter *adapter = data; 3999 struct igc_hw *hw = &adapter->hw; 4000 u32 icr = rd32(IGC_ICR); 4001 4002 /* reading ICR causes bit 31 of EICR to be cleared */ 4003 if (icr & IGC_ICR_DRSTA) 4004 schedule_work(&adapter->reset_task); 4005 4006 if (icr & IGC_ICR_DOUTSYNC) { 4007 /* HW is reporting DMA is out of sync */ 4008 adapter->stats.doosync++; 4009 } 4010 4011 if (icr & IGC_ICR_LSC) { 4012 hw->mac.get_link_status = 1; 4013 /* guard against interrupt when we're going down */ 4014 if (!test_bit(__IGC_DOWN, &adapter->state)) 4015 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4016 } 4017 4018 if (icr & IGC_ICR_TS) 4019 igc_tsync_interrupt(adapter); 4020 4021 wr32(IGC_EIMS, adapter->eims_other); 4022 4023 return IRQ_HANDLED; 4024 } 4025 4026 static void igc_write_itr(struct igc_q_vector *q_vector) 4027 { 4028 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 4029 4030 if (!q_vector->set_itr) 4031 return; 4032 4033 if (!itr_val) 4034 itr_val = IGC_ITR_VAL_MASK; 4035 4036 itr_val |= IGC_EITR_CNT_IGNR; 4037 4038 writel(itr_val, q_vector->itr_register); 4039 q_vector->set_itr = 0; 4040 } 4041 4042 static irqreturn_t igc_msix_ring(int irq, void *data) 4043 { 4044 struct igc_q_vector *q_vector = data; 4045 4046 /* Write the ITR value calculated from the previous interrupt. */ 4047 igc_write_itr(q_vector); 4048 4049 napi_schedule(&q_vector->napi); 4050 4051 return IRQ_HANDLED; 4052 } 4053 4054 /** 4055 * igc_request_msix - Initialize MSI-X interrupts 4056 * @adapter: Pointer to adapter structure 4057 * 4058 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 4059 * kernel. 4060 */ 4061 static int igc_request_msix(struct igc_adapter *adapter) 4062 { 4063 int i = 0, err = 0, vector = 0, free_vector = 0; 4064 struct net_device *netdev = adapter->netdev; 4065 4066 err = request_irq(adapter->msix_entries[vector].vector, 4067 &igc_msix_other, 0, netdev->name, adapter); 4068 if (err) 4069 goto err_out; 4070 4071 for (i = 0; i < adapter->num_q_vectors; i++) { 4072 struct igc_q_vector *q_vector = adapter->q_vector[i]; 4073 4074 vector++; 4075 4076 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 4077 4078 if (q_vector->rx.ring && q_vector->tx.ring) 4079 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 4080 q_vector->rx.ring->queue_index); 4081 else if (q_vector->tx.ring) 4082 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 4083 q_vector->tx.ring->queue_index); 4084 else if (q_vector->rx.ring) 4085 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 4086 q_vector->rx.ring->queue_index); 4087 else 4088 sprintf(q_vector->name, "%s-unused", netdev->name); 4089 4090 err = request_irq(adapter->msix_entries[vector].vector, 4091 igc_msix_ring, 0, q_vector->name, 4092 q_vector); 4093 if (err) 4094 goto err_free; 4095 } 4096 4097 igc_configure_msix(adapter); 4098 return 0; 4099 4100 err_free: 4101 /* free already assigned IRQs */ 4102 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 4103 4104 vector--; 4105 for (i = 0; i < vector; i++) { 4106 free_irq(adapter->msix_entries[free_vector++].vector, 4107 adapter->q_vector[i]); 4108 } 4109 err_out: 4110 return err; 4111 } 4112 4113 /** 4114 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 4115 * @adapter: Pointer to adapter structure 4116 * 4117 * This function resets the device so that it has 0 rx queues, tx queues, and 4118 * MSI-X interrupts allocated. 4119 */ 4120 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 4121 { 4122 igc_free_q_vectors(adapter); 4123 igc_reset_interrupt_capability(adapter); 4124 } 4125 4126 /* Need to wait a few seconds after link up to get diagnostic information from 4127 * the phy 4128 */ 4129 static void igc_update_phy_info(struct timer_list *t) 4130 { 4131 struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); 4132 4133 igc_get_phy_info(&adapter->hw); 4134 } 4135 4136 /** 4137 * igc_has_link - check shared code for link and determine up/down 4138 * @adapter: pointer to driver private info 4139 */ 4140 bool igc_has_link(struct igc_adapter *adapter) 4141 { 4142 struct igc_hw *hw = &adapter->hw; 4143 bool link_active = false; 4144 4145 /* get_link_status is set on LSC (link status) interrupt or 4146 * rx sequence error interrupt. get_link_status will stay 4147 * false until the igc_check_for_link establishes link 4148 * for copper adapters ONLY 4149 */ 4150 switch (hw->phy.media_type) { 4151 case igc_media_type_copper: 4152 if (!hw->mac.get_link_status) 4153 return true; 4154 hw->mac.ops.check_for_link(hw); 4155 link_active = !hw->mac.get_link_status; 4156 break; 4157 default: 4158 case igc_media_type_unknown: 4159 break; 4160 } 4161 4162 if (hw->mac.type == igc_i225 && 4163 hw->phy.id == I225_I_PHY_ID) { 4164 if (!netif_carrier_ok(adapter->netdev)) { 4165 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4166 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 4167 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 4168 adapter->link_check_timeout = jiffies; 4169 } 4170 } 4171 4172 return link_active; 4173 } 4174 4175 /** 4176 * igc_watchdog - Timer Call-back 4177 * @t: timer for the watchdog 4178 */ 4179 static void igc_watchdog(struct timer_list *t) 4180 { 4181 struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); 4182 /* Do the rest outside of interrupt context */ 4183 schedule_work(&adapter->watchdog_task); 4184 } 4185 4186 static void igc_watchdog_task(struct work_struct *work) 4187 { 4188 struct igc_adapter *adapter = container_of(work, 4189 struct igc_adapter, 4190 watchdog_task); 4191 struct net_device *netdev = adapter->netdev; 4192 struct igc_hw *hw = &adapter->hw; 4193 struct igc_phy_info *phy = &hw->phy; 4194 u16 phy_data, retry_count = 20; 4195 u32 link; 4196 int i; 4197 4198 link = igc_has_link(adapter); 4199 4200 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 4201 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 4202 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4203 else 4204 link = false; 4205 } 4206 4207 if (link) { 4208 /* Cancel scheduled suspend requests. */ 4209 pm_runtime_resume(netdev->dev.parent); 4210 4211 if (!netif_carrier_ok(netdev)) { 4212 u32 ctrl; 4213 4214 hw->mac.ops.get_speed_and_duplex(hw, 4215 &adapter->link_speed, 4216 &adapter->link_duplex); 4217 4218 ctrl = rd32(IGC_CTRL); 4219 /* Link status message must follow this format */ 4220 netdev_info(netdev, 4221 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 4222 adapter->link_speed, 4223 adapter->link_duplex == FULL_DUPLEX ? 4224 "Full" : "Half", 4225 (ctrl & IGC_CTRL_TFCE) && 4226 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 4227 (ctrl & IGC_CTRL_RFCE) ? "RX" : 4228 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 4229 4230 /* disable EEE if enabled */ 4231 if ((adapter->flags & IGC_FLAG_EEE) && 4232 adapter->link_duplex == HALF_DUPLEX) { 4233 netdev_info(netdev, 4234 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n"); 4235 adapter->hw.dev_spec._base.eee_enable = false; 4236 adapter->flags &= ~IGC_FLAG_EEE; 4237 } 4238 4239 /* check if SmartSpeed worked */ 4240 igc_check_downshift(hw); 4241 if (phy->speed_downgraded) 4242 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 4243 4244 /* adjust timeout factor according to speed/duplex */ 4245 adapter->tx_timeout_factor = 1; 4246 switch (adapter->link_speed) { 4247 case SPEED_10: 4248 adapter->tx_timeout_factor = 14; 4249 break; 4250 case SPEED_100: 4251 /* maybe add some timeout factor ? */ 4252 break; 4253 } 4254 4255 if (adapter->link_speed != SPEED_1000) 4256 goto no_wait; 4257 4258 /* wait for Remote receiver status OK */ 4259 retry_read_status: 4260 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 4261 &phy_data)) { 4262 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 4263 retry_count) { 4264 msleep(100); 4265 retry_count--; 4266 goto retry_read_status; 4267 } else if (!retry_count) { 4268 netdev_err(netdev, "exceed max 2 second\n"); 4269 } 4270 } else { 4271 netdev_err(netdev, "read 1000Base-T Status Reg\n"); 4272 } 4273 no_wait: 4274 netif_carrier_on(netdev); 4275 4276 /* link state has changed, schedule phy info update */ 4277 if (!test_bit(__IGC_DOWN, &adapter->state)) 4278 mod_timer(&adapter->phy_info_timer, 4279 round_jiffies(jiffies + 2 * HZ)); 4280 } 4281 } else { 4282 if (netif_carrier_ok(netdev)) { 4283 adapter->link_speed = 0; 4284 adapter->link_duplex = 0; 4285 4286 /* Links status message must follow this format */ 4287 netdev_info(netdev, "NIC Link is Down\n"); 4288 netif_carrier_off(netdev); 4289 4290 /* link state has changed, schedule phy info update */ 4291 if (!test_bit(__IGC_DOWN, &adapter->state)) 4292 mod_timer(&adapter->phy_info_timer, 4293 round_jiffies(jiffies + 2 * HZ)); 4294 4295 /* link is down, time to check for alternate media */ 4296 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 4297 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4298 schedule_work(&adapter->reset_task); 4299 /* return immediately */ 4300 return; 4301 } 4302 } 4303 pm_schedule_suspend(netdev->dev.parent, 4304 MSEC_PER_SEC * 5); 4305 4306 /* also check for alternate media here */ 4307 } else if (!netif_carrier_ok(netdev) && 4308 (adapter->flags & IGC_FLAG_MAS_ENABLE)) { 4309 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4310 schedule_work(&adapter->reset_task); 4311 /* return immediately */ 4312 return; 4313 } 4314 } 4315 } 4316 4317 spin_lock(&adapter->stats64_lock); 4318 igc_update_stats(adapter); 4319 spin_unlock(&adapter->stats64_lock); 4320 4321 for (i = 0; i < adapter->num_tx_queues; i++) { 4322 struct igc_ring *tx_ring = adapter->tx_ring[i]; 4323 4324 if (!netif_carrier_ok(netdev)) { 4325 /* We've lost link, so the controller stops DMA, 4326 * but we've got queued Tx work that's never going 4327 * to get done, so reset controller to flush Tx. 4328 * (Do the reset outside of interrupt context). 4329 */ 4330 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 4331 adapter->tx_timeout_count++; 4332 schedule_work(&adapter->reset_task); 4333 /* return immediately since reset is imminent */ 4334 return; 4335 } 4336 } 4337 4338 /* Force detection of hung controller every watchdog period */ 4339 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 4340 } 4341 4342 /* Cause software interrupt to ensure Rx ring is cleaned */ 4343 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4344 u32 eics = 0; 4345 4346 for (i = 0; i < adapter->num_q_vectors; i++) 4347 eics |= adapter->q_vector[i]->eims_value; 4348 wr32(IGC_EICS, eics); 4349 } else { 4350 wr32(IGC_ICS, IGC_ICS_RXDMT0); 4351 } 4352 4353 igc_ptp_tx_hang(adapter); 4354 4355 /* Reset the timer */ 4356 if (!test_bit(__IGC_DOWN, &adapter->state)) { 4357 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 4358 mod_timer(&adapter->watchdog_timer, 4359 round_jiffies(jiffies + HZ)); 4360 else 4361 mod_timer(&adapter->watchdog_timer, 4362 round_jiffies(jiffies + 2 * HZ)); 4363 } 4364 } 4365 4366 /** 4367 * igc_intr_msi - Interrupt Handler 4368 * @irq: interrupt number 4369 * @data: pointer to a network interface device structure 4370 */ 4371 static irqreturn_t igc_intr_msi(int irq, void *data) 4372 { 4373 struct igc_adapter *adapter = data; 4374 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4375 struct igc_hw *hw = &adapter->hw; 4376 /* read ICR disables interrupts using IAM */ 4377 u32 icr = rd32(IGC_ICR); 4378 4379 igc_write_itr(q_vector); 4380 4381 if (icr & IGC_ICR_DRSTA) 4382 schedule_work(&adapter->reset_task); 4383 4384 if (icr & IGC_ICR_DOUTSYNC) { 4385 /* HW is reporting DMA is out of sync */ 4386 adapter->stats.doosync++; 4387 } 4388 4389 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4390 hw->mac.get_link_status = 1; 4391 if (!test_bit(__IGC_DOWN, &adapter->state)) 4392 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4393 } 4394 4395 napi_schedule(&q_vector->napi); 4396 4397 return IRQ_HANDLED; 4398 } 4399 4400 /** 4401 * igc_intr - Legacy Interrupt Handler 4402 * @irq: interrupt number 4403 * @data: pointer to a network interface device structure 4404 */ 4405 static irqreturn_t igc_intr(int irq, void *data) 4406 { 4407 struct igc_adapter *adapter = data; 4408 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4409 struct igc_hw *hw = &adapter->hw; 4410 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 4411 * need for the IMC write 4412 */ 4413 u32 icr = rd32(IGC_ICR); 4414 4415 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 4416 * not set, then the adapter didn't send an interrupt 4417 */ 4418 if (!(icr & IGC_ICR_INT_ASSERTED)) 4419 return IRQ_NONE; 4420 4421 igc_write_itr(q_vector); 4422 4423 if (icr & IGC_ICR_DRSTA) 4424 schedule_work(&adapter->reset_task); 4425 4426 if (icr & IGC_ICR_DOUTSYNC) { 4427 /* HW is reporting DMA is out of sync */ 4428 adapter->stats.doosync++; 4429 } 4430 4431 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4432 hw->mac.get_link_status = 1; 4433 /* guard against interrupt when we're going down */ 4434 if (!test_bit(__IGC_DOWN, &adapter->state)) 4435 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4436 } 4437 4438 napi_schedule(&q_vector->napi); 4439 4440 return IRQ_HANDLED; 4441 } 4442 4443 static void igc_free_irq(struct igc_adapter *adapter) 4444 { 4445 if (adapter->msix_entries) { 4446 int vector = 0, i; 4447 4448 free_irq(adapter->msix_entries[vector++].vector, adapter); 4449 4450 for (i = 0; i < adapter->num_q_vectors; i++) 4451 free_irq(adapter->msix_entries[vector++].vector, 4452 adapter->q_vector[i]); 4453 } else { 4454 free_irq(adapter->pdev->irq, adapter); 4455 } 4456 } 4457 4458 /** 4459 * igc_request_irq - initialize interrupts 4460 * @adapter: Pointer to adapter structure 4461 * 4462 * Attempts to configure interrupts using the best available 4463 * capabilities of the hardware and kernel. 4464 */ 4465 static int igc_request_irq(struct igc_adapter *adapter) 4466 { 4467 struct net_device *netdev = adapter->netdev; 4468 struct pci_dev *pdev = adapter->pdev; 4469 int err = 0; 4470 4471 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4472 err = igc_request_msix(adapter); 4473 if (!err) 4474 goto request_done; 4475 /* fall back to MSI */ 4476 igc_free_all_tx_resources(adapter); 4477 igc_free_all_rx_resources(adapter); 4478 4479 igc_clear_interrupt_scheme(adapter); 4480 err = igc_init_interrupt_scheme(adapter, false); 4481 if (err) 4482 goto request_done; 4483 igc_setup_all_tx_resources(adapter); 4484 igc_setup_all_rx_resources(adapter); 4485 igc_configure(adapter); 4486 } 4487 4488 igc_assign_vector(adapter->q_vector[0], 0); 4489 4490 if (adapter->flags & IGC_FLAG_HAS_MSI) { 4491 err = request_irq(pdev->irq, &igc_intr_msi, 0, 4492 netdev->name, adapter); 4493 if (!err) 4494 goto request_done; 4495 4496 /* fall back to legacy interrupts */ 4497 igc_reset_interrupt_capability(adapter); 4498 adapter->flags &= ~IGC_FLAG_HAS_MSI; 4499 } 4500 4501 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 4502 netdev->name, adapter); 4503 4504 if (err) 4505 netdev_err(netdev, "Error %d getting interrupt\n", err); 4506 4507 request_done: 4508 return err; 4509 } 4510 4511 /** 4512 * __igc_open - Called when a network interface is made active 4513 * @netdev: network interface device structure 4514 * @resuming: boolean indicating if the device is resuming 4515 * 4516 * Returns 0 on success, negative value on failure 4517 * 4518 * The open entry point is called when a network interface is made 4519 * active by the system (IFF_UP). At this point all resources needed 4520 * for transmit and receive operations are allocated, the interrupt 4521 * handler is registered with the OS, the watchdog timer is started, 4522 * and the stack is notified that the interface is ready. 4523 */ 4524 static int __igc_open(struct net_device *netdev, bool resuming) 4525 { 4526 struct igc_adapter *adapter = netdev_priv(netdev); 4527 struct pci_dev *pdev = adapter->pdev; 4528 struct igc_hw *hw = &adapter->hw; 4529 int err = 0; 4530 int i = 0; 4531 4532 /* disallow open during test */ 4533 4534 if (test_bit(__IGC_TESTING, &adapter->state)) { 4535 WARN_ON(resuming); 4536 return -EBUSY; 4537 } 4538 4539 if (!resuming) 4540 pm_runtime_get_sync(&pdev->dev); 4541 4542 netif_carrier_off(netdev); 4543 4544 /* allocate transmit descriptors */ 4545 err = igc_setup_all_tx_resources(adapter); 4546 if (err) 4547 goto err_setup_tx; 4548 4549 /* allocate receive descriptors */ 4550 err = igc_setup_all_rx_resources(adapter); 4551 if (err) 4552 goto err_setup_rx; 4553 4554 igc_power_up_link(adapter); 4555 4556 igc_configure(adapter); 4557 4558 err = igc_request_irq(adapter); 4559 if (err) 4560 goto err_req_irq; 4561 4562 /* Notify the stack of the actual queue counts. */ 4563 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 4564 if (err) 4565 goto err_set_queues; 4566 4567 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 4568 if (err) 4569 goto err_set_queues; 4570 4571 clear_bit(__IGC_DOWN, &adapter->state); 4572 4573 for (i = 0; i < adapter->num_q_vectors; i++) 4574 napi_enable(&adapter->q_vector[i]->napi); 4575 4576 /* Clear any pending interrupts. */ 4577 rd32(IGC_ICR); 4578 igc_irq_enable(adapter); 4579 4580 if (!resuming) 4581 pm_runtime_put(&pdev->dev); 4582 4583 netif_tx_start_all_queues(netdev); 4584 4585 /* start the watchdog. */ 4586 hw->mac.get_link_status = 1; 4587 schedule_work(&adapter->watchdog_task); 4588 4589 return IGC_SUCCESS; 4590 4591 err_set_queues: 4592 igc_free_irq(adapter); 4593 err_req_irq: 4594 igc_release_hw_control(adapter); 4595 igc_power_down_phy_copper_base(&adapter->hw); 4596 igc_free_all_rx_resources(adapter); 4597 err_setup_rx: 4598 igc_free_all_tx_resources(adapter); 4599 err_setup_tx: 4600 igc_reset(adapter); 4601 if (!resuming) 4602 pm_runtime_put(&pdev->dev); 4603 4604 return err; 4605 } 4606 4607 int igc_open(struct net_device *netdev) 4608 { 4609 return __igc_open(netdev, false); 4610 } 4611 4612 /** 4613 * __igc_close - Disables a network interface 4614 * @netdev: network interface device structure 4615 * @suspending: boolean indicating the device is suspending 4616 * 4617 * Returns 0, this is not allowed to fail 4618 * 4619 * The close entry point is called when an interface is de-activated 4620 * by the OS. The hardware is still under the driver's control, but 4621 * needs to be disabled. A global MAC reset is issued to stop the 4622 * hardware, and all transmit and receive resources are freed. 4623 */ 4624 static int __igc_close(struct net_device *netdev, bool suspending) 4625 { 4626 struct igc_adapter *adapter = netdev_priv(netdev); 4627 struct pci_dev *pdev = adapter->pdev; 4628 4629 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 4630 4631 if (!suspending) 4632 pm_runtime_get_sync(&pdev->dev); 4633 4634 igc_down(adapter); 4635 4636 igc_release_hw_control(adapter); 4637 4638 igc_free_irq(adapter); 4639 4640 igc_free_all_tx_resources(adapter); 4641 igc_free_all_rx_resources(adapter); 4642 4643 if (!suspending) 4644 pm_runtime_put_sync(&pdev->dev); 4645 4646 return 0; 4647 } 4648 4649 int igc_close(struct net_device *netdev) 4650 { 4651 if (netif_device_present(netdev) || netdev->dismantle) 4652 return __igc_close(netdev, false); 4653 return 0; 4654 } 4655 4656 /** 4657 * igc_ioctl - Access the hwtstamp interface 4658 * @netdev: network interface device structure 4659 * @ifr: interface request data 4660 * @cmd: ioctl command 4661 **/ 4662 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 4663 { 4664 switch (cmd) { 4665 case SIOCGHWTSTAMP: 4666 return igc_ptp_get_ts_config(netdev, ifr); 4667 case SIOCSHWTSTAMP: 4668 return igc_ptp_set_ts_config(netdev, ifr); 4669 default: 4670 return -EOPNOTSUPP; 4671 } 4672 } 4673 4674 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, 4675 bool enable) 4676 { 4677 struct igc_ring *ring; 4678 int i; 4679 4680 if (queue < 0 || queue >= adapter->num_tx_queues) 4681 return -EINVAL; 4682 4683 ring = adapter->tx_ring[queue]; 4684 ring->launchtime_enable = enable; 4685 4686 if (adapter->base_time) 4687 return 0; 4688 4689 adapter->cycle_time = NSEC_PER_SEC; 4690 4691 for (i = 0; i < adapter->num_tx_queues; i++) { 4692 ring = adapter->tx_ring[i]; 4693 ring->start_time = 0; 4694 ring->end_time = NSEC_PER_SEC; 4695 } 4696 4697 return 0; 4698 } 4699 4700 static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now) 4701 { 4702 struct timespec64 b; 4703 4704 b = ktime_to_timespec64(base_time); 4705 4706 return timespec64_compare(now, &b) > 0; 4707 } 4708 4709 static bool validate_schedule(struct igc_adapter *adapter, 4710 const struct tc_taprio_qopt_offload *qopt) 4711 { 4712 int queue_uses[IGC_MAX_TX_QUEUES] = { }; 4713 struct timespec64 now; 4714 size_t n; 4715 4716 if (qopt->cycle_time_extension) 4717 return false; 4718 4719 igc_ptp_read(adapter, &now); 4720 4721 /* If we program the controller's BASET registers with a time 4722 * in the future, it will hold all the packets until that 4723 * time, causing a lot of TX Hangs, so to avoid that, we 4724 * reject schedules that would start in the future. 4725 */ 4726 if (!is_base_time_past(qopt->base_time, &now)) 4727 return false; 4728 4729 for (n = 0; n < qopt->num_entries; n++) { 4730 const struct tc_taprio_sched_entry *e; 4731 int i; 4732 4733 e = &qopt->entries[n]; 4734 4735 /* i225 only supports "global" frame preemption 4736 * settings. 4737 */ 4738 if (e->command != TC_TAPRIO_CMD_SET_GATES) 4739 return false; 4740 4741 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4742 if (e->gate_mask & BIT(i)) 4743 queue_uses[i]++; 4744 4745 if (queue_uses[i] > 1) 4746 return false; 4747 } 4748 } 4749 4750 return true; 4751 } 4752 4753 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, 4754 struct tc_etf_qopt_offload *qopt) 4755 { 4756 struct igc_hw *hw = &adapter->hw; 4757 int err; 4758 4759 if (hw->mac.type != igc_i225) 4760 return -EOPNOTSUPP; 4761 4762 err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable); 4763 if (err) 4764 return err; 4765 4766 return igc_tsn_offload_apply(adapter); 4767 } 4768 4769 static int igc_save_qbv_schedule(struct igc_adapter *adapter, 4770 struct tc_taprio_qopt_offload *qopt) 4771 { 4772 u32 start_time = 0, end_time = 0; 4773 size_t n; 4774 4775 if (!qopt->enable) { 4776 adapter->base_time = 0; 4777 return 0; 4778 } 4779 4780 if (adapter->base_time) 4781 return -EALREADY; 4782 4783 if (!validate_schedule(adapter, qopt)) 4784 return -EINVAL; 4785 4786 adapter->cycle_time = qopt->cycle_time; 4787 adapter->base_time = qopt->base_time; 4788 4789 /* FIXME: be a little smarter about cases when the gate for a 4790 * queue stays open for more than one entry. 4791 */ 4792 for (n = 0; n < qopt->num_entries; n++) { 4793 struct tc_taprio_sched_entry *e = &qopt->entries[n]; 4794 int i; 4795 4796 end_time += e->interval; 4797 4798 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4799 struct igc_ring *ring = adapter->tx_ring[i]; 4800 4801 if (!(e->gate_mask & BIT(i))) 4802 continue; 4803 4804 ring->start_time = start_time; 4805 ring->end_time = end_time; 4806 } 4807 4808 start_time += e->interval; 4809 } 4810 4811 return 0; 4812 } 4813 4814 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, 4815 struct tc_taprio_qopt_offload *qopt) 4816 { 4817 struct igc_hw *hw = &adapter->hw; 4818 int err; 4819 4820 if (hw->mac.type != igc_i225) 4821 return -EOPNOTSUPP; 4822 4823 err = igc_save_qbv_schedule(adapter, qopt); 4824 if (err) 4825 return err; 4826 4827 return igc_tsn_offload_apply(adapter); 4828 } 4829 4830 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, 4831 void *type_data) 4832 { 4833 struct igc_adapter *adapter = netdev_priv(dev); 4834 4835 switch (type) { 4836 case TC_SETUP_QDISC_TAPRIO: 4837 return igc_tsn_enable_qbv_scheduling(adapter, type_data); 4838 4839 case TC_SETUP_QDISC_ETF: 4840 return igc_tsn_enable_launchtime(adapter, type_data); 4841 4842 default: 4843 return -EOPNOTSUPP; 4844 } 4845 } 4846 4847 static const struct net_device_ops igc_netdev_ops = { 4848 .ndo_open = igc_open, 4849 .ndo_stop = igc_close, 4850 .ndo_start_xmit = igc_xmit_frame, 4851 .ndo_set_rx_mode = igc_set_rx_mode, 4852 .ndo_set_mac_address = igc_set_mac, 4853 .ndo_change_mtu = igc_change_mtu, 4854 .ndo_get_stats64 = igc_get_stats64, 4855 .ndo_fix_features = igc_fix_features, 4856 .ndo_set_features = igc_set_features, 4857 .ndo_features_check = igc_features_check, 4858 .ndo_do_ioctl = igc_ioctl, 4859 .ndo_setup_tc = igc_setup_tc, 4860 }; 4861 4862 /* PCIe configuration access */ 4863 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4864 { 4865 struct igc_adapter *adapter = hw->back; 4866 4867 pci_read_config_word(adapter->pdev, reg, value); 4868 } 4869 4870 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4871 { 4872 struct igc_adapter *adapter = hw->back; 4873 4874 pci_write_config_word(adapter->pdev, reg, *value); 4875 } 4876 4877 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4878 { 4879 struct igc_adapter *adapter = hw->back; 4880 4881 if (!pci_is_pcie(adapter->pdev)) 4882 return -IGC_ERR_CONFIG; 4883 4884 pcie_capability_read_word(adapter->pdev, reg, value); 4885 4886 return IGC_SUCCESS; 4887 } 4888 4889 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4890 { 4891 struct igc_adapter *adapter = hw->back; 4892 4893 if (!pci_is_pcie(adapter->pdev)) 4894 return -IGC_ERR_CONFIG; 4895 4896 pcie_capability_write_word(adapter->pdev, reg, *value); 4897 4898 return IGC_SUCCESS; 4899 } 4900 4901 u32 igc_rd32(struct igc_hw *hw, u32 reg) 4902 { 4903 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 4904 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 4905 u32 value = 0; 4906 4907 value = readl(&hw_addr[reg]); 4908 4909 /* reads should not return all F's */ 4910 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 4911 struct net_device *netdev = igc->netdev; 4912 4913 hw->hw_addr = NULL; 4914 netif_device_detach(netdev); 4915 netdev_err(netdev, "PCIe link lost, device now detached\n"); 4916 WARN(pci_device_is_present(igc->pdev), 4917 "igc: Failed to read reg 0x%x!\n", reg); 4918 } 4919 4920 return value; 4921 } 4922 4923 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) 4924 { 4925 struct igc_mac_info *mac = &adapter->hw.mac; 4926 4927 mac->autoneg = 0; 4928 4929 /* Make sure dplx is at most 1 bit and lsb of speed is not set 4930 * for the switch() below to work 4931 */ 4932 if ((spd & 1) || (dplx & ~1)) 4933 goto err_inval; 4934 4935 switch (spd + dplx) { 4936 case SPEED_10 + DUPLEX_HALF: 4937 mac->forced_speed_duplex = ADVERTISE_10_HALF; 4938 break; 4939 case SPEED_10 + DUPLEX_FULL: 4940 mac->forced_speed_duplex = ADVERTISE_10_FULL; 4941 break; 4942 case SPEED_100 + DUPLEX_HALF: 4943 mac->forced_speed_duplex = ADVERTISE_100_HALF; 4944 break; 4945 case SPEED_100 + DUPLEX_FULL: 4946 mac->forced_speed_duplex = ADVERTISE_100_FULL; 4947 break; 4948 case SPEED_1000 + DUPLEX_FULL: 4949 mac->autoneg = 1; 4950 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 4951 break; 4952 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 4953 goto err_inval; 4954 case SPEED_2500 + DUPLEX_FULL: 4955 mac->autoneg = 1; 4956 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 4957 break; 4958 case SPEED_2500 + DUPLEX_HALF: /* not supported */ 4959 default: 4960 goto err_inval; 4961 } 4962 4963 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 4964 adapter->hw.phy.mdix = AUTO_ALL_MODES; 4965 4966 return 0; 4967 4968 err_inval: 4969 netdev_err(adapter->netdev, "Unsupported Speed/Duplex configuration\n"); 4970 return -EINVAL; 4971 } 4972 4973 /** 4974 * igc_probe - Device Initialization Routine 4975 * @pdev: PCI device information struct 4976 * @ent: entry in igc_pci_tbl 4977 * 4978 * Returns 0 on success, negative on failure 4979 * 4980 * igc_probe initializes an adapter identified by a pci_dev structure. 4981 * The OS initialization, configuring the adapter private structure, 4982 * and a hardware reset occur. 4983 */ 4984 static int igc_probe(struct pci_dev *pdev, 4985 const struct pci_device_id *ent) 4986 { 4987 struct igc_adapter *adapter; 4988 struct net_device *netdev; 4989 struct igc_hw *hw; 4990 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 4991 int err, pci_using_dac; 4992 4993 err = pci_enable_device_mem(pdev); 4994 if (err) 4995 return err; 4996 4997 pci_using_dac = 0; 4998 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4999 if (!err) { 5000 pci_using_dac = 1; 5001 } else { 5002 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 5003 if (err) { 5004 dev_err(&pdev->dev, 5005 "No usable DMA configuration, aborting\n"); 5006 goto err_dma; 5007 } 5008 } 5009 5010 err = pci_request_mem_regions(pdev, igc_driver_name); 5011 if (err) 5012 goto err_pci_reg; 5013 5014 pci_enable_pcie_error_reporting(pdev); 5015 5016 pci_set_master(pdev); 5017 5018 err = -ENOMEM; 5019 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 5020 IGC_MAX_TX_QUEUES); 5021 5022 if (!netdev) 5023 goto err_alloc_etherdev; 5024 5025 SET_NETDEV_DEV(netdev, &pdev->dev); 5026 5027 pci_set_drvdata(pdev, netdev); 5028 adapter = netdev_priv(netdev); 5029 adapter->netdev = netdev; 5030 adapter->pdev = pdev; 5031 hw = &adapter->hw; 5032 hw->back = adapter; 5033 adapter->port_num = hw->bus.func; 5034 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 5035 5036 err = pci_save_state(pdev); 5037 if (err) 5038 goto err_ioremap; 5039 5040 err = -EIO; 5041 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 5042 pci_resource_len(pdev, 0)); 5043 if (!adapter->io_addr) 5044 goto err_ioremap; 5045 5046 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 5047 hw->hw_addr = adapter->io_addr; 5048 5049 netdev->netdev_ops = &igc_netdev_ops; 5050 igc_ethtool_set_ops(netdev); 5051 netdev->watchdog_timeo = 5 * HZ; 5052 5053 netdev->mem_start = pci_resource_start(pdev, 0); 5054 netdev->mem_end = pci_resource_end(pdev, 0); 5055 5056 /* PCI config space info */ 5057 hw->vendor_id = pdev->vendor; 5058 hw->device_id = pdev->device; 5059 hw->revision_id = pdev->revision; 5060 hw->subsystem_vendor_id = pdev->subsystem_vendor; 5061 hw->subsystem_device_id = pdev->subsystem_device; 5062 5063 /* Copy the default MAC and PHY function pointers */ 5064 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 5065 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 5066 5067 /* Initialize skew-specific constants */ 5068 err = ei->get_invariants(hw); 5069 if (err) 5070 goto err_sw_init; 5071 5072 /* Add supported features to the features list*/ 5073 netdev->features |= NETIF_F_SG; 5074 netdev->features |= NETIF_F_TSO; 5075 netdev->features |= NETIF_F_TSO6; 5076 netdev->features |= NETIF_F_TSO_ECN; 5077 netdev->features |= NETIF_F_RXCSUM; 5078 netdev->features |= NETIF_F_HW_CSUM; 5079 netdev->features |= NETIF_F_SCTP_CRC; 5080 netdev->features |= NETIF_F_HW_TC; 5081 5082 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 5083 NETIF_F_GSO_GRE_CSUM | \ 5084 NETIF_F_GSO_IPXIP4 | \ 5085 NETIF_F_GSO_IPXIP6 | \ 5086 NETIF_F_GSO_UDP_TUNNEL | \ 5087 NETIF_F_GSO_UDP_TUNNEL_CSUM) 5088 5089 netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; 5090 netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; 5091 5092 /* setup the private structure */ 5093 err = igc_sw_init(adapter); 5094 if (err) 5095 goto err_sw_init; 5096 5097 /* copy netdev features into list of user selectable features */ 5098 netdev->hw_features |= NETIF_F_NTUPLE; 5099 netdev->hw_features |= netdev->features; 5100 5101 if (pci_using_dac) 5102 netdev->features |= NETIF_F_HIGHDMA; 5103 5104 /* MTU range: 68 - 9216 */ 5105 netdev->min_mtu = ETH_MIN_MTU; 5106 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 5107 5108 /* before reading the NVM, reset the controller to put the device in a 5109 * known good starting state 5110 */ 5111 hw->mac.ops.reset_hw(hw); 5112 5113 if (igc_get_flash_presence_i225(hw)) { 5114 if (hw->nvm.ops.validate(hw) < 0) { 5115 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 5116 err = -EIO; 5117 goto err_eeprom; 5118 } 5119 } 5120 5121 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 5122 /* copy the MAC address out of the NVM */ 5123 if (hw->mac.ops.read_mac_addr(hw)) 5124 dev_err(&pdev->dev, "NVM Read Error\n"); 5125 } 5126 5127 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 5128 5129 if (!is_valid_ether_addr(netdev->dev_addr)) { 5130 dev_err(&pdev->dev, "Invalid MAC Address\n"); 5131 err = -EIO; 5132 goto err_eeprom; 5133 } 5134 5135 /* configure RXPBSIZE and TXPBSIZE */ 5136 wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT); 5137 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); 5138 5139 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 5140 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 5141 5142 INIT_WORK(&adapter->reset_task, igc_reset_task); 5143 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 5144 5145 /* Initialize link properties that are user-changeable */ 5146 adapter->fc_autoneg = true; 5147 hw->mac.autoneg = true; 5148 hw->phy.autoneg_advertised = 0xaf; 5149 5150 hw->fc.requested_mode = igc_fc_default; 5151 hw->fc.current_mode = igc_fc_default; 5152 5153 /* By default, support wake on port A */ 5154 adapter->flags |= IGC_FLAG_WOL_SUPPORTED; 5155 5156 /* initialize the wol settings based on the eeprom settings */ 5157 if (adapter->flags & IGC_FLAG_WOL_SUPPORTED) 5158 adapter->wol |= IGC_WUFC_MAG; 5159 5160 device_set_wakeup_enable(&adapter->pdev->dev, 5161 adapter->flags & IGC_FLAG_WOL_SUPPORTED); 5162 5163 igc_ptp_init(adapter); 5164 5165 /* reset the hardware with the new settings */ 5166 igc_reset(adapter); 5167 5168 /* let the f/w know that the h/w is now under the control of the 5169 * driver. 5170 */ 5171 igc_get_hw_control(adapter); 5172 5173 strncpy(netdev->name, "eth%d", IFNAMSIZ); 5174 err = register_netdev(netdev); 5175 if (err) 5176 goto err_register; 5177 5178 /* carrier off reporting is important to ethtool even BEFORE open */ 5179 netif_carrier_off(netdev); 5180 5181 /* Check if Media Autosense is enabled */ 5182 adapter->ei = *ei; 5183 5184 /* print pcie link status and MAC address */ 5185 pcie_print_link_status(pdev); 5186 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 5187 5188 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); 5189 /* Disable EEE for internal PHY devices */ 5190 hw->dev_spec._base.eee_enable = false; 5191 adapter->flags &= ~IGC_FLAG_EEE; 5192 igc_set_eee_i225(hw, false, false, false); 5193 5194 pm_runtime_put_noidle(&pdev->dev); 5195 5196 return 0; 5197 5198 err_register: 5199 igc_release_hw_control(adapter); 5200 err_eeprom: 5201 if (!igc_check_reset_block(hw)) 5202 igc_reset_phy(hw); 5203 err_sw_init: 5204 igc_clear_interrupt_scheme(adapter); 5205 iounmap(adapter->io_addr); 5206 err_ioremap: 5207 free_netdev(netdev); 5208 err_alloc_etherdev: 5209 pci_release_mem_regions(pdev); 5210 err_pci_reg: 5211 err_dma: 5212 pci_disable_device(pdev); 5213 return err; 5214 } 5215 5216 /** 5217 * igc_remove - Device Removal Routine 5218 * @pdev: PCI device information struct 5219 * 5220 * igc_remove is called by the PCI subsystem to alert the driver 5221 * that it should release a PCI device. This could be caused by a 5222 * Hot-Plug event, or because the driver is going to be removed from 5223 * memory. 5224 */ 5225 static void igc_remove(struct pci_dev *pdev) 5226 { 5227 struct net_device *netdev = pci_get_drvdata(pdev); 5228 struct igc_adapter *adapter = netdev_priv(netdev); 5229 5230 pm_runtime_get_noresume(&pdev->dev); 5231 5232 igc_flush_nfc_rules(adapter); 5233 5234 igc_ptp_stop(adapter); 5235 5236 set_bit(__IGC_DOWN, &adapter->state); 5237 5238 del_timer_sync(&adapter->watchdog_timer); 5239 del_timer_sync(&adapter->phy_info_timer); 5240 5241 cancel_work_sync(&adapter->reset_task); 5242 cancel_work_sync(&adapter->watchdog_task); 5243 5244 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5245 * would have already happened in close and is redundant. 5246 */ 5247 igc_release_hw_control(adapter); 5248 unregister_netdev(netdev); 5249 5250 igc_clear_interrupt_scheme(adapter); 5251 pci_iounmap(pdev, adapter->io_addr); 5252 pci_release_mem_regions(pdev); 5253 5254 free_netdev(netdev); 5255 5256 pci_disable_pcie_error_reporting(pdev); 5257 5258 pci_disable_device(pdev); 5259 } 5260 5261 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, 5262 bool runtime) 5263 { 5264 struct net_device *netdev = pci_get_drvdata(pdev); 5265 struct igc_adapter *adapter = netdev_priv(netdev); 5266 u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol; 5267 struct igc_hw *hw = &adapter->hw; 5268 u32 ctrl, rctl, status; 5269 bool wake; 5270 5271 rtnl_lock(); 5272 netif_device_detach(netdev); 5273 5274 if (netif_running(netdev)) 5275 __igc_close(netdev, true); 5276 5277 igc_ptp_suspend(adapter); 5278 5279 igc_clear_interrupt_scheme(adapter); 5280 rtnl_unlock(); 5281 5282 status = rd32(IGC_STATUS); 5283 if (status & IGC_STATUS_LU) 5284 wufc &= ~IGC_WUFC_LNKC; 5285 5286 if (wufc) { 5287 igc_setup_rctl(adapter); 5288 igc_set_rx_mode(netdev); 5289 5290 /* turn on all-multi mode if wake on multicast is enabled */ 5291 if (wufc & IGC_WUFC_MC) { 5292 rctl = rd32(IGC_RCTL); 5293 rctl |= IGC_RCTL_MPE; 5294 wr32(IGC_RCTL, rctl); 5295 } 5296 5297 ctrl = rd32(IGC_CTRL); 5298 ctrl |= IGC_CTRL_ADVD3WUC; 5299 wr32(IGC_CTRL, ctrl); 5300 5301 /* Allow time for pending master requests to run */ 5302 igc_disable_pcie_master(hw); 5303 5304 wr32(IGC_WUC, IGC_WUC_PME_EN); 5305 wr32(IGC_WUFC, wufc); 5306 } else { 5307 wr32(IGC_WUC, 0); 5308 wr32(IGC_WUFC, 0); 5309 } 5310 5311 wake = wufc || adapter->en_mng_pt; 5312 if (!wake) 5313 igc_power_down_phy_copper_base(&adapter->hw); 5314 else 5315 igc_power_up_link(adapter); 5316 5317 if (enable_wake) 5318 *enable_wake = wake; 5319 5320 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5321 * would have already happened in close and is redundant. 5322 */ 5323 igc_release_hw_control(adapter); 5324 5325 pci_disable_device(pdev); 5326 5327 return 0; 5328 } 5329 5330 #ifdef CONFIG_PM 5331 static int __maybe_unused igc_runtime_suspend(struct device *dev) 5332 { 5333 return __igc_shutdown(to_pci_dev(dev), NULL, 1); 5334 } 5335 5336 static void igc_deliver_wake_packet(struct net_device *netdev) 5337 { 5338 struct igc_adapter *adapter = netdev_priv(netdev); 5339 struct igc_hw *hw = &adapter->hw; 5340 struct sk_buff *skb; 5341 u32 wupl; 5342 5343 wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK; 5344 5345 /* WUPM stores only the first 128 bytes of the wake packet. 5346 * Read the packet only if we have the whole thing. 5347 */ 5348 if (wupl == 0 || wupl > IGC_WUPM_BYTES) 5349 return; 5350 5351 skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES); 5352 if (!skb) 5353 return; 5354 5355 skb_put(skb, wupl); 5356 5357 /* Ensure reads are 32-bit aligned */ 5358 wupl = roundup(wupl, 4); 5359 5360 memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl); 5361 5362 skb->protocol = eth_type_trans(skb, netdev); 5363 netif_rx(skb); 5364 } 5365 5366 static int __maybe_unused igc_resume(struct device *dev) 5367 { 5368 struct pci_dev *pdev = to_pci_dev(dev); 5369 struct net_device *netdev = pci_get_drvdata(pdev); 5370 struct igc_adapter *adapter = netdev_priv(netdev); 5371 struct igc_hw *hw = &adapter->hw; 5372 u32 err, val; 5373 5374 pci_set_power_state(pdev, PCI_D0); 5375 pci_restore_state(pdev); 5376 pci_save_state(pdev); 5377 5378 if (!pci_device_is_present(pdev)) 5379 return -ENODEV; 5380 err = pci_enable_device_mem(pdev); 5381 if (err) { 5382 netdev_err(netdev, "Cannot enable PCI device from suspend\n"); 5383 return err; 5384 } 5385 pci_set_master(pdev); 5386 5387 pci_enable_wake(pdev, PCI_D3hot, 0); 5388 pci_enable_wake(pdev, PCI_D3cold, 0); 5389 5390 if (igc_init_interrupt_scheme(adapter, true)) { 5391 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5392 return -ENOMEM; 5393 } 5394 5395 igc_reset(adapter); 5396 5397 /* let the f/w know that the h/w is now under the control of the 5398 * driver. 5399 */ 5400 igc_get_hw_control(adapter); 5401 5402 val = rd32(IGC_WUS); 5403 if (val & WAKE_PKT_WUS) 5404 igc_deliver_wake_packet(netdev); 5405 5406 wr32(IGC_WUS, ~0); 5407 5408 rtnl_lock(); 5409 if (!err && netif_running(netdev)) 5410 err = __igc_open(netdev, true); 5411 5412 if (!err) 5413 netif_device_attach(netdev); 5414 rtnl_unlock(); 5415 5416 return err; 5417 } 5418 5419 static int __maybe_unused igc_runtime_resume(struct device *dev) 5420 { 5421 return igc_resume(dev); 5422 } 5423 5424 static int __maybe_unused igc_suspend(struct device *dev) 5425 { 5426 return __igc_shutdown(to_pci_dev(dev), NULL, 0); 5427 } 5428 5429 static int __maybe_unused igc_runtime_idle(struct device *dev) 5430 { 5431 struct net_device *netdev = dev_get_drvdata(dev); 5432 struct igc_adapter *adapter = netdev_priv(netdev); 5433 5434 if (!igc_has_link(adapter)) 5435 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 5436 5437 return -EBUSY; 5438 } 5439 #endif /* CONFIG_PM */ 5440 5441 static void igc_shutdown(struct pci_dev *pdev) 5442 { 5443 bool wake; 5444 5445 __igc_shutdown(pdev, &wake, 0); 5446 5447 if (system_state == SYSTEM_POWER_OFF) { 5448 pci_wake_from_d3(pdev, wake); 5449 pci_set_power_state(pdev, PCI_D3hot); 5450 } 5451 } 5452 5453 /** 5454 * igc_io_error_detected - called when PCI error is detected 5455 * @pdev: Pointer to PCI device 5456 * @state: The current PCI connection state 5457 * 5458 * This function is called after a PCI bus error affecting 5459 * this device has been detected. 5460 **/ 5461 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, 5462 pci_channel_state_t state) 5463 { 5464 struct net_device *netdev = pci_get_drvdata(pdev); 5465 struct igc_adapter *adapter = netdev_priv(netdev); 5466 5467 netif_device_detach(netdev); 5468 5469 if (state == pci_channel_io_perm_failure) 5470 return PCI_ERS_RESULT_DISCONNECT; 5471 5472 if (netif_running(netdev)) 5473 igc_down(adapter); 5474 pci_disable_device(pdev); 5475 5476 /* Request a slot reset. */ 5477 return PCI_ERS_RESULT_NEED_RESET; 5478 } 5479 5480 /** 5481 * igc_io_slot_reset - called after the PCI bus has been reset. 5482 * @pdev: Pointer to PCI device 5483 * 5484 * Restart the card from scratch, as if from a cold-boot. Implementation 5485 * resembles the first-half of the igc_resume routine. 5486 **/ 5487 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) 5488 { 5489 struct net_device *netdev = pci_get_drvdata(pdev); 5490 struct igc_adapter *adapter = netdev_priv(netdev); 5491 struct igc_hw *hw = &adapter->hw; 5492 pci_ers_result_t result; 5493 5494 if (pci_enable_device_mem(pdev)) { 5495 netdev_err(netdev, "Could not re-enable PCI device after reset\n"); 5496 result = PCI_ERS_RESULT_DISCONNECT; 5497 } else { 5498 pci_set_master(pdev); 5499 pci_restore_state(pdev); 5500 pci_save_state(pdev); 5501 5502 pci_enable_wake(pdev, PCI_D3hot, 0); 5503 pci_enable_wake(pdev, PCI_D3cold, 0); 5504 5505 /* In case of PCI error, adapter loses its HW address 5506 * so we should re-assign it here. 5507 */ 5508 hw->hw_addr = adapter->io_addr; 5509 5510 igc_reset(adapter); 5511 wr32(IGC_WUS, ~0); 5512 result = PCI_ERS_RESULT_RECOVERED; 5513 } 5514 5515 return result; 5516 } 5517 5518 /** 5519 * igc_io_resume - called when traffic can start to flow again. 5520 * @pdev: Pointer to PCI device 5521 * 5522 * This callback is called when the error recovery driver tells us that 5523 * its OK to resume normal operation. Implementation resembles the 5524 * second-half of the igc_resume routine. 5525 */ 5526 static void igc_io_resume(struct pci_dev *pdev) 5527 { 5528 struct net_device *netdev = pci_get_drvdata(pdev); 5529 struct igc_adapter *adapter = netdev_priv(netdev); 5530 5531 rtnl_lock(); 5532 if (netif_running(netdev)) { 5533 if (igc_open(netdev)) { 5534 netdev_err(netdev, "igc_open failed after reset\n"); 5535 return; 5536 } 5537 } 5538 5539 netif_device_attach(netdev); 5540 5541 /* let the f/w know that the h/w is now under the control of the 5542 * driver. 5543 */ 5544 igc_get_hw_control(adapter); 5545 rtnl_unlock(); 5546 } 5547 5548 static const struct pci_error_handlers igc_err_handler = { 5549 .error_detected = igc_io_error_detected, 5550 .slot_reset = igc_io_slot_reset, 5551 .resume = igc_io_resume, 5552 }; 5553 5554 #ifdef CONFIG_PM 5555 static const struct dev_pm_ops igc_pm_ops = { 5556 SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume) 5557 SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume, 5558 igc_runtime_idle) 5559 }; 5560 #endif 5561 5562 static struct pci_driver igc_driver = { 5563 .name = igc_driver_name, 5564 .id_table = igc_pci_tbl, 5565 .probe = igc_probe, 5566 .remove = igc_remove, 5567 #ifdef CONFIG_PM 5568 .driver.pm = &igc_pm_ops, 5569 #endif 5570 .shutdown = igc_shutdown, 5571 .err_handler = &igc_err_handler, 5572 }; 5573 5574 /** 5575 * igc_reinit_queues - return error 5576 * @adapter: pointer to adapter structure 5577 */ 5578 int igc_reinit_queues(struct igc_adapter *adapter) 5579 { 5580 struct net_device *netdev = adapter->netdev; 5581 int err = 0; 5582 5583 if (netif_running(netdev)) 5584 igc_close(netdev); 5585 5586 igc_reset_interrupt_capability(adapter); 5587 5588 if (igc_init_interrupt_scheme(adapter, true)) { 5589 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5590 return -ENOMEM; 5591 } 5592 5593 if (netif_running(netdev)) 5594 err = igc_open(netdev); 5595 5596 return err; 5597 } 5598 5599 /** 5600 * igc_get_hw_dev - return device 5601 * @hw: pointer to hardware structure 5602 * 5603 * used by hardware layer to print debugging information 5604 */ 5605 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 5606 { 5607 struct igc_adapter *adapter = hw->back; 5608 5609 return adapter->netdev; 5610 } 5611 5612 /** 5613 * igc_init_module - Driver Registration Routine 5614 * 5615 * igc_init_module is the first routine called when the driver is 5616 * loaded. All it does is register with the PCI subsystem. 5617 */ 5618 static int __init igc_init_module(void) 5619 { 5620 int ret; 5621 5622 pr_info("%s\n", igc_driver_string); 5623 pr_info("%s\n", igc_copyright); 5624 5625 ret = pci_register_driver(&igc_driver); 5626 return ret; 5627 } 5628 5629 module_init(igc_init_module); 5630 5631 /** 5632 * igc_exit_module - Driver Exit Cleanup Routine 5633 * 5634 * igc_exit_module is called just before the driver is removed 5635 * from memory. 5636 */ 5637 static void __exit igc_exit_module(void) 5638 { 5639 pci_unregister_driver(&igc_driver); 5640 } 5641 5642 module_exit(igc_exit_module); 5643 /* igc_main.c */ 5644