1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/aer.h> 8 #include <linux/tcp.h> 9 #include <linux/udp.h> 10 #include <linux/ip.h> 11 #include <linux/pm_runtime.h> 12 #include <net/pkt_sched.h> 13 14 #include <net/ipv6.h> 15 16 #include "igc.h" 17 #include "igc_hw.h" 18 #include "igc_tsn.h" 19 20 #define DRV_VERSION "0.0.1-k" 21 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 22 23 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 24 25 static int debug = -1; 26 27 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 28 MODULE_DESCRIPTION(DRV_SUMMARY); 29 MODULE_LICENSE("GPL v2"); 30 MODULE_VERSION(DRV_VERSION); 31 module_param(debug, int, 0); 32 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 33 34 char igc_driver_name[] = "igc"; 35 char igc_driver_version[] = DRV_VERSION; 36 static const char igc_driver_string[] = DRV_SUMMARY; 37 static const char igc_copyright[] = 38 "Copyright(c) 2018 Intel Corporation."; 39 40 static const struct igc_info *igc_info_tbl[] = { 41 [board_base] = &igc_base_info, 42 }; 43 44 static const struct pci_device_id igc_pci_tbl[] = { 45 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base }, 46 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base }, 47 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, 48 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, 49 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, 50 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base }, 51 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base }, 52 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base }, 53 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, 54 /* required last entry */ 55 {0, } 56 }; 57 58 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 59 60 enum latency_range { 61 lowest_latency = 0, 62 low_latency = 1, 63 bulk_latency = 2, 64 latency_invalid = 255 65 }; 66 67 /** 68 * igc_power_down_link - Power down the phy/serdes link 69 * @adapter: address of board private structure 70 */ 71 static void igc_power_down_link(struct igc_adapter *adapter) 72 { 73 if (adapter->hw.phy.media_type == igc_media_type_copper) 74 igc_power_down_phy_copper_base(&adapter->hw); 75 } 76 77 void igc_reset(struct igc_adapter *adapter) 78 { 79 struct net_device *dev = adapter->netdev; 80 struct igc_hw *hw = &adapter->hw; 81 struct igc_fc_info *fc = &hw->fc; 82 u32 pba, hwm; 83 84 /* Repartition PBA for greater than 9k MTU if required */ 85 pba = IGC_PBA_34K; 86 87 /* flow control settings 88 * The high water mark must be low enough to fit one full frame 89 * after transmitting the pause frame. As such we must have enough 90 * space to allow for us to complete our current transmit and then 91 * receive the frame that is in progress from the link partner. 92 * Set it to: 93 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 94 */ 95 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 96 97 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 98 fc->low_water = fc->high_water - 16; 99 fc->pause_time = 0xFFFF; 100 fc->send_xon = 1; 101 fc->current_mode = fc->requested_mode; 102 103 hw->mac.ops.reset_hw(hw); 104 105 if (hw->mac.ops.init_hw(hw)) 106 netdev_err(dev, "Error on hardware initialization\n"); 107 108 if (!netif_running(adapter->netdev)) 109 igc_power_down_link(adapter); 110 111 /* Re-enable PTP, where applicable. */ 112 igc_ptp_reset(adapter); 113 114 /* Re-enable TSN offloading, where applicable. */ 115 igc_tsn_offload_apply(adapter); 116 117 igc_get_phy_info(hw); 118 } 119 120 /** 121 * igc_power_up_link - Power up the phy link 122 * @adapter: address of board private structure 123 */ 124 static void igc_power_up_link(struct igc_adapter *adapter) 125 { 126 igc_reset_phy(&adapter->hw); 127 128 if (adapter->hw.phy.media_type == igc_media_type_copper) 129 igc_power_up_phy_copper(&adapter->hw); 130 131 igc_setup_link(&adapter->hw); 132 } 133 134 /** 135 * igc_release_hw_control - release control of the h/w to f/w 136 * @adapter: address of board private structure 137 * 138 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 139 * For ASF and Pass Through versions of f/w this means that the 140 * driver is no longer loaded. 141 */ 142 static void igc_release_hw_control(struct igc_adapter *adapter) 143 { 144 struct igc_hw *hw = &adapter->hw; 145 u32 ctrl_ext; 146 147 /* Let firmware take over control of h/w */ 148 ctrl_ext = rd32(IGC_CTRL_EXT); 149 wr32(IGC_CTRL_EXT, 150 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 151 } 152 153 /** 154 * igc_get_hw_control - get control of the h/w from f/w 155 * @adapter: address of board private structure 156 * 157 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 158 * For ASF and Pass Through versions of f/w this means that 159 * the driver is loaded. 160 */ 161 static void igc_get_hw_control(struct igc_adapter *adapter) 162 { 163 struct igc_hw *hw = &adapter->hw; 164 u32 ctrl_ext; 165 166 /* Let firmware know the driver has taken over */ 167 ctrl_ext = rd32(IGC_CTRL_EXT); 168 wr32(IGC_CTRL_EXT, 169 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 170 } 171 172 /** 173 * igc_clean_tx_ring - Free Tx Buffers 174 * @tx_ring: ring to be cleaned 175 */ 176 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 177 { 178 u16 i = tx_ring->next_to_clean; 179 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 180 181 while (i != tx_ring->next_to_use) { 182 union igc_adv_tx_desc *eop_desc, *tx_desc; 183 184 /* Free all the Tx ring sk_buffs */ 185 dev_kfree_skb_any(tx_buffer->skb); 186 187 /* unmap skb header data */ 188 dma_unmap_single(tx_ring->dev, 189 dma_unmap_addr(tx_buffer, dma), 190 dma_unmap_len(tx_buffer, len), 191 DMA_TO_DEVICE); 192 193 /* check for eop_desc to determine the end of the packet */ 194 eop_desc = tx_buffer->next_to_watch; 195 tx_desc = IGC_TX_DESC(tx_ring, i); 196 197 /* unmap remaining buffers */ 198 while (tx_desc != eop_desc) { 199 tx_buffer++; 200 tx_desc++; 201 i++; 202 if (unlikely(i == tx_ring->count)) { 203 i = 0; 204 tx_buffer = tx_ring->tx_buffer_info; 205 tx_desc = IGC_TX_DESC(tx_ring, 0); 206 } 207 208 /* unmap any remaining paged data */ 209 if (dma_unmap_len(tx_buffer, len)) 210 dma_unmap_page(tx_ring->dev, 211 dma_unmap_addr(tx_buffer, dma), 212 dma_unmap_len(tx_buffer, len), 213 DMA_TO_DEVICE); 214 } 215 216 /* move us one more past the eop_desc for start of next pkt */ 217 tx_buffer++; 218 i++; 219 if (unlikely(i == tx_ring->count)) { 220 i = 0; 221 tx_buffer = tx_ring->tx_buffer_info; 222 } 223 } 224 225 /* reset BQL for queue */ 226 netdev_tx_reset_queue(txring_txq(tx_ring)); 227 228 /* reset next_to_use and next_to_clean */ 229 tx_ring->next_to_use = 0; 230 tx_ring->next_to_clean = 0; 231 } 232 233 /** 234 * igc_free_tx_resources - Free Tx Resources per Queue 235 * @tx_ring: Tx descriptor ring for a specific queue 236 * 237 * Free all transmit software resources 238 */ 239 void igc_free_tx_resources(struct igc_ring *tx_ring) 240 { 241 igc_clean_tx_ring(tx_ring); 242 243 vfree(tx_ring->tx_buffer_info); 244 tx_ring->tx_buffer_info = NULL; 245 246 /* if not set, then don't free */ 247 if (!tx_ring->desc) 248 return; 249 250 dma_free_coherent(tx_ring->dev, tx_ring->size, 251 tx_ring->desc, tx_ring->dma); 252 253 tx_ring->desc = NULL; 254 } 255 256 /** 257 * igc_free_all_tx_resources - Free Tx Resources for All Queues 258 * @adapter: board private structure 259 * 260 * Free all transmit software resources 261 */ 262 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 263 { 264 int i; 265 266 for (i = 0; i < adapter->num_tx_queues; i++) 267 igc_free_tx_resources(adapter->tx_ring[i]); 268 } 269 270 /** 271 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 272 * @adapter: board private structure 273 */ 274 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 275 { 276 int i; 277 278 for (i = 0; i < adapter->num_tx_queues; i++) 279 if (adapter->tx_ring[i]) 280 igc_clean_tx_ring(adapter->tx_ring[i]); 281 } 282 283 /** 284 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 285 * @tx_ring: tx descriptor ring (for a specific queue) to setup 286 * 287 * Return 0 on success, negative on failure 288 */ 289 int igc_setup_tx_resources(struct igc_ring *tx_ring) 290 { 291 struct net_device *ndev = tx_ring->netdev; 292 struct device *dev = tx_ring->dev; 293 int size = 0; 294 295 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 296 tx_ring->tx_buffer_info = vzalloc(size); 297 if (!tx_ring->tx_buffer_info) 298 goto err; 299 300 /* round up to nearest 4K */ 301 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 302 tx_ring->size = ALIGN(tx_ring->size, 4096); 303 304 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 305 &tx_ring->dma, GFP_KERNEL); 306 307 if (!tx_ring->desc) 308 goto err; 309 310 tx_ring->next_to_use = 0; 311 tx_ring->next_to_clean = 0; 312 313 return 0; 314 315 err: 316 vfree(tx_ring->tx_buffer_info); 317 netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n"); 318 return -ENOMEM; 319 } 320 321 /** 322 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 323 * @adapter: board private structure 324 * 325 * Return 0 on success, negative on failure 326 */ 327 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 328 { 329 struct net_device *dev = adapter->netdev; 330 int i, err = 0; 331 332 for (i = 0; i < adapter->num_tx_queues; i++) { 333 err = igc_setup_tx_resources(adapter->tx_ring[i]); 334 if (err) { 335 netdev_err(dev, "Error on Tx queue %u setup\n", i); 336 for (i--; i >= 0; i--) 337 igc_free_tx_resources(adapter->tx_ring[i]); 338 break; 339 } 340 } 341 342 return err; 343 } 344 345 /** 346 * igc_clean_rx_ring - Free Rx Buffers per Queue 347 * @rx_ring: ring to free buffers from 348 */ 349 static void igc_clean_rx_ring(struct igc_ring *rx_ring) 350 { 351 u16 i = rx_ring->next_to_clean; 352 353 dev_kfree_skb(rx_ring->skb); 354 rx_ring->skb = NULL; 355 356 /* Free all the Rx ring sk_buffs */ 357 while (i != rx_ring->next_to_alloc) { 358 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 359 360 /* Invalidate cache lines that may have been written to by 361 * device so that we avoid corrupting memory. 362 */ 363 dma_sync_single_range_for_cpu(rx_ring->dev, 364 buffer_info->dma, 365 buffer_info->page_offset, 366 igc_rx_bufsz(rx_ring), 367 DMA_FROM_DEVICE); 368 369 /* free resources associated with mapping */ 370 dma_unmap_page_attrs(rx_ring->dev, 371 buffer_info->dma, 372 igc_rx_pg_size(rx_ring), 373 DMA_FROM_DEVICE, 374 IGC_RX_DMA_ATTR); 375 __page_frag_cache_drain(buffer_info->page, 376 buffer_info->pagecnt_bias); 377 378 i++; 379 if (i == rx_ring->count) 380 i = 0; 381 } 382 383 rx_ring->next_to_alloc = 0; 384 rx_ring->next_to_clean = 0; 385 rx_ring->next_to_use = 0; 386 } 387 388 /** 389 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 390 * @adapter: board private structure 391 */ 392 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 393 { 394 int i; 395 396 for (i = 0; i < adapter->num_rx_queues; i++) 397 if (adapter->rx_ring[i]) 398 igc_clean_rx_ring(adapter->rx_ring[i]); 399 } 400 401 /** 402 * igc_free_rx_resources - Free Rx Resources 403 * @rx_ring: ring to clean the resources from 404 * 405 * Free all receive software resources 406 */ 407 void igc_free_rx_resources(struct igc_ring *rx_ring) 408 { 409 igc_clean_rx_ring(rx_ring); 410 411 vfree(rx_ring->rx_buffer_info); 412 rx_ring->rx_buffer_info = NULL; 413 414 /* if not set, then don't free */ 415 if (!rx_ring->desc) 416 return; 417 418 dma_free_coherent(rx_ring->dev, rx_ring->size, 419 rx_ring->desc, rx_ring->dma); 420 421 rx_ring->desc = NULL; 422 } 423 424 /** 425 * igc_free_all_rx_resources - Free Rx Resources for All Queues 426 * @adapter: board private structure 427 * 428 * Free all receive software resources 429 */ 430 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 431 { 432 int i; 433 434 for (i = 0; i < adapter->num_rx_queues; i++) 435 igc_free_rx_resources(adapter->rx_ring[i]); 436 } 437 438 /** 439 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 440 * @rx_ring: rx descriptor ring (for a specific queue) to setup 441 * 442 * Returns 0 on success, negative on failure 443 */ 444 int igc_setup_rx_resources(struct igc_ring *rx_ring) 445 { 446 struct net_device *ndev = rx_ring->netdev; 447 struct device *dev = rx_ring->dev; 448 int size, desc_len; 449 450 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 451 rx_ring->rx_buffer_info = vzalloc(size); 452 if (!rx_ring->rx_buffer_info) 453 goto err; 454 455 desc_len = sizeof(union igc_adv_rx_desc); 456 457 /* Round up to nearest 4K */ 458 rx_ring->size = rx_ring->count * desc_len; 459 rx_ring->size = ALIGN(rx_ring->size, 4096); 460 461 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 462 &rx_ring->dma, GFP_KERNEL); 463 464 if (!rx_ring->desc) 465 goto err; 466 467 rx_ring->next_to_alloc = 0; 468 rx_ring->next_to_clean = 0; 469 rx_ring->next_to_use = 0; 470 471 return 0; 472 473 err: 474 vfree(rx_ring->rx_buffer_info); 475 rx_ring->rx_buffer_info = NULL; 476 netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n"); 477 return -ENOMEM; 478 } 479 480 /** 481 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 482 * (Descriptors) for all queues 483 * @adapter: board private structure 484 * 485 * Return 0 on success, negative on failure 486 */ 487 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 488 { 489 struct net_device *dev = adapter->netdev; 490 int i, err = 0; 491 492 for (i = 0; i < adapter->num_rx_queues; i++) { 493 err = igc_setup_rx_resources(adapter->rx_ring[i]); 494 if (err) { 495 netdev_err(dev, "Error on Rx queue %u setup\n", i); 496 for (i--; i >= 0; i--) 497 igc_free_rx_resources(adapter->rx_ring[i]); 498 break; 499 } 500 } 501 502 return err; 503 } 504 505 /** 506 * igc_configure_rx_ring - Configure a receive ring after Reset 507 * @adapter: board private structure 508 * @ring: receive ring to be configured 509 * 510 * Configure the Rx unit of the MAC after a reset. 511 */ 512 static void igc_configure_rx_ring(struct igc_adapter *adapter, 513 struct igc_ring *ring) 514 { 515 struct igc_hw *hw = &adapter->hw; 516 union igc_adv_rx_desc *rx_desc; 517 int reg_idx = ring->reg_idx; 518 u32 srrctl = 0, rxdctl = 0; 519 u64 rdba = ring->dma; 520 521 /* disable the queue */ 522 wr32(IGC_RXDCTL(reg_idx), 0); 523 524 /* Set DMA base address registers */ 525 wr32(IGC_RDBAL(reg_idx), 526 rdba & 0x00000000ffffffffULL); 527 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 528 wr32(IGC_RDLEN(reg_idx), 529 ring->count * sizeof(union igc_adv_rx_desc)); 530 531 /* initialize head and tail */ 532 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 533 wr32(IGC_RDH(reg_idx), 0); 534 writel(0, ring->tail); 535 536 /* reset next-to- use/clean to place SW in sync with hardware */ 537 ring->next_to_clean = 0; 538 ring->next_to_use = 0; 539 540 /* set descriptor configuration */ 541 srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; 542 if (ring_uses_large_buffer(ring)) 543 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 544 else 545 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 546 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 547 548 wr32(IGC_SRRCTL(reg_idx), srrctl); 549 550 rxdctl |= IGC_RX_PTHRESH; 551 rxdctl |= IGC_RX_HTHRESH << 8; 552 rxdctl |= IGC_RX_WTHRESH << 16; 553 554 /* initialize rx_buffer_info */ 555 memset(ring->rx_buffer_info, 0, 556 sizeof(struct igc_rx_buffer) * ring->count); 557 558 /* initialize Rx descriptor 0 */ 559 rx_desc = IGC_RX_DESC(ring, 0); 560 rx_desc->wb.upper.length = 0; 561 562 /* enable receive descriptor fetching */ 563 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 564 565 wr32(IGC_RXDCTL(reg_idx), rxdctl); 566 } 567 568 /** 569 * igc_configure_rx - Configure receive Unit after Reset 570 * @adapter: board private structure 571 * 572 * Configure the Rx unit of the MAC after a reset. 573 */ 574 static void igc_configure_rx(struct igc_adapter *adapter) 575 { 576 int i; 577 578 /* Setup the HW Rx Head and Tail Descriptor Pointers and 579 * the Base and Length of the Rx Descriptor Ring 580 */ 581 for (i = 0; i < adapter->num_rx_queues; i++) 582 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 583 } 584 585 /** 586 * igc_configure_tx_ring - Configure transmit ring after Reset 587 * @adapter: board private structure 588 * @ring: tx ring to configure 589 * 590 * Configure a transmit ring after a reset. 591 */ 592 static void igc_configure_tx_ring(struct igc_adapter *adapter, 593 struct igc_ring *ring) 594 { 595 struct igc_hw *hw = &adapter->hw; 596 int reg_idx = ring->reg_idx; 597 u64 tdba = ring->dma; 598 u32 txdctl = 0; 599 600 /* disable the queue */ 601 wr32(IGC_TXDCTL(reg_idx), 0); 602 wrfl(); 603 mdelay(10); 604 605 wr32(IGC_TDLEN(reg_idx), 606 ring->count * sizeof(union igc_adv_tx_desc)); 607 wr32(IGC_TDBAL(reg_idx), 608 tdba & 0x00000000ffffffffULL); 609 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 610 611 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 612 wr32(IGC_TDH(reg_idx), 0); 613 writel(0, ring->tail); 614 615 txdctl |= IGC_TX_PTHRESH; 616 txdctl |= IGC_TX_HTHRESH << 8; 617 txdctl |= IGC_TX_WTHRESH << 16; 618 619 txdctl |= IGC_TXDCTL_QUEUE_ENABLE; 620 wr32(IGC_TXDCTL(reg_idx), txdctl); 621 } 622 623 /** 624 * igc_configure_tx - Configure transmit Unit after Reset 625 * @adapter: board private structure 626 * 627 * Configure the Tx unit of the MAC after a reset. 628 */ 629 static void igc_configure_tx(struct igc_adapter *adapter) 630 { 631 int i; 632 633 for (i = 0; i < adapter->num_tx_queues; i++) 634 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 635 } 636 637 /** 638 * igc_setup_mrqc - configure the multiple receive queue control registers 639 * @adapter: Board private structure 640 */ 641 static void igc_setup_mrqc(struct igc_adapter *adapter) 642 { 643 struct igc_hw *hw = &adapter->hw; 644 u32 j, num_rx_queues; 645 u32 mrqc, rxcsum; 646 u32 rss_key[10]; 647 648 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 649 for (j = 0; j < 10; j++) 650 wr32(IGC_RSSRK(j), rss_key[j]); 651 652 num_rx_queues = adapter->rss_queues; 653 654 if (adapter->rss_indir_tbl_init != num_rx_queues) { 655 for (j = 0; j < IGC_RETA_SIZE; j++) 656 adapter->rss_indir_tbl[j] = 657 (j * num_rx_queues) / IGC_RETA_SIZE; 658 adapter->rss_indir_tbl_init = num_rx_queues; 659 } 660 igc_write_rss_indir_tbl(adapter); 661 662 /* Disable raw packet checksumming so that RSS hash is placed in 663 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 664 * offloads as they are enabled by default 665 */ 666 rxcsum = rd32(IGC_RXCSUM); 667 rxcsum |= IGC_RXCSUM_PCSD; 668 669 /* Enable Receive Checksum Offload for SCTP */ 670 rxcsum |= IGC_RXCSUM_CRCOFL; 671 672 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 673 wr32(IGC_RXCSUM, rxcsum); 674 675 /* Generate RSS hash based on packet types, TCP/UDP 676 * port numbers and/or IPv4/v6 src and dst addresses 677 */ 678 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 679 IGC_MRQC_RSS_FIELD_IPV4_TCP | 680 IGC_MRQC_RSS_FIELD_IPV6 | 681 IGC_MRQC_RSS_FIELD_IPV6_TCP | 682 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 683 684 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 685 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 686 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 687 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 688 689 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 690 691 wr32(IGC_MRQC, mrqc); 692 } 693 694 /** 695 * igc_setup_rctl - configure the receive control registers 696 * @adapter: Board private structure 697 */ 698 static void igc_setup_rctl(struct igc_adapter *adapter) 699 { 700 struct igc_hw *hw = &adapter->hw; 701 u32 rctl; 702 703 rctl = rd32(IGC_RCTL); 704 705 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 706 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 707 708 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 709 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 710 711 /* enable stripping of CRC. Newer features require 712 * that the HW strips the CRC. 713 */ 714 rctl |= IGC_RCTL_SECRC; 715 716 /* disable store bad packets and clear size bits. */ 717 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 718 719 /* enable LPE to allow for reception of jumbo frames */ 720 rctl |= IGC_RCTL_LPE; 721 722 /* disable queue 0 to prevent tail write w/o re-config */ 723 wr32(IGC_RXDCTL(0), 0); 724 725 /* This is useful for sniffing bad packets. */ 726 if (adapter->netdev->features & NETIF_F_RXALL) { 727 /* UPE and MPE will be handled by normal PROMISC logic 728 * in set_rx_mode 729 */ 730 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 731 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 732 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 733 734 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 735 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 736 } 737 738 wr32(IGC_RCTL, rctl); 739 } 740 741 /** 742 * igc_setup_tctl - configure the transmit control registers 743 * @adapter: Board private structure 744 */ 745 static void igc_setup_tctl(struct igc_adapter *adapter) 746 { 747 struct igc_hw *hw = &adapter->hw; 748 u32 tctl; 749 750 /* disable queue 0 which icould be enabled by default */ 751 wr32(IGC_TXDCTL(0), 0); 752 753 /* Program the Transmit Control Register */ 754 tctl = rd32(IGC_TCTL); 755 tctl &= ~IGC_TCTL_CT; 756 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 757 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 758 759 /* Enable transmits */ 760 tctl |= IGC_TCTL_EN; 761 762 wr32(IGC_TCTL, tctl); 763 } 764 765 /** 766 * igc_set_mac_filter_hw() - Set MAC address filter in hardware 767 * @adapter: Pointer to adapter where the filter should be set 768 * @index: Filter index 769 * @type: MAC address filter type (source or destination) 770 * @addr: MAC address 771 * @queue: If non-negative, queue assignment feature is enabled and frames 772 * matching the filter are enqueued onto 'queue'. Otherwise, queue 773 * assignment is disabled. 774 */ 775 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, 776 enum igc_mac_filter_type type, 777 const u8 *addr, int queue) 778 { 779 struct net_device *dev = adapter->netdev; 780 struct igc_hw *hw = &adapter->hw; 781 u32 ral, rah; 782 783 if (WARN_ON(index >= hw->mac.rar_entry_count)) 784 return; 785 786 ral = le32_to_cpup((__le32 *)(addr)); 787 rah = le16_to_cpup((__le16 *)(addr + 4)); 788 789 if (type == IGC_MAC_FILTER_TYPE_SRC) { 790 rah &= ~IGC_RAH_ASEL_MASK; 791 rah |= IGC_RAH_ASEL_SRC_ADDR; 792 } 793 794 if (queue >= 0) { 795 rah &= ~IGC_RAH_QSEL_MASK; 796 rah |= (queue << IGC_RAH_QSEL_SHIFT); 797 rah |= IGC_RAH_QSEL_ENABLE; 798 } 799 800 rah |= IGC_RAH_AV; 801 802 wr32(IGC_RAL(index), ral); 803 wr32(IGC_RAH(index), rah); 804 805 netdev_dbg(dev, "MAC address filter set in HW: index %d", index); 806 } 807 808 /** 809 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware 810 * @adapter: Pointer to adapter where the filter should be cleared 811 * @index: Filter index 812 */ 813 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) 814 { 815 struct net_device *dev = adapter->netdev; 816 struct igc_hw *hw = &adapter->hw; 817 818 if (WARN_ON(index >= hw->mac.rar_entry_count)) 819 return; 820 821 wr32(IGC_RAL(index), 0); 822 wr32(IGC_RAH(index), 0); 823 824 netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index); 825 } 826 827 /* Set default MAC address for the PF in the first RAR entry */ 828 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 829 { 830 struct net_device *dev = adapter->netdev; 831 u8 *addr = adapter->hw.mac.addr; 832 833 netdev_dbg(dev, "Set default MAC address filter: address %pM", addr); 834 835 igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1); 836 } 837 838 /** 839 * igc_set_mac - Change the Ethernet Address of the NIC 840 * @netdev: network interface device structure 841 * @p: pointer to an address structure 842 * 843 * Returns 0 on success, negative on failure 844 */ 845 static int igc_set_mac(struct net_device *netdev, void *p) 846 { 847 struct igc_adapter *adapter = netdev_priv(netdev); 848 struct igc_hw *hw = &adapter->hw; 849 struct sockaddr *addr = p; 850 851 if (!is_valid_ether_addr(addr->sa_data)) 852 return -EADDRNOTAVAIL; 853 854 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 855 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 856 857 /* set the correct pool for the new PF MAC address in entry 0 */ 858 igc_set_default_mac_filter(adapter); 859 860 return 0; 861 } 862 863 /** 864 * igc_write_mc_addr_list - write multicast addresses to MTA 865 * @netdev: network interface device structure 866 * 867 * Writes multicast address list to the MTA hash table. 868 * Returns: -ENOMEM on failure 869 * 0 on no addresses written 870 * X on writing X addresses to MTA 871 **/ 872 static int igc_write_mc_addr_list(struct net_device *netdev) 873 { 874 struct igc_adapter *adapter = netdev_priv(netdev); 875 struct igc_hw *hw = &adapter->hw; 876 struct netdev_hw_addr *ha; 877 u8 *mta_list; 878 int i; 879 880 if (netdev_mc_empty(netdev)) { 881 /* nothing to program, so clear mc list */ 882 igc_update_mc_addr_list(hw, NULL, 0); 883 return 0; 884 } 885 886 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 887 if (!mta_list) 888 return -ENOMEM; 889 890 /* The shared function expects a packed array of only addresses. */ 891 i = 0; 892 netdev_for_each_mc_addr(ha, netdev) 893 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 894 895 igc_update_mc_addr_list(hw, mta_list, i); 896 kfree(mta_list); 897 898 return netdev_mc_count(netdev); 899 } 900 901 static __le32 igc_tx_launchtime(struct igc_adapter *adapter, ktime_t txtime) 902 { 903 ktime_t cycle_time = adapter->cycle_time; 904 ktime_t base_time = adapter->base_time; 905 u32 launchtime; 906 907 /* FIXME: when using ETF together with taprio, we may have a 908 * case where 'delta' is larger than the cycle_time, this may 909 * cause problems if we don't read the current value of 910 * IGC_BASET, as the value writen into the launchtime 911 * descriptor field may be misinterpreted. 912 */ 913 div_s64_rem(ktime_sub_ns(txtime, base_time), cycle_time, &launchtime); 914 915 return cpu_to_le32(launchtime); 916 } 917 918 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, 919 struct igc_tx_buffer *first, 920 u32 vlan_macip_lens, u32 type_tucmd, 921 u32 mss_l4len_idx) 922 { 923 struct igc_adv_tx_context_desc *context_desc; 924 u16 i = tx_ring->next_to_use; 925 926 context_desc = IGC_TX_CTXTDESC(tx_ring, i); 927 928 i++; 929 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 930 931 /* set bits to identify this as an advanced context descriptor */ 932 type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 933 934 /* For i225, context index must be unique per ring. */ 935 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 936 mss_l4len_idx |= tx_ring->reg_idx << 4; 937 938 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 939 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 940 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 941 942 /* We assume there is always a valid Tx time available. Invalid times 943 * should have been handled by the upper layers. 944 */ 945 if (tx_ring->launchtime_enable) { 946 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 947 ktime_t txtime = first->skb->tstamp; 948 949 first->skb->tstamp = ktime_set(0, 0); 950 context_desc->launch_time = igc_tx_launchtime(adapter, 951 txtime); 952 } else { 953 context_desc->launch_time = 0; 954 } 955 } 956 957 static inline bool igc_ipv6_csum_is_sctp(struct sk_buff *skb) 958 { 959 unsigned int offset = 0; 960 961 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 962 963 return offset == skb_checksum_start_offset(skb); 964 } 965 966 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first) 967 { 968 struct sk_buff *skb = first->skb; 969 u32 vlan_macip_lens = 0; 970 u32 type_tucmd = 0; 971 972 if (skb->ip_summed != CHECKSUM_PARTIAL) { 973 csum_failed: 974 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && 975 !tx_ring->launchtime_enable) 976 return; 977 goto no_csum; 978 } 979 980 switch (skb->csum_offset) { 981 case offsetof(struct tcphdr, check): 982 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 983 /* fall through */ 984 case offsetof(struct udphdr, check): 985 break; 986 case offsetof(struct sctphdr, checksum): 987 /* validate that this is actually an SCTP request */ 988 if ((first->protocol == htons(ETH_P_IP) && 989 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 990 (first->protocol == htons(ETH_P_IPV6) && 991 igc_ipv6_csum_is_sctp(skb))) { 992 type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; 993 break; 994 } 995 /* fall through */ 996 default: 997 skb_checksum_help(skb); 998 goto csum_failed; 999 } 1000 1001 /* update TX checksum flag */ 1002 first->tx_flags |= IGC_TX_FLAGS_CSUM; 1003 vlan_macip_lens = skb_checksum_start_offset(skb) - 1004 skb_network_offset(skb); 1005 no_csum: 1006 vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; 1007 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1008 1009 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0); 1010 } 1011 1012 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1013 { 1014 struct net_device *netdev = tx_ring->netdev; 1015 1016 netif_stop_subqueue(netdev, tx_ring->queue_index); 1017 1018 /* memory barriier comment */ 1019 smp_mb(); 1020 1021 /* We need to check again in a case another CPU has just 1022 * made room available. 1023 */ 1024 if (igc_desc_unused(tx_ring) < size) 1025 return -EBUSY; 1026 1027 /* A reprieve! */ 1028 netif_wake_subqueue(netdev, tx_ring->queue_index); 1029 1030 u64_stats_update_begin(&tx_ring->tx_syncp2); 1031 tx_ring->tx_stats.restart_queue2++; 1032 u64_stats_update_end(&tx_ring->tx_syncp2); 1033 1034 return 0; 1035 } 1036 1037 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1038 { 1039 if (igc_desc_unused(tx_ring) >= size) 1040 return 0; 1041 return __igc_maybe_stop_tx(tx_ring, size); 1042 } 1043 1044 #define IGC_SET_FLAG(_input, _flag, _result) \ 1045 (((_flag) <= (_result)) ? \ 1046 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ 1047 ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) 1048 1049 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 1050 { 1051 /* set type for advanced descriptor with frame checksum insertion */ 1052 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 1053 IGC_ADVTXD_DCMD_DEXT | 1054 IGC_ADVTXD_DCMD_IFCS; 1055 1056 /* set segmentation bits for TSO */ 1057 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, 1058 (IGC_ADVTXD_DCMD_TSE)); 1059 1060 /* set timestamp bit if present */ 1061 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, 1062 (IGC_ADVTXD_MAC_TSTAMP)); 1063 1064 return cmd_type; 1065 } 1066 1067 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 1068 union igc_adv_tx_desc *tx_desc, 1069 u32 tx_flags, unsigned int paylen) 1070 { 1071 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 1072 1073 /* insert L4 checksum */ 1074 olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) * 1075 ((IGC_TXD_POPTS_TXSM << 8) / 1076 IGC_TX_FLAGS_CSUM); 1077 1078 /* insert IPv4 checksum */ 1079 olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) * 1080 (((IGC_TXD_POPTS_IXSM << 8)) / 1081 IGC_TX_FLAGS_IPV4); 1082 1083 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 1084 } 1085 1086 static int igc_tx_map(struct igc_ring *tx_ring, 1087 struct igc_tx_buffer *first, 1088 const u8 hdr_len) 1089 { 1090 struct sk_buff *skb = first->skb; 1091 struct igc_tx_buffer *tx_buffer; 1092 union igc_adv_tx_desc *tx_desc; 1093 u32 tx_flags = first->tx_flags; 1094 skb_frag_t *frag; 1095 u16 i = tx_ring->next_to_use; 1096 unsigned int data_len, size; 1097 dma_addr_t dma; 1098 u32 cmd_type = igc_tx_cmd_type(skb, tx_flags); 1099 1100 tx_desc = IGC_TX_DESC(tx_ring, i); 1101 1102 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 1103 1104 size = skb_headlen(skb); 1105 data_len = skb->data_len; 1106 1107 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1108 1109 tx_buffer = first; 1110 1111 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1112 if (dma_mapping_error(tx_ring->dev, dma)) 1113 goto dma_error; 1114 1115 /* record length, and DMA address */ 1116 dma_unmap_len_set(tx_buffer, len, size); 1117 dma_unmap_addr_set(tx_buffer, dma, dma); 1118 1119 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1120 1121 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 1122 tx_desc->read.cmd_type_len = 1123 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 1124 1125 i++; 1126 tx_desc++; 1127 if (i == tx_ring->count) { 1128 tx_desc = IGC_TX_DESC(tx_ring, 0); 1129 i = 0; 1130 } 1131 tx_desc->read.olinfo_status = 0; 1132 1133 dma += IGC_MAX_DATA_PER_TXD; 1134 size -= IGC_MAX_DATA_PER_TXD; 1135 1136 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1137 } 1138 1139 if (likely(!data_len)) 1140 break; 1141 1142 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 1143 1144 i++; 1145 tx_desc++; 1146 if (i == tx_ring->count) { 1147 tx_desc = IGC_TX_DESC(tx_ring, 0); 1148 i = 0; 1149 } 1150 tx_desc->read.olinfo_status = 0; 1151 1152 size = skb_frag_size(frag); 1153 data_len -= size; 1154 1155 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 1156 size, DMA_TO_DEVICE); 1157 1158 tx_buffer = &tx_ring->tx_buffer_info[i]; 1159 } 1160 1161 /* write last descriptor with RS and EOP bits */ 1162 cmd_type |= size | IGC_TXD_DCMD; 1163 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1164 1165 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1166 1167 /* set the timestamp */ 1168 first->time_stamp = jiffies; 1169 1170 skb_tx_timestamp(skb); 1171 1172 /* Force memory writes to complete before letting h/w know there 1173 * are new descriptors to fetch. (Only applicable for weak-ordered 1174 * memory model archs, such as IA-64). 1175 * 1176 * We also need this memory barrier to make certain all of the 1177 * status bits have been updated before next_to_watch is written. 1178 */ 1179 wmb(); 1180 1181 /* set next_to_watch value indicating a packet is present */ 1182 first->next_to_watch = tx_desc; 1183 1184 i++; 1185 if (i == tx_ring->count) 1186 i = 0; 1187 1188 tx_ring->next_to_use = i; 1189 1190 /* Make sure there is space in the ring for the next send. */ 1191 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 1192 1193 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1194 writel(i, tx_ring->tail); 1195 } 1196 1197 return 0; 1198 dma_error: 1199 netdev_err(tx_ring->netdev, "TX DMA map failed\n"); 1200 tx_buffer = &tx_ring->tx_buffer_info[i]; 1201 1202 /* clear dma mappings for failed tx_buffer_info map */ 1203 while (tx_buffer != first) { 1204 if (dma_unmap_len(tx_buffer, len)) 1205 dma_unmap_page(tx_ring->dev, 1206 dma_unmap_addr(tx_buffer, dma), 1207 dma_unmap_len(tx_buffer, len), 1208 DMA_TO_DEVICE); 1209 dma_unmap_len_set(tx_buffer, len, 0); 1210 1211 if (i-- == 0) 1212 i += tx_ring->count; 1213 tx_buffer = &tx_ring->tx_buffer_info[i]; 1214 } 1215 1216 if (dma_unmap_len(tx_buffer, len)) 1217 dma_unmap_single(tx_ring->dev, 1218 dma_unmap_addr(tx_buffer, dma), 1219 dma_unmap_len(tx_buffer, len), 1220 DMA_TO_DEVICE); 1221 dma_unmap_len_set(tx_buffer, len, 0); 1222 1223 dev_kfree_skb_any(tx_buffer->skb); 1224 tx_buffer->skb = NULL; 1225 1226 tx_ring->next_to_use = i; 1227 1228 return -1; 1229 } 1230 1231 static int igc_tso(struct igc_ring *tx_ring, 1232 struct igc_tx_buffer *first, 1233 u8 *hdr_len) 1234 { 1235 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 1236 struct sk_buff *skb = first->skb; 1237 union { 1238 struct iphdr *v4; 1239 struct ipv6hdr *v6; 1240 unsigned char *hdr; 1241 } ip; 1242 union { 1243 struct tcphdr *tcp; 1244 struct udphdr *udp; 1245 unsigned char *hdr; 1246 } l4; 1247 u32 paylen, l4_offset; 1248 int err; 1249 1250 if (skb->ip_summed != CHECKSUM_PARTIAL) 1251 return 0; 1252 1253 if (!skb_is_gso(skb)) 1254 return 0; 1255 1256 err = skb_cow_head(skb, 0); 1257 if (err < 0) 1258 return err; 1259 1260 ip.hdr = skb_network_header(skb); 1261 l4.hdr = skb_checksum_start(skb); 1262 1263 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 1264 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 1265 1266 /* initialize outer IP header fields */ 1267 if (ip.v4->version == 4) { 1268 unsigned char *csum_start = skb_checksum_start(skb); 1269 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 1270 1271 /* IP header will have to cancel out any data that 1272 * is not a part of the outer IP header 1273 */ 1274 ip.v4->check = csum_fold(csum_partial(trans_start, 1275 csum_start - trans_start, 1276 0)); 1277 type_tucmd |= IGC_ADVTXD_TUCMD_IPV4; 1278 1279 ip.v4->tot_len = 0; 1280 first->tx_flags |= IGC_TX_FLAGS_TSO | 1281 IGC_TX_FLAGS_CSUM | 1282 IGC_TX_FLAGS_IPV4; 1283 } else { 1284 ip.v6->payload_len = 0; 1285 first->tx_flags |= IGC_TX_FLAGS_TSO | 1286 IGC_TX_FLAGS_CSUM; 1287 } 1288 1289 /* determine offset of inner transport header */ 1290 l4_offset = l4.hdr - skb->data; 1291 1292 /* remove payload length from inner checksum */ 1293 paylen = skb->len - l4_offset; 1294 if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) { 1295 /* compute length of segmentation header */ 1296 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 1297 csum_replace_by_diff(&l4.tcp->check, 1298 (__force __wsum)htonl(paylen)); 1299 } else { 1300 /* compute length of segmentation header */ 1301 *hdr_len = sizeof(*l4.udp) + l4_offset; 1302 csum_replace_by_diff(&l4.udp->check, 1303 (__force __wsum)htonl(paylen)); 1304 } 1305 1306 /* update gso size and bytecount with header size */ 1307 first->gso_segs = skb_shinfo(skb)->gso_segs; 1308 first->bytecount += (first->gso_segs - 1) * *hdr_len; 1309 1310 /* MSS L4LEN IDX */ 1311 mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT; 1312 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT; 1313 1314 /* VLAN MACLEN IPLEN */ 1315 vlan_macip_lens = l4.hdr - ip.hdr; 1316 vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT; 1317 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1318 1319 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, 1320 type_tucmd, mss_l4len_idx); 1321 1322 return 1; 1323 } 1324 1325 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1326 struct igc_ring *tx_ring) 1327 { 1328 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1329 __be16 protocol = vlan_get_protocol(skb); 1330 struct igc_tx_buffer *first; 1331 u32 tx_flags = 0; 1332 unsigned short f; 1333 u8 hdr_len = 0; 1334 int tso = 0; 1335 1336 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1337 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1338 * + 2 desc gap to keep tail from touching head, 1339 * + 1 desc for context descriptor, 1340 * otherwise try next time 1341 */ 1342 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1343 count += TXD_USE_COUNT(skb_frag_size( 1344 &skb_shinfo(skb)->frags[f])); 1345 1346 if (igc_maybe_stop_tx(tx_ring, count + 3)) { 1347 /* this is a hard error */ 1348 return NETDEV_TX_BUSY; 1349 } 1350 1351 /* record the location of the first descriptor for this packet */ 1352 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1353 first->skb = skb; 1354 first->bytecount = skb->len; 1355 first->gso_segs = 1; 1356 1357 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 1358 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 1359 1360 /* FIXME: add support for retrieving timestamps from 1361 * the other timer registers before skipping the 1362 * timestamping request. 1363 */ 1364 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && 1365 !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS, 1366 &adapter->state)) { 1367 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1368 tx_flags |= IGC_TX_FLAGS_TSTAMP; 1369 1370 adapter->ptp_tx_skb = skb_get(skb); 1371 adapter->ptp_tx_start = jiffies; 1372 } else { 1373 adapter->tx_hwtstamp_skipped++; 1374 } 1375 } 1376 1377 /* record initial flags and protocol */ 1378 first->tx_flags = tx_flags; 1379 first->protocol = protocol; 1380 1381 tso = igc_tso(tx_ring, first, &hdr_len); 1382 if (tso < 0) 1383 goto out_drop; 1384 else if (!tso) 1385 igc_tx_csum(tx_ring, first); 1386 1387 igc_tx_map(tx_ring, first, hdr_len); 1388 1389 return NETDEV_TX_OK; 1390 1391 out_drop: 1392 dev_kfree_skb_any(first->skb); 1393 first->skb = NULL; 1394 1395 return NETDEV_TX_OK; 1396 } 1397 1398 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1399 struct sk_buff *skb) 1400 { 1401 unsigned int r_idx = skb->queue_mapping; 1402 1403 if (r_idx >= adapter->num_tx_queues) 1404 r_idx = r_idx % adapter->num_tx_queues; 1405 1406 return adapter->tx_ring[r_idx]; 1407 } 1408 1409 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1410 struct net_device *netdev) 1411 { 1412 struct igc_adapter *adapter = netdev_priv(netdev); 1413 1414 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1415 * in order to meet this minimum size requirement. 1416 */ 1417 if (skb->len < 17) { 1418 if (skb_padto(skb, 17)) 1419 return NETDEV_TX_OK; 1420 skb->len = 17; 1421 } 1422 1423 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1424 } 1425 1426 static void igc_rx_checksum(struct igc_ring *ring, 1427 union igc_adv_rx_desc *rx_desc, 1428 struct sk_buff *skb) 1429 { 1430 skb_checksum_none_assert(skb); 1431 1432 /* Ignore Checksum bit is set */ 1433 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) 1434 return; 1435 1436 /* Rx checksum disabled via ethtool */ 1437 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1438 return; 1439 1440 /* TCP/UDP checksum error bit is set */ 1441 if (igc_test_staterr(rx_desc, 1442 IGC_RXDEXT_STATERR_TCPE | 1443 IGC_RXDEXT_STATERR_IPE)) { 1444 /* work around errata with sctp packets where the TCPE aka 1445 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 1446 * packets (aka let the stack check the crc32c) 1447 */ 1448 if (!(skb->len == 60 && 1449 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 1450 u64_stats_update_begin(&ring->rx_syncp); 1451 ring->rx_stats.csum_err++; 1452 u64_stats_update_end(&ring->rx_syncp); 1453 } 1454 /* let the stack verify checksum errors */ 1455 return; 1456 } 1457 /* It must be a TCP or UDP packet with a valid checksum */ 1458 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | 1459 IGC_RXD_STAT_UDPCS)) 1460 skb->ip_summed = CHECKSUM_UNNECESSARY; 1461 1462 netdev_dbg(ring->netdev, "cksum success: bits %08X\n", 1463 le32_to_cpu(rx_desc->wb.upper.status_error)); 1464 } 1465 1466 static inline void igc_rx_hash(struct igc_ring *ring, 1467 union igc_adv_rx_desc *rx_desc, 1468 struct sk_buff *skb) 1469 { 1470 if (ring->netdev->features & NETIF_F_RXHASH) 1471 skb_set_hash(skb, 1472 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 1473 PKT_HASH_TYPE_L3); 1474 } 1475 1476 /** 1477 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1478 * @rx_ring: rx descriptor ring packet is being transacted on 1479 * @rx_desc: pointer to the EOP Rx descriptor 1480 * @skb: pointer to current skb being populated 1481 * 1482 * This function checks the ring, descriptor, and packet information in 1483 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 1484 * other fields within the skb. 1485 */ 1486 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1487 union igc_adv_rx_desc *rx_desc, 1488 struct sk_buff *skb) 1489 { 1490 igc_rx_hash(rx_ring, rx_desc, skb); 1491 1492 igc_rx_checksum(rx_ring, rx_desc, skb); 1493 1494 if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TS) && 1495 !igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) 1496 igc_ptp_rx_rgtstamp(rx_ring->q_vector, skb); 1497 1498 skb_record_rx_queue(skb, rx_ring->queue_index); 1499 1500 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1501 } 1502 1503 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1504 const unsigned int size) 1505 { 1506 struct igc_rx_buffer *rx_buffer; 1507 1508 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1509 prefetchw(rx_buffer->page); 1510 1511 /* we are reusing so sync this buffer for CPU use */ 1512 dma_sync_single_range_for_cpu(rx_ring->dev, 1513 rx_buffer->dma, 1514 rx_buffer->page_offset, 1515 size, 1516 DMA_FROM_DEVICE); 1517 1518 rx_buffer->pagecnt_bias--; 1519 1520 return rx_buffer; 1521 } 1522 1523 /** 1524 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1525 * @rx_ring: rx descriptor ring to transact packets on 1526 * @rx_buffer: buffer containing page to add 1527 * @skb: sk_buff to place the data into 1528 * @size: size of buffer to be added 1529 * 1530 * This function will add the data contained in rx_buffer->page to the skb. 1531 */ 1532 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1533 struct igc_rx_buffer *rx_buffer, 1534 struct sk_buff *skb, 1535 unsigned int size) 1536 { 1537 #if (PAGE_SIZE < 8192) 1538 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1539 1540 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1541 rx_buffer->page_offset, size, truesize); 1542 rx_buffer->page_offset ^= truesize; 1543 #else 1544 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1545 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1546 SKB_DATA_ALIGN(size); 1547 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1548 rx_buffer->page_offset, size, truesize); 1549 rx_buffer->page_offset += truesize; 1550 #endif 1551 } 1552 1553 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1554 struct igc_rx_buffer *rx_buffer, 1555 union igc_adv_rx_desc *rx_desc, 1556 unsigned int size) 1557 { 1558 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1559 #if (PAGE_SIZE < 8192) 1560 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1561 #else 1562 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1563 SKB_DATA_ALIGN(IGC_SKB_PAD + size); 1564 #endif 1565 struct sk_buff *skb; 1566 1567 /* prefetch first cache line of first page */ 1568 prefetch(va); 1569 #if L1_CACHE_BYTES < 128 1570 prefetch(va + L1_CACHE_BYTES); 1571 #endif 1572 1573 /* build an skb around the page buffer */ 1574 skb = build_skb(va - IGC_SKB_PAD, truesize); 1575 if (unlikely(!skb)) 1576 return NULL; 1577 1578 /* update pointers within the skb to store the data */ 1579 skb_reserve(skb, IGC_SKB_PAD); 1580 __skb_put(skb, size); 1581 1582 /* update buffer offset */ 1583 #if (PAGE_SIZE < 8192) 1584 rx_buffer->page_offset ^= truesize; 1585 #else 1586 rx_buffer->page_offset += truesize; 1587 #endif 1588 1589 return skb; 1590 } 1591 1592 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1593 struct igc_rx_buffer *rx_buffer, 1594 union igc_adv_rx_desc *rx_desc, 1595 unsigned int size) 1596 { 1597 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1598 #if (PAGE_SIZE < 8192) 1599 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1600 #else 1601 unsigned int truesize = SKB_DATA_ALIGN(size); 1602 #endif 1603 unsigned int headlen; 1604 struct sk_buff *skb; 1605 1606 /* prefetch first cache line of first page */ 1607 prefetch(va); 1608 #if L1_CACHE_BYTES < 128 1609 prefetch(va + L1_CACHE_BYTES); 1610 #endif 1611 1612 /* allocate a skb to store the frags */ 1613 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); 1614 if (unlikely(!skb)) 1615 return NULL; 1616 1617 if (unlikely(igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP))) { 1618 igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb); 1619 va += IGC_TS_HDR_LEN; 1620 size -= IGC_TS_HDR_LEN; 1621 } 1622 1623 /* Determine available headroom for copy */ 1624 headlen = size; 1625 if (headlen > IGC_RX_HDR_LEN) 1626 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 1627 1628 /* align pull length to size of long to optimize memcpy performance */ 1629 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 1630 1631 /* update all of the pointers */ 1632 size -= headlen; 1633 if (size) { 1634 skb_add_rx_frag(skb, 0, rx_buffer->page, 1635 (va + headlen) - page_address(rx_buffer->page), 1636 size, truesize); 1637 #if (PAGE_SIZE < 8192) 1638 rx_buffer->page_offset ^= truesize; 1639 #else 1640 rx_buffer->page_offset += truesize; 1641 #endif 1642 } else { 1643 rx_buffer->pagecnt_bias++; 1644 } 1645 1646 return skb; 1647 } 1648 1649 /** 1650 * igc_reuse_rx_page - page flip buffer and store it back on the ring 1651 * @rx_ring: rx descriptor ring to store buffers on 1652 * @old_buff: donor buffer to have page reused 1653 * 1654 * Synchronizes page for reuse by the adapter 1655 */ 1656 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 1657 struct igc_rx_buffer *old_buff) 1658 { 1659 u16 nta = rx_ring->next_to_alloc; 1660 struct igc_rx_buffer *new_buff; 1661 1662 new_buff = &rx_ring->rx_buffer_info[nta]; 1663 1664 /* update, and store next to alloc */ 1665 nta++; 1666 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1667 1668 /* Transfer page from old buffer to new buffer. 1669 * Move each member individually to avoid possible store 1670 * forwarding stalls. 1671 */ 1672 new_buff->dma = old_buff->dma; 1673 new_buff->page = old_buff->page; 1674 new_buff->page_offset = old_buff->page_offset; 1675 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1676 } 1677 1678 static inline bool igc_page_is_reserved(struct page *page) 1679 { 1680 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 1681 } 1682 1683 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) 1684 { 1685 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1686 struct page *page = rx_buffer->page; 1687 1688 /* avoid re-using remote pages */ 1689 if (unlikely(igc_page_is_reserved(page))) 1690 return false; 1691 1692 #if (PAGE_SIZE < 8192) 1693 /* if we are only owner of page we can reuse it */ 1694 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 1695 return false; 1696 #else 1697 #define IGC_LAST_OFFSET \ 1698 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 1699 1700 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 1701 return false; 1702 #endif 1703 1704 /* If we have drained the page fragment pool we need to update 1705 * the pagecnt_bias and page count so that we fully restock the 1706 * number of references the driver holds. 1707 */ 1708 if (unlikely(!pagecnt_bias)) { 1709 page_ref_add(page, USHRT_MAX); 1710 rx_buffer->pagecnt_bias = USHRT_MAX; 1711 } 1712 1713 return true; 1714 } 1715 1716 /** 1717 * igc_is_non_eop - process handling of non-EOP buffers 1718 * @rx_ring: Rx ring being processed 1719 * @rx_desc: Rx descriptor for current buffer 1720 * 1721 * This function updates next to clean. If the buffer is an EOP buffer 1722 * this function exits returning false, otherwise it will place the 1723 * sk_buff in the next buffer to be chained and return true indicating 1724 * that this is in fact a non-EOP buffer. 1725 */ 1726 static bool igc_is_non_eop(struct igc_ring *rx_ring, 1727 union igc_adv_rx_desc *rx_desc) 1728 { 1729 u32 ntc = rx_ring->next_to_clean + 1; 1730 1731 /* fetch, update, and store next to clean */ 1732 ntc = (ntc < rx_ring->count) ? ntc : 0; 1733 rx_ring->next_to_clean = ntc; 1734 1735 prefetch(IGC_RX_DESC(rx_ring, ntc)); 1736 1737 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 1738 return false; 1739 1740 return true; 1741 } 1742 1743 /** 1744 * igc_cleanup_headers - Correct corrupted or empty headers 1745 * @rx_ring: rx descriptor ring packet is being transacted on 1746 * @rx_desc: pointer to the EOP Rx descriptor 1747 * @skb: pointer to current skb being fixed 1748 * 1749 * Address the case where we are pulling data in on pages only 1750 * and as such no data is present in the skb header. 1751 * 1752 * In addition if skb is not at least 60 bytes we need to pad it so that 1753 * it is large enough to qualify as a valid Ethernet frame. 1754 * 1755 * Returns true if an error was encountered and skb was freed. 1756 */ 1757 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 1758 union igc_adv_rx_desc *rx_desc, 1759 struct sk_buff *skb) 1760 { 1761 if (unlikely((igc_test_staterr(rx_desc, 1762 IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) { 1763 struct net_device *netdev = rx_ring->netdev; 1764 1765 if (!(netdev->features & NETIF_F_RXALL)) { 1766 dev_kfree_skb_any(skb); 1767 return true; 1768 } 1769 } 1770 1771 /* if eth_skb_pad returns an error the skb was freed */ 1772 if (eth_skb_pad(skb)) 1773 return true; 1774 1775 return false; 1776 } 1777 1778 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 1779 struct igc_rx_buffer *rx_buffer) 1780 { 1781 if (igc_can_reuse_rx_page(rx_buffer)) { 1782 /* hand second half of page back to the ring */ 1783 igc_reuse_rx_page(rx_ring, rx_buffer); 1784 } else { 1785 /* We are not reusing the buffer so unmap it and free 1786 * any references we are holding to it 1787 */ 1788 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1789 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 1790 IGC_RX_DMA_ATTR); 1791 __page_frag_cache_drain(rx_buffer->page, 1792 rx_buffer->pagecnt_bias); 1793 } 1794 1795 /* clear contents of rx_buffer */ 1796 rx_buffer->page = NULL; 1797 } 1798 1799 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 1800 { 1801 return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; 1802 } 1803 1804 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 1805 struct igc_rx_buffer *bi) 1806 { 1807 struct page *page = bi->page; 1808 dma_addr_t dma; 1809 1810 /* since we are recycling buffers we should seldom need to alloc */ 1811 if (likely(page)) 1812 return true; 1813 1814 /* alloc new page for storage */ 1815 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 1816 if (unlikely(!page)) { 1817 rx_ring->rx_stats.alloc_failed++; 1818 return false; 1819 } 1820 1821 /* map page for use */ 1822 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1823 igc_rx_pg_size(rx_ring), 1824 DMA_FROM_DEVICE, 1825 IGC_RX_DMA_ATTR); 1826 1827 /* if mapping failed free memory back to system since 1828 * there isn't much point in holding memory we can't use 1829 */ 1830 if (dma_mapping_error(rx_ring->dev, dma)) { 1831 __free_page(page); 1832 1833 rx_ring->rx_stats.alloc_failed++; 1834 return false; 1835 } 1836 1837 bi->dma = dma; 1838 bi->page = page; 1839 bi->page_offset = igc_rx_offset(rx_ring); 1840 bi->pagecnt_bias = 1; 1841 1842 return true; 1843 } 1844 1845 /** 1846 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 1847 * @rx_ring: rx descriptor ring 1848 * @cleaned_count: number of buffers to clean 1849 */ 1850 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 1851 { 1852 union igc_adv_rx_desc *rx_desc; 1853 u16 i = rx_ring->next_to_use; 1854 struct igc_rx_buffer *bi; 1855 u16 bufsz; 1856 1857 /* nothing to do */ 1858 if (!cleaned_count) 1859 return; 1860 1861 rx_desc = IGC_RX_DESC(rx_ring, i); 1862 bi = &rx_ring->rx_buffer_info[i]; 1863 i -= rx_ring->count; 1864 1865 bufsz = igc_rx_bufsz(rx_ring); 1866 1867 do { 1868 if (!igc_alloc_mapped_page(rx_ring, bi)) 1869 break; 1870 1871 /* sync the buffer for use by the device */ 1872 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1873 bi->page_offset, bufsz, 1874 DMA_FROM_DEVICE); 1875 1876 /* Refresh the desc even if buffer_addrs didn't change 1877 * because each write-back erases this info. 1878 */ 1879 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1880 1881 rx_desc++; 1882 bi++; 1883 i++; 1884 if (unlikely(!i)) { 1885 rx_desc = IGC_RX_DESC(rx_ring, 0); 1886 bi = rx_ring->rx_buffer_info; 1887 i -= rx_ring->count; 1888 } 1889 1890 /* clear the length for the next_to_use descriptor */ 1891 rx_desc->wb.upper.length = 0; 1892 1893 cleaned_count--; 1894 } while (cleaned_count); 1895 1896 i += rx_ring->count; 1897 1898 if (rx_ring->next_to_use != i) { 1899 /* record the next descriptor to use */ 1900 rx_ring->next_to_use = i; 1901 1902 /* update next to alloc since we have filled the ring */ 1903 rx_ring->next_to_alloc = i; 1904 1905 /* Force memory writes to complete before letting h/w 1906 * know there are new descriptors to fetch. (Only 1907 * applicable for weak-ordered memory model archs, 1908 * such as IA-64). 1909 */ 1910 wmb(); 1911 writel(i, rx_ring->tail); 1912 } 1913 } 1914 1915 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 1916 { 1917 unsigned int total_bytes = 0, total_packets = 0; 1918 struct igc_ring *rx_ring = q_vector->rx.ring; 1919 struct sk_buff *skb = rx_ring->skb; 1920 u16 cleaned_count = igc_desc_unused(rx_ring); 1921 1922 while (likely(total_packets < budget)) { 1923 union igc_adv_rx_desc *rx_desc; 1924 struct igc_rx_buffer *rx_buffer; 1925 unsigned int size; 1926 1927 /* return some buffers to hardware, one at a time is too slow */ 1928 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 1929 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1930 cleaned_count = 0; 1931 } 1932 1933 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 1934 size = le16_to_cpu(rx_desc->wb.upper.length); 1935 if (!size) 1936 break; 1937 1938 /* This memory barrier is needed to keep us from reading 1939 * any other fields out of the rx_desc until we know the 1940 * descriptor has been written back 1941 */ 1942 dma_rmb(); 1943 1944 rx_buffer = igc_get_rx_buffer(rx_ring, size); 1945 1946 /* retrieve a buffer from the ring */ 1947 if (skb) 1948 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 1949 else if (ring_uses_build_skb(rx_ring)) 1950 skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); 1951 else 1952 skb = igc_construct_skb(rx_ring, rx_buffer, 1953 rx_desc, size); 1954 1955 /* exit if we failed to retrieve a buffer */ 1956 if (!skb) { 1957 rx_ring->rx_stats.alloc_failed++; 1958 rx_buffer->pagecnt_bias++; 1959 break; 1960 } 1961 1962 igc_put_rx_buffer(rx_ring, rx_buffer); 1963 cleaned_count++; 1964 1965 /* fetch next buffer in frame if non-eop */ 1966 if (igc_is_non_eop(rx_ring, rx_desc)) 1967 continue; 1968 1969 /* verify the packet layout is correct */ 1970 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) { 1971 skb = NULL; 1972 continue; 1973 } 1974 1975 /* probably a little skewed due to removing CRC */ 1976 total_bytes += skb->len; 1977 1978 /* populate checksum, timestamp, VLAN, and protocol */ 1979 igc_process_skb_fields(rx_ring, rx_desc, skb); 1980 1981 napi_gro_receive(&q_vector->napi, skb); 1982 1983 /* reset skb pointer */ 1984 skb = NULL; 1985 1986 /* update budget accounting */ 1987 total_packets++; 1988 } 1989 1990 /* place incomplete frames back on ring for completion */ 1991 rx_ring->skb = skb; 1992 1993 u64_stats_update_begin(&rx_ring->rx_syncp); 1994 rx_ring->rx_stats.packets += total_packets; 1995 rx_ring->rx_stats.bytes += total_bytes; 1996 u64_stats_update_end(&rx_ring->rx_syncp); 1997 q_vector->rx.total_packets += total_packets; 1998 q_vector->rx.total_bytes += total_bytes; 1999 2000 if (cleaned_count) 2001 igc_alloc_rx_buffers(rx_ring, cleaned_count); 2002 2003 return total_packets; 2004 } 2005 2006 /** 2007 * igc_clean_tx_irq - Reclaim resources after transmit completes 2008 * @q_vector: pointer to q_vector containing needed info 2009 * @napi_budget: Used to determine if we are in netpoll 2010 * 2011 * returns true if ring is completely cleaned 2012 */ 2013 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 2014 { 2015 struct igc_adapter *adapter = q_vector->adapter; 2016 unsigned int total_bytes = 0, total_packets = 0; 2017 unsigned int budget = q_vector->tx.work_limit; 2018 struct igc_ring *tx_ring = q_vector->tx.ring; 2019 unsigned int i = tx_ring->next_to_clean; 2020 struct igc_tx_buffer *tx_buffer; 2021 union igc_adv_tx_desc *tx_desc; 2022 2023 if (test_bit(__IGC_DOWN, &adapter->state)) 2024 return true; 2025 2026 tx_buffer = &tx_ring->tx_buffer_info[i]; 2027 tx_desc = IGC_TX_DESC(tx_ring, i); 2028 i -= tx_ring->count; 2029 2030 do { 2031 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 2032 2033 /* if next_to_watch is not set then there is no work pending */ 2034 if (!eop_desc) 2035 break; 2036 2037 /* prevent any other reads prior to eop_desc */ 2038 smp_rmb(); 2039 2040 /* if DD is not set pending work has not been completed */ 2041 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 2042 break; 2043 2044 /* clear next_to_watch to prevent false hangs */ 2045 tx_buffer->next_to_watch = NULL; 2046 2047 /* update the statistics for this packet */ 2048 total_bytes += tx_buffer->bytecount; 2049 total_packets += tx_buffer->gso_segs; 2050 2051 /* free the skb */ 2052 napi_consume_skb(tx_buffer->skb, napi_budget); 2053 2054 /* unmap skb header data */ 2055 dma_unmap_single(tx_ring->dev, 2056 dma_unmap_addr(tx_buffer, dma), 2057 dma_unmap_len(tx_buffer, len), 2058 DMA_TO_DEVICE); 2059 2060 /* clear tx_buffer data */ 2061 dma_unmap_len_set(tx_buffer, len, 0); 2062 2063 /* clear last DMA location and unmap remaining buffers */ 2064 while (tx_desc != eop_desc) { 2065 tx_buffer++; 2066 tx_desc++; 2067 i++; 2068 if (unlikely(!i)) { 2069 i -= tx_ring->count; 2070 tx_buffer = tx_ring->tx_buffer_info; 2071 tx_desc = IGC_TX_DESC(tx_ring, 0); 2072 } 2073 2074 /* unmap any remaining paged data */ 2075 if (dma_unmap_len(tx_buffer, len)) { 2076 dma_unmap_page(tx_ring->dev, 2077 dma_unmap_addr(tx_buffer, dma), 2078 dma_unmap_len(tx_buffer, len), 2079 DMA_TO_DEVICE); 2080 dma_unmap_len_set(tx_buffer, len, 0); 2081 } 2082 } 2083 2084 /* move us one more past the eop_desc for start of next pkt */ 2085 tx_buffer++; 2086 tx_desc++; 2087 i++; 2088 if (unlikely(!i)) { 2089 i -= tx_ring->count; 2090 tx_buffer = tx_ring->tx_buffer_info; 2091 tx_desc = IGC_TX_DESC(tx_ring, 0); 2092 } 2093 2094 /* issue prefetch for next Tx descriptor */ 2095 prefetch(tx_desc); 2096 2097 /* update budget accounting */ 2098 budget--; 2099 } while (likely(budget)); 2100 2101 netdev_tx_completed_queue(txring_txq(tx_ring), 2102 total_packets, total_bytes); 2103 2104 i += tx_ring->count; 2105 tx_ring->next_to_clean = i; 2106 u64_stats_update_begin(&tx_ring->tx_syncp); 2107 tx_ring->tx_stats.bytes += total_bytes; 2108 tx_ring->tx_stats.packets += total_packets; 2109 u64_stats_update_end(&tx_ring->tx_syncp); 2110 q_vector->tx.total_bytes += total_bytes; 2111 q_vector->tx.total_packets += total_packets; 2112 2113 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 2114 struct igc_hw *hw = &adapter->hw; 2115 2116 /* Detect a transmit hang in hardware, this serializes the 2117 * check with the clearing of time_stamp and movement of i 2118 */ 2119 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 2120 if (tx_buffer->next_to_watch && 2121 time_after(jiffies, tx_buffer->time_stamp + 2122 (adapter->tx_timeout_factor * HZ)) && 2123 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { 2124 /* detected Tx unit hang */ 2125 netdev_err(tx_ring->netdev, 2126 "Detected Tx Unit Hang\n" 2127 " Tx Queue <%d>\n" 2128 " TDH <%x>\n" 2129 " TDT <%x>\n" 2130 " next_to_use <%x>\n" 2131 " next_to_clean <%x>\n" 2132 "buffer_info[next_to_clean]\n" 2133 " time_stamp <%lx>\n" 2134 " next_to_watch <%p>\n" 2135 " jiffies <%lx>\n" 2136 " desc.status <%x>\n", 2137 tx_ring->queue_index, 2138 rd32(IGC_TDH(tx_ring->reg_idx)), 2139 readl(tx_ring->tail), 2140 tx_ring->next_to_use, 2141 tx_ring->next_to_clean, 2142 tx_buffer->time_stamp, 2143 tx_buffer->next_to_watch, 2144 jiffies, 2145 tx_buffer->next_to_watch->wb.status); 2146 netif_stop_subqueue(tx_ring->netdev, 2147 tx_ring->queue_index); 2148 2149 /* we are about to reset, no point in enabling stuff */ 2150 return true; 2151 } 2152 } 2153 2154 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 2155 if (unlikely(total_packets && 2156 netif_carrier_ok(tx_ring->netdev) && 2157 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 2158 /* Make sure that anybody stopping the queue after this 2159 * sees the new next_to_clean. 2160 */ 2161 smp_mb(); 2162 if (__netif_subqueue_stopped(tx_ring->netdev, 2163 tx_ring->queue_index) && 2164 !(test_bit(__IGC_DOWN, &adapter->state))) { 2165 netif_wake_subqueue(tx_ring->netdev, 2166 tx_ring->queue_index); 2167 2168 u64_stats_update_begin(&tx_ring->tx_syncp); 2169 tx_ring->tx_stats.restart_queue++; 2170 u64_stats_update_end(&tx_ring->tx_syncp); 2171 } 2172 } 2173 2174 return !!budget; 2175 } 2176 2177 static int igc_find_mac_filter(struct igc_adapter *adapter, 2178 enum igc_mac_filter_type type, const u8 *addr) 2179 { 2180 struct igc_hw *hw = &adapter->hw; 2181 int max_entries = hw->mac.rar_entry_count; 2182 u32 ral, rah; 2183 int i; 2184 2185 for (i = 0; i < max_entries; i++) { 2186 ral = rd32(IGC_RAL(i)); 2187 rah = rd32(IGC_RAH(i)); 2188 2189 if (!(rah & IGC_RAH_AV)) 2190 continue; 2191 if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type) 2192 continue; 2193 if ((rah & IGC_RAH_RAH_MASK) != 2194 le16_to_cpup((__le16 *)(addr + 4))) 2195 continue; 2196 if (ral != le32_to_cpup((__le32 *)(addr))) 2197 continue; 2198 2199 return i; 2200 } 2201 2202 return -1; 2203 } 2204 2205 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) 2206 { 2207 struct igc_hw *hw = &adapter->hw; 2208 int max_entries = hw->mac.rar_entry_count; 2209 u32 rah; 2210 int i; 2211 2212 for (i = 0; i < max_entries; i++) { 2213 rah = rd32(IGC_RAH(i)); 2214 2215 if (!(rah & IGC_RAH_AV)) 2216 return i; 2217 } 2218 2219 return -1; 2220 } 2221 2222 /** 2223 * igc_add_mac_filter() - Add MAC address filter 2224 * @adapter: Pointer to adapter where the filter should be added 2225 * @type: MAC address filter type (source or destination) 2226 * @addr: MAC address 2227 * @queue: If non-negative, queue assignment feature is enabled and frames 2228 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2229 * assignment is disabled. 2230 * 2231 * Return: 0 in case of success, negative errno code otherwise. 2232 */ 2233 static int igc_add_mac_filter(struct igc_adapter *adapter, 2234 enum igc_mac_filter_type type, const u8 *addr, 2235 int queue) 2236 { 2237 struct net_device *dev = adapter->netdev; 2238 int index; 2239 2240 index = igc_find_mac_filter(adapter, type, addr); 2241 if (index >= 0) 2242 goto update_filter; 2243 2244 index = igc_get_avail_mac_filter_slot(adapter); 2245 if (index < 0) 2246 return -ENOSPC; 2247 2248 netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n", 2249 index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2250 addr, queue); 2251 2252 update_filter: 2253 igc_set_mac_filter_hw(adapter, index, type, addr, queue); 2254 return 0; 2255 } 2256 2257 /** 2258 * igc_del_mac_filter() - Delete MAC address filter 2259 * @adapter: Pointer to adapter where the filter should be deleted from 2260 * @type: MAC address filter type (source or destination) 2261 * @addr: MAC address 2262 */ 2263 static void igc_del_mac_filter(struct igc_adapter *adapter, 2264 enum igc_mac_filter_type type, const u8 *addr) 2265 { 2266 struct net_device *dev = adapter->netdev; 2267 int index; 2268 2269 index = igc_find_mac_filter(adapter, type, addr); 2270 if (index < 0) 2271 return; 2272 2273 if (index == 0) { 2274 /* If this is the default filter, we don't actually delete it. 2275 * We just reset to its default value i.e. disable queue 2276 * assignment. 2277 */ 2278 netdev_dbg(dev, "Disable default MAC filter queue assignment"); 2279 2280 igc_set_mac_filter_hw(adapter, 0, type, addr, -1); 2281 } else { 2282 netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n", 2283 index, 2284 type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2285 addr); 2286 2287 igc_clear_mac_filter_hw(adapter, index); 2288 } 2289 } 2290 2291 /** 2292 * igc_add_vlan_prio_filter() - Add VLAN priority filter 2293 * @adapter: Pointer to adapter where the filter should be added 2294 * @prio: VLAN priority value 2295 * @queue: Queue number which matching frames are assigned to 2296 * 2297 * Return: 0 in case of success, negative errno code otherwise. 2298 */ 2299 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, 2300 int queue) 2301 { 2302 struct net_device *dev = adapter->netdev; 2303 struct igc_hw *hw = &adapter->hw; 2304 u32 vlanpqf; 2305 2306 vlanpqf = rd32(IGC_VLANPQF); 2307 2308 if (vlanpqf & IGC_VLANPQF_VALID(prio)) { 2309 netdev_dbg(dev, "VLAN priority filter already in use\n"); 2310 return -EEXIST; 2311 } 2312 2313 vlanpqf |= IGC_VLANPQF_QSEL(prio, queue); 2314 vlanpqf |= IGC_VLANPQF_VALID(prio); 2315 2316 wr32(IGC_VLANPQF, vlanpqf); 2317 2318 netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n", 2319 prio, queue); 2320 return 0; 2321 } 2322 2323 /** 2324 * igc_del_vlan_prio_filter() - Delete VLAN priority filter 2325 * @adapter: Pointer to adapter where the filter should be deleted from 2326 * @prio: VLAN priority value 2327 */ 2328 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) 2329 { 2330 struct igc_hw *hw = &adapter->hw; 2331 u32 vlanpqf; 2332 2333 vlanpqf = rd32(IGC_VLANPQF); 2334 2335 vlanpqf &= ~IGC_VLANPQF_VALID(prio); 2336 vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK); 2337 2338 wr32(IGC_VLANPQF, vlanpqf); 2339 2340 netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n", 2341 prio); 2342 } 2343 2344 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) 2345 { 2346 struct igc_hw *hw = &adapter->hw; 2347 int i; 2348 2349 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2350 u32 etqf = rd32(IGC_ETQF(i)); 2351 2352 if (!(etqf & IGC_ETQF_FILTER_ENABLE)) 2353 return i; 2354 } 2355 2356 return -1; 2357 } 2358 2359 /** 2360 * igc_add_etype_filter() - Add ethertype filter 2361 * @adapter: Pointer to adapter where the filter should be added 2362 * @etype: Ethertype value 2363 * @queue: If non-negative, queue assignment feature is enabled and frames 2364 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2365 * assignment is disabled. 2366 * 2367 * Return: 0 in case of success, negative errno code otherwise. 2368 */ 2369 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, 2370 int queue) 2371 { 2372 struct igc_hw *hw = &adapter->hw; 2373 int index; 2374 u32 etqf; 2375 2376 index = igc_get_avail_etype_filter_slot(adapter); 2377 if (index < 0) 2378 return -ENOSPC; 2379 2380 etqf = rd32(IGC_ETQF(index)); 2381 2382 etqf &= ~IGC_ETQF_ETYPE_MASK; 2383 etqf |= etype; 2384 2385 if (queue >= 0) { 2386 etqf &= ~IGC_ETQF_QUEUE_MASK; 2387 etqf |= (queue << IGC_ETQF_QUEUE_SHIFT); 2388 etqf |= IGC_ETQF_QUEUE_ENABLE; 2389 } 2390 2391 etqf |= IGC_ETQF_FILTER_ENABLE; 2392 2393 wr32(IGC_ETQF(index), etqf); 2394 2395 netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n", 2396 etype, queue); 2397 return 0; 2398 } 2399 2400 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) 2401 { 2402 struct igc_hw *hw = &adapter->hw; 2403 int i; 2404 2405 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2406 u32 etqf = rd32(IGC_ETQF(i)); 2407 2408 if ((etqf & IGC_ETQF_ETYPE_MASK) == etype) 2409 return i; 2410 } 2411 2412 return -1; 2413 } 2414 2415 /** 2416 * igc_del_etype_filter() - Delete ethertype filter 2417 * @adapter: Pointer to adapter where the filter should be deleted from 2418 * @etype: Ethertype value 2419 */ 2420 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) 2421 { 2422 struct igc_hw *hw = &adapter->hw; 2423 int index; 2424 2425 index = igc_find_etype_filter(adapter, etype); 2426 if (index < 0) 2427 return; 2428 2429 wr32(IGC_ETQF(index), 0); 2430 2431 netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n", 2432 etype); 2433 } 2434 2435 static int igc_enable_nfc_rule(struct igc_adapter *adapter, 2436 const struct igc_nfc_rule *rule) 2437 { 2438 int err; 2439 2440 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { 2441 err = igc_add_etype_filter(adapter, rule->filter.etype, 2442 rule->action); 2443 if (err) 2444 return err; 2445 } 2446 2447 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { 2448 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2449 rule->filter.src_addr, rule->action); 2450 if (err) 2451 return err; 2452 } 2453 2454 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { 2455 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2456 rule->filter.dst_addr, rule->action); 2457 if (err) 2458 return err; 2459 } 2460 2461 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2462 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2463 VLAN_PRIO_SHIFT; 2464 2465 err = igc_add_vlan_prio_filter(adapter, prio, rule->action); 2466 if (err) 2467 return err; 2468 } 2469 2470 return 0; 2471 } 2472 2473 static void igc_disable_nfc_rule(struct igc_adapter *adapter, 2474 const struct igc_nfc_rule *rule) 2475 { 2476 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) 2477 igc_del_etype_filter(adapter, rule->filter.etype); 2478 2479 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2480 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2481 VLAN_PRIO_SHIFT; 2482 2483 igc_del_vlan_prio_filter(adapter, prio); 2484 } 2485 2486 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) 2487 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2488 rule->filter.src_addr); 2489 2490 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) 2491 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2492 rule->filter.dst_addr); 2493 } 2494 2495 /** 2496 * igc_get_nfc_rule() - Get NFC rule 2497 * @adapter: Pointer to adapter 2498 * @location: Rule location 2499 * 2500 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2501 * 2502 * Return: Pointer to NFC rule at @location. If not found, NULL. 2503 */ 2504 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, 2505 u32 location) 2506 { 2507 struct igc_nfc_rule *rule; 2508 2509 list_for_each_entry(rule, &adapter->nfc_rule_list, list) { 2510 if (rule->location == location) 2511 return rule; 2512 if (rule->location > location) 2513 break; 2514 } 2515 2516 return NULL; 2517 } 2518 2519 /** 2520 * igc_del_nfc_rule() - Delete NFC rule 2521 * @adapter: Pointer to adapter 2522 * @rule: Pointer to rule to be deleted 2523 * 2524 * Disable NFC rule in hardware and delete it from adapter. 2525 * 2526 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2527 */ 2528 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2529 { 2530 igc_disable_nfc_rule(adapter, rule); 2531 2532 list_del(&rule->list); 2533 adapter->nfc_rule_count--; 2534 2535 kfree(rule); 2536 } 2537 2538 static void igc_flush_nfc_rules(struct igc_adapter *adapter) 2539 { 2540 struct igc_nfc_rule *rule, *tmp; 2541 2542 mutex_lock(&adapter->nfc_rule_lock); 2543 2544 list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list) 2545 igc_del_nfc_rule(adapter, rule); 2546 2547 mutex_unlock(&adapter->nfc_rule_lock); 2548 } 2549 2550 /** 2551 * igc_add_nfc_rule() - Add NFC rule 2552 * @adapter: Pointer to adapter 2553 * @rule: Pointer to rule to be added 2554 * 2555 * Enable NFC rule in hardware and add it to adapter. 2556 * 2557 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2558 * 2559 * Return: 0 on success, negative errno on failure. 2560 */ 2561 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2562 { 2563 struct igc_nfc_rule *pred, *cur; 2564 int err; 2565 2566 err = igc_enable_nfc_rule(adapter, rule); 2567 if (err) 2568 return err; 2569 2570 pred = NULL; 2571 list_for_each_entry(cur, &adapter->nfc_rule_list, list) { 2572 if (cur->location >= rule->location) 2573 break; 2574 pred = cur; 2575 } 2576 2577 list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list); 2578 adapter->nfc_rule_count++; 2579 return 0; 2580 } 2581 2582 static void igc_restore_nfc_rules(struct igc_adapter *adapter) 2583 { 2584 struct igc_nfc_rule *rule; 2585 2586 mutex_lock(&adapter->nfc_rule_lock); 2587 2588 list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list) 2589 igc_enable_nfc_rule(adapter, rule); 2590 2591 mutex_unlock(&adapter->nfc_rule_lock); 2592 } 2593 2594 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) 2595 { 2596 struct igc_adapter *adapter = netdev_priv(netdev); 2597 2598 return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1); 2599 } 2600 2601 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) 2602 { 2603 struct igc_adapter *adapter = netdev_priv(netdev); 2604 2605 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr); 2606 return 0; 2607 } 2608 2609 /** 2610 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 2611 * @netdev: network interface device structure 2612 * 2613 * The set_rx_mode entry point is called whenever the unicast or multicast 2614 * address lists or the network interface flags are updated. This routine is 2615 * responsible for configuring the hardware for proper unicast, multicast, 2616 * promiscuous mode, and all-multi behavior. 2617 */ 2618 static void igc_set_rx_mode(struct net_device *netdev) 2619 { 2620 struct igc_adapter *adapter = netdev_priv(netdev); 2621 struct igc_hw *hw = &adapter->hw; 2622 u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 2623 int count; 2624 2625 /* Check for Promiscuous and All Multicast modes */ 2626 if (netdev->flags & IFF_PROMISC) { 2627 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; 2628 } else { 2629 if (netdev->flags & IFF_ALLMULTI) { 2630 rctl |= IGC_RCTL_MPE; 2631 } else { 2632 /* Write addresses to the MTA, if the attempt fails 2633 * then we should just turn on promiscuous mode so 2634 * that we can at least receive multicast traffic 2635 */ 2636 count = igc_write_mc_addr_list(netdev); 2637 if (count < 0) 2638 rctl |= IGC_RCTL_MPE; 2639 } 2640 } 2641 2642 /* Write addresses to available RAR registers, if there is not 2643 * sufficient space to store all the addresses then enable 2644 * unicast promiscuous mode 2645 */ 2646 if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) 2647 rctl |= IGC_RCTL_UPE; 2648 2649 /* update state of unicast and multicast */ 2650 rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 2651 wr32(IGC_RCTL, rctl); 2652 2653 #if (PAGE_SIZE < 8192) 2654 if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) 2655 rlpml = IGC_MAX_FRAME_BUILD_SKB; 2656 #endif 2657 wr32(IGC_RLPML, rlpml); 2658 } 2659 2660 /** 2661 * igc_configure - configure the hardware for RX and TX 2662 * @adapter: private board structure 2663 */ 2664 static void igc_configure(struct igc_adapter *adapter) 2665 { 2666 struct net_device *netdev = adapter->netdev; 2667 int i = 0; 2668 2669 igc_get_hw_control(adapter); 2670 igc_set_rx_mode(netdev); 2671 2672 igc_setup_tctl(adapter); 2673 igc_setup_mrqc(adapter); 2674 igc_setup_rctl(adapter); 2675 2676 igc_set_default_mac_filter(adapter); 2677 igc_restore_nfc_rules(adapter); 2678 2679 igc_configure_tx(adapter); 2680 igc_configure_rx(adapter); 2681 2682 igc_rx_fifo_flush_base(&adapter->hw); 2683 2684 /* call igc_desc_unused which always leaves 2685 * at least 1 descriptor unused to make sure 2686 * next_to_use != next_to_clean 2687 */ 2688 for (i = 0; i < adapter->num_rx_queues; i++) { 2689 struct igc_ring *ring = adapter->rx_ring[i]; 2690 2691 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 2692 } 2693 } 2694 2695 /** 2696 * igc_write_ivar - configure ivar for given MSI-X vector 2697 * @hw: pointer to the HW structure 2698 * @msix_vector: vector number we are allocating to a given ring 2699 * @index: row index of IVAR register to write within IVAR table 2700 * @offset: column offset of in IVAR, should be multiple of 8 2701 * 2702 * The IVAR table consists of 2 columns, 2703 * each containing an cause allocation for an Rx and Tx ring, and a 2704 * variable number of rows depending on the number of queues supported. 2705 */ 2706 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 2707 int index, int offset) 2708 { 2709 u32 ivar = array_rd32(IGC_IVAR0, index); 2710 2711 /* clear any bits that are currently set */ 2712 ivar &= ~((u32)0xFF << offset); 2713 2714 /* write vector and valid bit */ 2715 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 2716 2717 array_wr32(IGC_IVAR0, index, ivar); 2718 } 2719 2720 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 2721 { 2722 struct igc_adapter *adapter = q_vector->adapter; 2723 struct igc_hw *hw = &adapter->hw; 2724 int rx_queue = IGC_N0_QUEUE; 2725 int tx_queue = IGC_N0_QUEUE; 2726 2727 if (q_vector->rx.ring) 2728 rx_queue = q_vector->rx.ring->reg_idx; 2729 if (q_vector->tx.ring) 2730 tx_queue = q_vector->tx.ring->reg_idx; 2731 2732 switch (hw->mac.type) { 2733 case igc_i225: 2734 if (rx_queue > IGC_N0_QUEUE) 2735 igc_write_ivar(hw, msix_vector, 2736 rx_queue >> 1, 2737 (rx_queue & 0x1) << 4); 2738 if (tx_queue > IGC_N0_QUEUE) 2739 igc_write_ivar(hw, msix_vector, 2740 tx_queue >> 1, 2741 ((tx_queue & 0x1) << 4) + 8); 2742 q_vector->eims_value = BIT(msix_vector); 2743 break; 2744 default: 2745 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 2746 break; 2747 } 2748 2749 /* add q_vector eims value to global eims_enable_mask */ 2750 adapter->eims_enable_mask |= q_vector->eims_value; 2751 2752 /* configure q_vector to set itr on first interrupt */ 2753 q_vector->set_itr = 1; 2754 } 2755 2756 /** 2757 * igc_configure_msix - Configure MSI-X hardware 2758 * @adapter: Pointer to adapter structure 2759 * 2760 * igc_configure_msix sets up the hardware to properly 2761 * generate MSI-X interrupts. 2762 */ 2763 static void igc_configure_msix(struct igc_adapter *adapter) 2764 { 2765 struct igc_hw *hw = &adapter->hw; 2766 int i, vector = 0; 2767 u32 tmp; 2768 2769 adapter->eims_enable_mask = 0; 2770 2771 /* set vector for other causes, i.e. link changes */ 2772 switch (hw->mac.type) { 2773 case igc_i225: 2774 /* Turn on MSI-X capability first, or our settings 2775 * won't stick. And it will take days to debug. 2776 */ 2777 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 2778 IGC_GPIE_PBA | IGC_GPIE_EIAME | 2779 IGC_GPIE_NSICR); 2780 2781 /* enable msix_other interrupt */ 2782 adapter->eims_other = BIT(vector); 2783 tmp = (vector++ | IGC_IVAR_VALID) << 8; 2784 2785 wr32(IGC_IVAR_MISC, tmp); 2786 break; 2787 default: 2788 /* do nothing, since nothing else supports MSI-X */ 2789 break; 2790 } /* switch (hw->mac.type) */ 2791 2792 adapter->eims_enable_mask |= adapter->eims_other; 2793 2794 for (i = 0; i < adapter->num_q_vectors; i++) 2795 igc_assign_vector(adapter->q_vector[i], vector++); 2796 2797 wrfl(); 2798 } 2799 2800 /** 2801 * igc_irq_enable - Enable default interrupt generation settings 2802 * @adapter: board private structure 2803 */ 2804 static void igc_irq_enable(struct igc_adapter *adapter) 2805 { 2806 struct igc_hw *hw = &adapter->hw; 2807 2808 if (adapter->msix_entries) { 2809 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 2810 u32 regval = rd32(IGC_EIAC); 2811 2812 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 2813 regval = rd32(IGC_EIAM); 2814 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 2815 wr32(IGC_EIMS, adapter->eims_enable_mask); 2816 wr32(IGC_IMS, ims); 2817 } else { 2818 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2819 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2820 } 2821 } 2822 2823 /** 2824 * igc_irq_disable - Mask off interrupt generation on the NIC 2825 * @adapter: board private structure 2826 */ 2827 static void igc_irq_disable(struct igc_adapter *adapter) 2828 { 2829 struct igc_hw *hw = &adapter->hw; 2830 2831 if (adapter->msix_entries) { 2832 u32 regval = rd32(IGC_EIAM); 2833 2834 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 2835 wr32(IGC_EIMC, adapter->eims_enable_mask); 2836 regval = rd32(IGC_EIAC); 2837 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 2838 } 2839 2840 wr32(IGC_IAM, 0); 2841 wr32(IGC_IMC, ~0); 2842 wrfl(); 2843 2844 if (adapter->msix_entries) { 2845 int vector = 0, i; 2846 2847 synchronize_irq(adapter->msix_entries[vector++].vector); 2848 2849 for (i = 0; i < adapter->num_q_vectors; i++) 2850 synchronize_irq(adapter->msix_entries[vector++].vector); 2851 } else { 2852 synchronize_irq(adapter->pdev->irq); 2853 } 2854 } 2855 2856 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 2857 const u32 max_rss_queues) 2858 { 2859 /* Determine if we need to pair queues. */ 2860 /* If rss_queues > half of max_rss_queues, pair the queues in 2861 * order to conserve interrupts due to limited supply. 2862 */ 2863 if (adapter->rss_queues > (max_rss_queues / 2)) 2864 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 2865 else 2866 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 2867 } 2868 2869 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 2870 { 2871 return IGC_MAX_RX_QUEUES; 2872 } 2873 2874 static void igc_init_queue_configuration(struct igc_adapter *adapter) 2875 { 2876 u32 max_rss_queues; 2877 2878 max_rss_queues = igc_get_max_rss_queues(adapter); 2879 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 2880 2881 igc_set_flag_queue_pairs(adapter, max_rss_queues); 2882 } 2883 2884 /** 2885 * igc_reset_q_vector - Reset config for interrupt vector 2886 * @adapter: board private structure to initialize 2887 * @v_idx: Index of vector to be reset 2888 * 2889 * If NAPI is enabled it will delete any references to the 2890 * NAPI struct. This is preparation for igc_free_q_vector. 2891 */ 2892 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 2893 { 2894 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2895 2896 /* if we're coming from igc_set_interrupt_capability, the vectors are 2897 * not yet allocated 2898 */ 2899 if (!q_vector) 2900 return; 2901 2902 if (q_vector->tx.ring) 2903 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 2904 2905 if (q_vector->rx.ring) 2906 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 2907 2908 netif_napi_del(&q_vector->napi); 2909 } 2910 2911 /** 2912 * igc_free_q_vector - Free memory allocated for specific interrupt vector 2913 * @adapter: board private structure to initialize 2914 * @v_idx: Index of vector to be freed 2915 * 2916 * This function frees the memory allocated to the q_vector. 2917 */ 2918 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 2919 { 2920 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2921 2922 adapter->q_vector[v_idx] = NULL; 2923 2924 /* igc_get_stats64() might access the rings on this vector, 2925 * we must wait a grace period before freeing it. 2926 */ 2927 if (q_vector) 2928 kfree_rcu(q_vector, rcu); 2929 } 2930 2931 /** 2932 * igc_free_q_vectors - Free memory allocated for interrupt vectors 2933 * @adapter: board private structure to initialize 2934 * 2935 * This function frees the memory allocated to the q_vectors. In addition if 2936 * NAPI is enabled it will delete any references to the NAPI struct prior 2937 * to freeing the q_vector. 2938 */ 2939 static void igc_free_q_vectors(struct igc_adapter *adapter) 2940 { 2941 int v_idx = adapter->num_q_vectors; 2942 2943 adapter->num_tx_queues = 0; 2944 adapter->num_rx_queues = 0; 2945 adapter->num_q_vectors = 0; 2946 2947 while (v_idx--) { 2948 igc_reset_q_vector(adapter, v_idx); 2949 igc_free_q_vector(adapter, v_idx); 2950 } 2951 } 2952 2953 /** 2954 * igc_update_itr - update the dynamic ITR value based on statistics 2955 * @q_vector: pointer to q_vector 2956 * @ring_container: ring info to update the itr for 2957 * 2958 * Stores a new ITR value based on packets and byte 2959 * counts during the last interrupt. The advantage of per interrupt 2960 * computation is faster updates and more accurate ITR for the current 2961 * traffic pattern. Constants in this function were computed 2962 * based on theoretical maximum wire speed and thresholds were set based 2963 * on testing data as well as attempting to minimize response time 2964 * while increasing bulk throughput. 2965 * NOTE: These calculations are only valid when operating in a single- 2966 * queue environment. 2967 */ 2968 static void igc_update_itr(struct igc_q_vector *q_vector, 2969 struct igc_ring_container *ring_container) 2970 { 2971 unsigned int packets = ring_container->total_packets; 2972 unsigned int bytes = ring_container->total_bytes; 2973 u8 itrval = ring_container->itr; 2974 2975 /* no packets, exit with status unchanged */ 2976 if (packets == 0) 2977 return; 2978 2979 switch (itrval) { 2980 case lowest_latency: 2981 /* handle TSO and jumbo frames */ 2982 if (bytes / packets > 8000) 2983 itrval = bulk_latency; 2984 else if ((packets < 5) && (bytes > 512)) 2985 itrval = low_latency; 2986 break; 2987 case low_latency: /* 50 usec aka 20000 ints/s */ 2988 if (bytes > 10000) { 2989 /* this if handles the TSO accounting */ 2990 if (bytes / packets > 8000) 2991 itrval = bulk_latency; 2992 else if ((packets < 10) || ((bytes / packets) > 1200)) 2993 itrval = bulk_latency; 2994 else if ((packets > 35)) 2995 itrval = lowest_latency; 2996 } else if (bytes / packets > 2000) { 2997 itrval = bulk_latency; 2998 } else if (packets <= 2 && bytes < 512) { 2999 itrval = lowest_latency; 3000 } 3001 break; 3002 case bulk_latency: /* 250 usec aka 4000 ints/s */ 3003 if (bytes > 25000) { 3004 if (packets > 35) 3005 itrval = low_latency; 3006 } else if (bytes < 1500) { 3007 itrval = low_latency; 3008 } 3009 break; 3010 } 3011 3012 /* clear work counters since we have the values we need */ 3013 ring_container->total_bytes = 0; 3014 ring_container->total_packets = 0; 3015 3016 /* write updated itr to ring container */ 3017 ring_container->itr = itrval; 3018 } 3019 3020 static void igc_set_itr(struct igc_q_vector *q_vector) 3021 { 3022 struct igc_adapter *adapter = q_vector->adapter; 3023 u32 new_itr = q_vector->itr_val; 3024 u8 current_itr = 0; 3025 3026 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 3027 switch (adapter->link_speed) { 3028 case SPEED_10: 3029 case SPEED_100: 3030 current_itr = 0; 3031 new_itr = IGC_4K_ITR; 3032 goto set_itr_now; 3033 default: 3034 break; 3035 } 3036 3037 igc_update_itr(q_vector, &q_vector->tx); 3038 igc_update_itr(q_vector, &q_vector->rx); 3039 3040 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 3041 3042 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3043 if (current_itr == lowest_latency && 3044 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3045 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3046 current_itr = low_latency; 3047 3048 switch (current_itr) { 3049 /* counts and packets in update_itr are dependent on these numbers */ 3050 case lowest_latency: 3051 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 3052 break; 3053 case low_latency: 3054 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 3055 break; 3056 case bulk_latency: 3057 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 3058 break; 3059 default: 3060 break; 3061 } 3062 3063 set_itr_now: 3064 if (new_itr != q_vector->itr_val) { 3065 /* this attempts to bias the interrupt rate towards Bulk 3066 * by adding intermediate steps when interrupt rate is 3067 * increasing 3068 */ 3069 new_itr = new_itr > q_vector->itr_val ? 3070 max((new_itr * q_vector->itr_val) / 3071 (new_itr + (q_vector->itr_val >> 2)), 3072 new_itr) : new_itr; 3073 /* Don't write the value here; it resets the adapter's 3074 * internal timer, and causes us to delay far longer than 3075 * we should between interrupts. Instead, we write the ITR 3076 * value at the beginning of the next interrupt so the timing 3077 * ends up being correct. 3078 */ 3079 q_vector->itr_val = new_itr; 3080 q_vector->set_itr = 1; 3081 } 3082 } 3083 3084 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 3085 { 3086 int v_idx = adapter->num_q_vectors; 3087 3088 if (adapter->msix_entries) { 3089 pci_disable_msix(adapter->pdev); 3090 kfree(adapter->msix_entries); 3091 adapter->msix_entries = NULL; 3092 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 3093 pci_disable_msi(adapter->pdev); 3094 } 3095 3096 while (v_idx--) 3097 igc_reset_q_vector(adapter, v_idx); 3098 } 3099 3100 /** 3101 * igc_set_interrupt_capability - set MSI or MSI-X if supported 3102 * @adapter: Pointer to adapter structure 3103 * @msix: boolean value for MSI-X capability 3104 * 3105 * Attempt to configure interrupts using the best available 3106 * capabilities of the hardware and kernel. 3107 */ 3108 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 3109 bool msix) 3110 { 3111 int numvecs, i; 3112 int err; 3113 3114 if (!msix) 3115 goto msi_only; 3116 adapter->flags |= IGC_FLAG_HAS_MSIX; 3117 3118 /* Number of supported queues. */ 3119 adapter->num_rx_queues = adapter->rss_queues; 3120 3121 adapter->num_tx_queues = adapter->rss_queues; 3122 3123 /* start with one vector for every Rx queue */ 3124 numvecs = adapter->num_rx_queues; 3125 3126 /* if Tx handler is separate add 1 for every Tx queue */ 3127 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 3128 numvecs += adapter->num_tx_queues; 3129 3130 /* store the number of vectors reserved for queues */ 3131 adapter->num_q_vectors = numvecs; 3132 3133 /* add 1 vector for link status interrupts */ 3134 numvecs++; 3135 3136 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), 3137 GFP_KERNEL); 3138 3139 if (!adapter->msix_entries) 3140 return; 3141 3142 /* populate entry values */ 3143 for (i = 0; i < numvecs; i++) 3144 adapter->msix_entries[i].entry = i; 3145 3146 err = pci_enable_msix_range(adapter->pdev, 3147 adapter->msix_entries, 3148 numvecs, 3149 numvecs); 3150 if (err > 0) 3151 return; 3152 3153 kfree(adapter->msix_entries); 3154 adapter->msix_entries = NULL; 3155 3156 igc_reset_interrupt_capability(adapter); 3157 3158 msi_only: 3159 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 3160 3161 adapter->rss_queues = 1; 3162 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 3163 adapter->num_rx_queues = 1; 3164 adapter->num_tx_queues = 1; 3165 adapter->num_q_vectors = 1; 3166 if (!pci_enable_msi(adapter->pdev)) 3167 adapter->flags |= IGC_FLAG_HAS_MSI; 3168 } 3169 3170 /** 3171 * igc_update_ring_itr - update the dynamic ITR value based on packet size 3172 * @q_vector: pointer to q_vector 3173 * 3174 * Stores a new ITR value based on strictly on packet size. This 3175 * algorithm is less sophisticated than that used in igc_update_itr, 3176 * due to the difficulty of synchronizing statistics across multiple 3177 * receive rings. The divisors and thresholds used by this function 3178 * were determined based on theoretical maximum wire speed and testing 3179 * data, in order to minimize response time while increasing bulk 3180 * throughput. 3181 * NOTE: This function is called only when operating in a multiqueue 3182 * receive environment. 3183 */ 3184 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 3185 { 3186 struct igc_adapter *adapter = q_vector->adapter; 3187 int new_val = q_vector->itr_val; 3188 int avg_wire_size = 0; 3189 unsigned int packets; 3190 3191 /* For non-gigabit speeds, just fix the interrupt rate at 4000 3192 * ints/sec - ITR timer value of 120 ticks. 3193 */ 3194 switch (adapter->link_speed) { 3195 case SPEED_10: 3196 case SPEED_100: 3197 new_val = IGC_4K_ITR; 3198 goto set_itr_val; 3199 default: 3200 break; 3201 } 3202 3203 packets = q_vector->rx.total_packets; 3204 if (packets) 3205 avg_wire_size = q_vector->rx.total_bytes / packets; 3206 3207 packets = q_vector->tx.total_packets; 3208 if (packets) 3209 avg_wire_size = max_t(u32, avg_wire_size, 3210 q_vector->tx.total_bytes / packets); 3211 3212 /* if avg_wire_size isn't set no work was done */ 3213 if (!avg_wire_size) 3214 goto clear_counts; 3215 3216 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 3217 avg_wire_size += 24; 3218 3219 /* Don't starve jumbo frames */ 3220 avg_wire_size = min(avg_wire_size, 3000); 3221 3222 /* Give a little boost to mid-size frames */ 3223 if (avg_wire_size > 300 && avg_wire_size < 1200) 3224 new_val = avg_wire_size / 3; 3225 else 3226 new_val = avg_wire_size / 2; 3227 3228 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3229 if (new_val < IGC_20K_ITR && 3230 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3231 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3232 new_val = IGC_20K_ITR; 3233 3234 set_itr_val: 3235 if (new_val != q_vector->itr_val) { 3236 q_vector->itr_val = new_val; 3237 q_vector->set_itr = 1; 3238 } 3239 clear_counts: 3240 q_vector->rx.total_bytes = 0; 3241 q_vector->rx.total_packets = 0; 3242 q_vector->tx.total_bytes = 0; 3243 q_vector->tx.total_packets = 0; 3244 } 3245 3246 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 3247 { 3248 struct igc_adapter *adapter = q_vector->adapter; 3249 struct igc_hw *hw = &adapter->hw; 3250 3251 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 3252 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 3253 if (adapter->num_q_vectors == 1) 3254 igc_set_itr(q_vector); 3255 else 3256 igc_update_ring_itr(q_vector); 3257 } 3258 3259 if (!test_bit(__IGC_DOWN, &adapter->state)) { 3260 if (adapter->msix_entries) 3261 wr32(IGC_EIMS, q_vector->eims_value); 3262 else 3263 igc_irq_enable(adapter); 3264 } 3265 } 3266 3267 static void igc_add_ring(struct igc_ring *ring, 3268 struct igc_ring_container *head) 3269 { 3270 head->ring = ring; 3271 head->count++; 3272 } 3273 3274 /** 3275 * igc_cache_ring_register - Descriptor ring to register mapping 3276 * @adapter: board private structure to initialize 3277 * 3278 * Once we know the feature-set enabled for the device, we'll cache 3279 * the register offset the descriptor ring is assigned to. 3280 */ 3281 static void igc_cache_ring_register(struct igc_adapter *adapter) 3282 { 3283 int i = 0, j = 0; 3284 3285 switch (adapter->hw.mac.type) { 3286 case igc_i225: 3287 /* Fall through */ 3288 default: 3289 for (; i < adapter->num_rx_queues; i++) 3290 adapter->rx_ring[i]->reg_idx = i; 3291 for (; j < adapter->num_tx_queues; j++) 3292 adapter->tx_ring[j]->reg_idx = j; 3293 break; 3294 } 3295 } 3296 3297 /** 3298 * igc_poll - NAPI Rx polling callback 3299 * @napi: napi polling structure 3300 * @budget: count of how many packets we should handle 3301 */ 3302 static int igc_poll(struct napi_struct *napi, int budget) 3303 { 3304 struct igc_q_vector *q_vector = container_of(napi, 3305 struct igc_q_vector, 3306 napi); 3307 bool clean_complete = true; 3308 int work_done = 0; 3309 3310 if (q_vector->tx.ring) 3311 clean_complete = igc_clean_tx_irq(q_vector, budget); 3312 3313 if (q_vector->rx.ring) { 3314 int cleaned = igc_clean_rx_irq(q_vector, budget); 3315 3316 work_done += cleaned; 3317 if (cleaned >= budget) 3318 clean_complete = false; 3319 } 3320 3321 /* If all work not completed, return budget and keep polling */ 3322 if (!clean_complete) 3323 return budget; 3324 3325 /* Exit the polling mode, but don't re-enable interrupts if stack might 3326 * poll us due to busy-polling 3327 */ 3328 if (likely(napi_complete_done(napi, work_done))) 3329 igc_ring_irq_enable(q_vector); 3330 3331 return min(work_done, budget - 1); 3332 } 3333 3334 /** 3335 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 3336 * @adapter: board private structure to initialize 3337 * @v_count: q_vectors allocated on adapter, used for ring interleaving 3338 * @v_idx: index of vector in adapter struct 3339 * @txr_count: total number of Tx rings to allocate 3340 * @txr_idx: index of first Tx ring to allocate 3341 * @rxr_count: total number of Rx rings to allocate 3342 * @rxr_idx: index of first Rx ring to allocate 3343 * 3344 * We allocate one q_vector. If allocation fails we return -ENOMEM. 3345 */ 3346 static int igc_alloc_q_vector(struct igc_adapter *adapter, 3347 unsigned int v_count, unsigned int v_idx, 3348 unsigned int txr_count, unsigned int txr_idx, 3349 unsigned int rxr_count, unsigned int rxr_idx) 3350 { 3351 struct igc_q_vector *q_vector; 3352 struct igc_ring *ring; 3353 int ring_count; 3354 3355 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 3356 if (txr_count > 1 || rxr_count > 1) 3357 return -ENOMEM; 3358 3359 ring_count = txr_count + rxr_count; 3360 3361 /* allocate q_vector and rings */ 3362 q_vector = adapter->q_vector[v_idx]; 3363 if (!q_vector) 3364 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), 3365 GFP_KERNEL); 3366 else 3367 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 3368 if (!q_vector) 3369 return -ENOMEM; 3370 3371 /* initialize NAPI */ 3372 netif_napi_add(adapter->netdev, &q_vector->napi, 3373 igc_poll, 64); 3374 3375 /* tie q_vector and adapter together */ 3376 adapter->q_vector[v_idx] = q_vector; 3377 q_vector->adapter = adapter; 3378 3379 /* initialize work limits */ 3380 q_vector->tx.work_limit = adapter->tx_work_limit; 3381 3382 /* initialize ITR configuration */ 3383 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 3384 q_vector->itr_val = IGC_START_ITR; 3385 3386 /* initialize pointer to rings */ 3387 ring = q_vector->ring; 3388 3389 /* initialize ITR */ 3390 if (rxr_count) { 3391 /* rx or rx/tx vector */ 3392 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 3393 q_vector->itr_val = adapter->rx_itr_setting; 3394 } else { 3395 /* tx only vector */ 3396 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 3397 q_vector->itr_val = adapter->tx_itr_setting; 3398 } 3399 3400 if (txr_count) { 3401 /* assign generic ring traits */ 3402 ring->dev = &adapter->pdev->dev; 3403 ring->netdev = adapter->netdev; 3404 3405 /* configure backlink on ring */ 3406 ring->q_vector = q_vector; 3407 3408 /* update q_vector Tx values */ 3409 igc_add_ring(ring, &q_vector->tx); 3410 3411 /* apply Tx specific ring traits */ 3412 ring->count = adapter->tx_ring_count; 3413 ring->queue_index = txr_idx; 3414 3415 /* assign ring to adapter */ 3416 adapter->tx_ring[txr_idx] = ring; 3417 3418 /* push pointer to next ring */ 3419 ring++; 3420 } 3421 3422 if (rxr_count) { 3423 /* assign generic ring traits */ 3424 ring->dev = &adapter->pdev->dev; 3425 ring->netdev = adapter->netdev; 3426 3427 /* configure backlink on ring */ 3428 ring->q_vector = q_vector; 3429 3430 /* update q_vector Rx values */ 3431 igc_add_ring(ring, &q_vector->rx); 3432 3433 /* apply Rx specific ring traits */ 3434 ring->count = adapter->rx_ring_count; 3435 ring->queue_index = rxr_idx; 3436 3437 /* assign ring to adapter */ 3438 adapter->rx_ring[rxr_idx] = ring; 3439 } 3440 3441 return 0; 3442 } 3443 3444 /** 3445 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 3446 * @adapter: board private structure to initialize 3447 * 3448 * We allocate one q_vector per queue interrupt. If allocation fails we 3449 * return -ENOMEM. 3450 */ 3451 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 3452 { 3453 int rxr_remaining = adapter->num_rx_queues; 3454 int txr_remaining = adapter->num_tx_queues; 3455 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 3456 int q_vectors = adapter->num_q_vectors; 3457 int err; 3458 3459 if (q_vectors >= (rxr_remaining + txr_remaining)) { 3460 for (; rxr_remaining; v_idx++) { 3461 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3462 0, 0, 1, rxr_idx); 3463 3464 if (err) 3465 goto err_out; 3466 3467 /* update counts and index */ 3468 rxr_remaining--; 3469 rxr_idx++; 3470 } 3471 } 3472 3473 for (; v_idx < q_vectors; v_idx++) { 3474 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 3475 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 3476 3477 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3478 tqpv, txr_idx, rqpv, rxr_idx); 3479 3480 if (err) 3481 goto err_out; 3482 3483 /* update counts and index */ 3484 rxr_remaining -= rqpv; 3485 txr_remaining -= tqpv; 3486 rxr_idx++; 3487 txr_idx++; 3488 } 3489 3490 return 0; 3491 3492 err_out: 3493 adapter->num_tx_queues = 0; 3494 adapter->num_rx_queues = 0; 3495 adapter->num_q_vectors = 0; 3496 3497 while (v_idx--) 3498 igc_free_q_vector(adapter, v_idx); 3499 3500 return -ENOMEM; 3501 } 3502 3503 /** 3504 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 3505 * @adapter: Pointer to adapter structure 3506 * @msix: boolean for MSI-X capability 3507 * 3508 * This function initializes the interrupts and allocates all of the queues. 3509 */ 3510 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 3511 { 3512 struct net_device *dev = adapter->netdev; 3513 int err = 0; 3514 3515 igc_set_interrupt_capability(adapter, msix); 3516 3517 err = igc_alloc_q_vectors(adapter); 3518 if (err) { 3519 netdev_err(dev, "Unable to allocate memory for vectors\n"); 3520 goto err_alloc_q_vectors; 3521 } 3522 3523 igc_cache_ring_register(adapter); 3524 3525 return 0; 3526 3527 err_alloc_q_vectors: 3528 igc_reset_interrupt_capability(adapter); 3529 return err; 3530 } 3531 3532 /** 3533 * igc_sw_init - Initialize general software structures (struct igc_adapter) 3534 * @adapter: board private structure to initialize 3535 * 3536 * igc_sw_init initializes the Adapter private data structure. 3537 * Fields are initialized based on PCI device information and 3538 * OS network device settings (MTU size). 3539 */ 3540 static int igc_sw_init(struct igc_adapter *adapter) 3541 { 3542 struct net_device *netdev = adapter->netdev; 3543 struct pci_dev *pdev = adapter->pdev; 3544 struct igc_hw *hw = &adapter->hw; 3545 3546 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 3547 3548 /* set default ring sizes */ 3549 adapter->tx_ring_count = IGC_DEFAULT_TXD; 3550 adapter->rx_ring_count = IGC_DEFAULT_RXD; 3551 3552 /* set default ITR values */ 3553 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 3554 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 3555 3556 /* set default work limits */ 3557 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 3558 3559 /* adjust max frame to be at least the size of a standard frame */ 3560 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 3561 VLAN_HLEN; 3562 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 3563 3564 mutex_init(&adapter->nfc_rule_lock); 3565 INIT_LIST_HEAD(&adapter->nfc_rule_list); 3566 adapter->nfc_rule_count = 0; 3567 3568 spin_lock_init(&adapter->stats64_lock); 3569 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 3570 adapter->flags |= IGC_FLAG_HAS_MSIX; 3571 3572 igc_init_queue_configuration(adapter); 3573 3574 /* This call may decrease the number of queues */ 3575 if (igc_init_interrupt_scheme(adapter, true)) { 3576 netdev_err(netdev, "Unable to allocate memory for queues\n"); 3577 return -ENOMEM; 3578 } 3579 3580 /* Explicitly disable IRQ since the NIC can be in any state. */ 3581 igc_irq_disable(adapter); 3582 3583 set_bit(__IGC_DOWN, &adapter->state); 3584 3585 return 0; 3586 } 3587 3588 /** 3589 * igc_up - Open the interface and prepare it to handle traffic 3590 * @adapter: board private structure 3591 */ 3592 void igc_up(struct igc_adapter *adapter) 3593 { 3594 struct igc_hw *hw = &adapter->hw; 3595 int i = 0; 3596 3597 /* hardware has been reset, we need to reload some things */ 3598 igc_configure(adapter); 3599 3600 clear_bit(__IGC_DOWN, &adapter->state); 3601 3602 for (i = 0; i < adapter->num_q_vectors; i++) 3603 napi_enable(&adapter->q_vector[i]->napi); 3604 3605 if (adapter->msix_entries) 3606 igc_configure_msix(adapter); 3607 else 3608 igc_assign_vector(adapter->q_vector[0], 0); 3609 3610 /* Clear any pending interrupts. */ 3611 rd32(IGC_ICR); 3612 igc_irq_enable(adapter); 3613 3614 netif_tx_start_all_queues(adapter->netdev); 3615 3616 /* start the watchdog. */ 3617 hw->mac.get_link_status = 1; 3618 schedule_work(&adapter->watchdog_task); 3619 } 3620 3621 /** 3622 * igc_update_stats - Update the board statistics counters 3623 * @adapter: board private structure 3624 */ 3625 void igc_update_stats(struct igc_adapter *adapter) 3626 { 3627 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 3628 struct pci_dev *pdev = adapter->pdev; 3629 struct igc_hw *hw = &adapter->hw; 3630 u64 _bytes, _packets; 3631 u64 bytes, packets; 3632 unsigned int start; 3633 u32 mpc; 3634 int i; 3635 3636 /* Prevent stats update while adapter is being reset, or if the pci 3637 * connection is down. 3638 */ 3639 if (adapter->link_speed == 0) 3640 return; 3641 if (pci_channel_offline(pdev)) 3642 return; 3643 3644 packets = 0; 3645 bytes = 0; 3646 3647 rcu_read_lock(); 3648 for (i = 0; i < adapter->num_rx_queues; i++) { 3649 struct igc_ring *ring = adapter->rx_ring[i]; 3650 u32 rqdpc = rd32(IGC_RQDPC(i)); 3651 3652 if (hw->mac.type >= igc_i225) 3653 wr32(IGC_RQDPC(i), 0); 3654 3655 if (rqdpc) { 3656 ring->rx_stats.drops += rqdpc; 3657 net_stats->rx_fifo_errors += rqdpc; 3658 } 3659 3660 do { 3661 start = u64_stats_fetch_begin_irq(&ring->rx_syncp); 3662 _bytes = ring->rx_stats.bytes; 3663 _packets = ring->rx_stats.packets; 3664 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); 3665 bytes += _bytes; 3666 packets += _packets; 3667 } 3668 3669 net_stats->rx_bytes = bytes; 3670 net_stats->rx_packets = packets; 3671 3672 packets = 0; 3673 bytes = 0; 3674 for (i = 0; i < adapter->num_tx_queues; i++) { 3675 struct igc_ring *ring = adapter->tx_ring[i]; 3676 3677 do { 3678 start = u64_stats_fetch_begin_irq(&ring->tx_syncp); 3679 _bytes = ring->tx_stats.bytes; 3680 _packets = ring->tx_stats.packets; 3681 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); 3682 bytes += _bytes; 3683 packets += _packets; 3684 } 3685 net_stats->tx_bytes = bytes; 3686 net_stats->tx_packets = packets; 3687 rcu_read_unlock(); 3688 3689 /* read stats registers */ 3690 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 3691 adapter->stats.gprc += rd32(IGC_GPRC); 3692 adapter->stats.gorc += rd32(IGC_GORCL); 3693 rd32(IGC_GORCH); /* clear GORCL */ 3694 adapter->stats.bprc += rd32(IGC_BPRC); 3695 adapter->stats.mprc += rd32(IGC_MPRC); 3696 adapter->stats.roc += rd32(IGC_ROC); 3697 3698 adapter->stats.prc64 += rd32(IGC_PRC64); 3699 adapter->stats.prc127 += rd32(IGC_PRC127); 3700 adapter->stats.prc255 += rd32(IGC_PRC255); 3701 adapter->stats.prc511 += rd32(IGC_PRC511); 3702 adapter->stats.prc1023 += rd32(IGC_PRC1023); 3703 adapter->stats.prc1522 += rd32(IGC_PRC1522); 3704 3705 mpc = rd32(IGC_MPC); 3706 adapter->stats.mpc += mpc; 3707 net_stats->rx_fifo_errors += mpc; 3708 adapter->stats.scc += rd32(IGC_SCC); 3709 adapter->stats.ecol += rd32(IGC_ECOL); 3710 adapter->stats.mcc += rd32(IGC_MCC); 3711 adapter->stats.latecol += rd32(IGC_LATECOL); 3712 adapter->stats.dc += rd32(IGC_DC); 3713 adapter->stats.rlec += rd32(IGC_RLEC); 3714 adapter->stats.xonrxc += rd32(IGC_XONRXC); 3715 adapter->stats.xontxc += rd32(IGC_XONTXC); 3716 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 3717 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 3718 adapter->stats.fcruc += rd32(IGC_FCRUC); 3719 adapter->stats.gptc += rd32(IGC_GPTC); 3720 adapter->stats.gotc += rd32(IGC_GOTCL); 3721 rd32(IGC_GOTCH); /* clear GOTCL */ 3722 adapter->stats.rnbc += rd32(IGC_RNBC); 3723 adapter->stats.ruc += rd32(IGC_RUC); 3724 adapter->stats.rfc += rd32(IGC_RFC); 3725 adapter->stats.rjc += rd32(IGC_RJC); 3726 adapter->stats.tor += rd32(IGC_TORH); 3727 adapter->stats.tot += rd32(IGC_TOTH); 3728 adapter->stats.tpr += rd32(IGC_TPR); 3729 3730 adapter->stats.ptc64 += rd32(IGC_PTC64); 3731 adapter->stats.ptc127 += rd32(IGC_PTC127); 3732 adapter->stats.ptc255 += rd32(IGC_PTC255); 3733 adapter->stats.ptc511 += rd32(IGC_PTC511); 3734 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 3735 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 3736 3737 adapter->stats.mptc += rd32(IGC_MPTC); 3738 adapter->stats.bptc += rd32(IGC_BPTC); 3739 3740 adapter->stats.tpt += rd32(IGC_TPT); 3741 adapter->stats.colc += rd32(IGC_COLC); 3742 adapter->stats.colc += rd32(IGC_RERC); 3743 3744 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 3745 3746 adapter->stats.tsctc += rd32(IGC_TSCTC); 3747 adapter->stats.tsctfc += rd32(IGC_TSCTFC); 3748 3749 adapter->stats.iac += rd32(IGC_IAC); 3750 adapter->stats.icrxoc += rd32(IGC_ICRXOC); 3751 adapter->stats.icrxptc += rd32(IGC_ICRXPTC); 3752 adapter->stats.icrxatc += rd32(IGC_ICRXATC); 3753 adapter->stats.ictxptc += rd32(IGC_ICTXPTC); 3754 adapter->stats.ictxatc += rd32(IGC_ICTXATC); 3755 adapter->stats.ictxqec += rd32(IGC_ICTXQEC); 3756 adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC); 3757 adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC); 3758 3759 /* Fill out the OS statistics structure */ 3760 net_stats->multicast = adapter->stats.mprc; 3761 net_stats->collisions = adapter->stats.colc; 3762 3763 /* Rx Errors */ 3764 3765 /* RLEC on some newer hardware can be incorrect so build 3766 * our own version based on RUC and ROC 3767 */ 3768 net_stats->rx_errors = adapter->stats.rxerrc + 3769 adapter->stats.crcerrs + adapter->stats.algnerrc + 3770 adapter->stats.ruc + adapter->stats.roc + 3771 adapter->stats.cexterr; 3772 net_stats->rx_length_errors = adapter->stats.ruc + 3773 adapter->stats.roc; 3774 net_stats->rx_crc_errors = adapter->stats.crcerrs; 3775 net_stats->rx_frame_errors = adapter->stats.algnerrc; 3776 net_stats->rx_missed_errors = adapter->stats.mpc; 3777 3778 /* Tx Errors */ 3779 net_stats->tx_errors = adapter->stats.ecol + 3780 adapter->stats.latecol; 3781 net_stats->tx_aborted_errors = adapter->stats.ecol; 3782 net_stats->tx_window_errors = adapter->stats.latecol; 3783 net_stats->tx_carrier_errors = adapter->stats.tncrs; 3784 3785 /* Tx Dropped needs to be maintained elsewhere */ 3786 3787 /* Management Stats */ 3788 adapter->stats.mgptc += rd32(IGC_MGTPTC); 3789 adapter->stats.mgprc += rd32(IGC_MGTPRC); 3790 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 3791 } 3792 3793 /** 3794 * igc_down - Close the interface 3795 * @adapter: board private structure 3796 */ 3797 void igc_down(struct igc_adapter *adapter) 3798 { 3799 struct net_device *netdev = adapter->netdev; 3800 struct igc_hw *hw = &adapter->hw; 3801 u32 tctl, rctl; 3802 int i = 0; 3803 3804 set_bit(__IGC_DOWN, &adapter->state); 3805 3806 /* disable receives in the hardware */ 3807 rctl = rd32(IGC_RCTL); 3808 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 3809 /* flush and sleep below */ 3810 3811 /* set trans_start so we don't get spurious watchdogs during reset */ 3812 netif_trans_update(netdev); 3813 3814 netif_carrier_off(netdev); 3815 netif_tx_stop_all_queues(netdev); 3816 3817 /* disable transmits in the hardware */ 3818 tctl = rd32(IGC_TCTL); 3819 tctl &= ~IGC_TCTL_EN; 3820 wr32(IGC_TCTL, tctl); 3821 /* flush both disables and wait for them to finish */ 3822 wrfl(); 3823 usleep_range(10000, 20000); 3824 3825 igc_irq_disable(adapter); 3826 3827 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3828 3829 for (i = 0; i < adapter->num_q_vectors; i++) { 3830 if (adapter->q_vector[i]) { 3831 napi_synchronize(&adapter->q_vector[i]->napi); 3832 napi_disable(&adapter->q_vector[i]->napi); 3833 } 3834 } 3835 3836 del_timer_sync(&adapter->watchdog_timer); 3837 del_timer_sync(&adapter->phy_info_timer); 3838 3839 /* record the stats before reset*/ 3840 spin_lock(&adapter->stats64_lock); 3841 igc_update_stats(adapter); 3842 spin_unlock(&adapter->stats64_lock); 3843 3844 adapter->link_speed = 0; 3845 adapter->link_duplex = 0; 3846 3847 if (!pci_channel_offline(adapter->pdev)) 3848 igc_reset(adapter); 3849 3850 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 3851 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 3852 3853 igc_clean_all_tx_rings(adapter); 3854 igc_clean_all_rx_rings(adapter); 3855 } 3856 3857 void igc_reinit_locked(struct igc_adapter *adapter) 3858 { 3859 WARN_ON(in_interrupt()); 3860 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3861 usleep_range(1000, 2000); 3862 igc_down(adapter); 3863 igc_up(adapter); 3864 clear_bit(__IGC_RESETTING, &adapter->state); 3865 } 3866 3867 static void igc_reset_task(struct work_struct *work) 3868 { 3869 struct igc_adapter *adapter; 3870 3871 adapter = container_of(work, struct igc_adapter, reset_task); 3872 3873 igc_rings_dump(adapter); 3874 igc_regs_dump(adapter); 3875 netdev_err(adapter->netdev, "Reset adapter\n"); 3876 igc_reinit_locked(adapter); 3877 } 3878 3879 /** 3880 * igc_change_mtu - Change the Maximum Transfer Unit 3881 * @netdev: network interface device structure 3882 * @new_mtu: new value for maximum frame size 3883 * 3884 * Returns 0 on success, negative on failure 3885 */ 3886 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 3887 { 3888 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 3889 struct igc_adapter *adapter = netdev_priv(netdev); 3890 3891 /* adjust max frame to be at least the size of a standard frame */ 3892 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 3893 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 3894 3895 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3896 usleep_range(1000, 2000); 3897 3898 /* igc_down has a dependency on max_frame_size */ 3899 adapter->max_frame_size = max_frame; 3900 3901 if (netif_running(netdev)) 3902 igc_down(adapter); 3903 3904 netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 3905 netdev->mtu = new_mtu; 3906 3907 if (netif_running(netdev)) 3908 igc_up(adapter); 3909 else 3910 igc_reset(adapter); 3911 3912 clear_bit(__IGC_RESETTING, &adapter->state); 3913 3914 return 0; 3915 } 3916 3917 /** 3918 * igc_get_stats - Get System Network Statistics 3919 * @netdev: network interface device structure 3920 * 3921 * Returns the address of the device statistics structure. 3922 * The statistics are updated here and also from the timer callback. 3923 */ 3924 static struct net_device_stats *igc_get_stats(struct net_device *netdev) 3925 { 3926 struct igc_adapter *adapter = netdev_priv(netdev); 3927 3928 if (!test_bit(__IGC_RESETTING, &adapter->state)) 3929 igc_update_stats(adapter); 3930 3931 /* only return the current stats */ 3932 return &netdev->stats; 3933 } 3934 3935 static netdev_features_t igc_fix_features(struct net_device *netdev, 3936 netdev_features_t features) 3937 { 3938 /* Since there is no support for separate Rx/Tx vlan accel 3939 * enable/disable make sure Tx flag is always in same state as Rx. 3940 */ 3941 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3942 features |= NETIF_F_HW_VLAN_CTAG_TX; 3943 else 3944 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 3945 3946 return features; 3947 } 3948 3949 static int igc_set_features(struct net_device *netdev, 3950 netdev_features_t features) 3951 { 3952 netdev_features_t changed = netdev->features ^ features; 3953 struct igc_adapter *adapter = netdev_priv(netdev); 3954 3955 /* Add VLAN support */ 3956 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 3957 return 0; 3958 3959 if (!(features & NETIF_F_NTUPLE)) 3960 igc_flush_nfc_rules(adapter); 3961 3962 netdev->features = features; 3963 3964 if (netif_running(netdev)) 3965 igc_reinit_locked(adapter); 3966 else 3967 igc_reset(adapter); 3968 3969 return 1; 3970 } 3971 3972 static netdev_features_t 3973 igc_features_check(struct sk_buff *skb, struct net_device *dev, 3974 netdev_features_t features) 3975 { 3976 unsigned int network_hdr_len, mac_hdr_len; 3977 3978 /* Make certain the headers can be described by a context descriptor */ 3979 mac_hdr_len = skb_network_header(skb) - skb->data; 3980 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 3981 return features & ~(NETIF_F_HW_CSUM | 3982 NETIF_F_SCTP_CRC | 3983 NETIF_F_HW_VLAN_CTAG_TX | 3984 NETIF_F_TSO | 3985 NETIF_F_TSO6); 3986 3987 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 3988 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 3989 return features & ~(NETIF_F_HW_CSUM | 3990 NETIF_F_SCTP_CRC | 3991 NETIF_F_TSO | 3992 NETIF_F_TSO6); 3993 3994 /* We can only support IPv4 TSO in tunnels if we can mangle the 3995 * inner IP ID field, so strip TSO if MANGLEID is not supported. 3996 */ 3997 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 3998 features &= ~NETIF_F_TSO; 3999 4000 return features; 4001 } 4002 4003 static void igc_tsync_interrupt(struct igc_adapter *adapter) 4004 { 4005 struct igc_hw *hw = &adapter->hw; 4006 u32 tsicr = rd32(IGC_TSICR); 4007 u32 ack = 0; 4008 4009 if (tsicr & IGC_TSICR_TXTS) { 4010 /* retrieve hardware timestamp */ 4011 schedule_work(&adapter->ptp_tx_work); 4012 ack |= IGC_TSICR_TXTS; 4013 } 4014 4015 /* acknowledge the interrupts */ 4016 wr32(IGC_TSICR, ack); 4017 } 4018 4019 /** 4020 * igc_msix_other - msix other interrupt handler 4021 * @irq: interrupt number 4022 * @data: pointer to a q_vector 4023 */ 4024 static irqreturn_t igc_msix_other(int irq, void *data) 4025 { 4026 struct igc_adapter *adapter = data; 4027 struct igc_hw *hw = &adapter->hw; 4028 u32 icr = rd32(IGC_ICR); 4029 4030 /* reading ICR causes bit 31 of EICR to be cleared */ 4031 if (icr & IGC_ICR_DRSTA) 4032 schedule_work(&adapter->reset_task); 4033 4034 if (icr & IGC_ICR_DOUTSYNC) { 4035 /* HW is reporting DMA is out of sync */ 4036 adapter->stats.doosync++; 4037 } 4038 4039 if (icr & IGC_ICR_LSC) { 4040 hw->mac.get_link_status = 1; 4041 /* guard against interrupt when we're going down */ 4042 if (!test_bit(__IGC_DOWN, &adapter->state)) 4043 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4044 } 4045 4046 if (icr & IGC_ICR_TS) 4047 igc_tsync_interrupt(adapter); 4048 4049 wr32(IGC_EIMS, adapter->eims_other); 4050 4051 return IRQ_HANDLED; 4052 } 4053 4054 static void igc_write_itr(struct igc_q_vector *q_vector) 4055 { 4056 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 4057 4058 if (!q_vector->set_itr) 4059 return; 4060 4061 if (!itr_val) 4062 itr_val = IGC_ITR_VAL_MASK; 4063 4064 itr_val |= IGC_EITR_CNT_IGNR; 4065 4066 writel(itr_val, q_vector->itr_register); 4067 q_vector->set_itr = 0; 4068 } 4069 4070 static irqreturn_t igc_msix_ring(int irq, void *data) 4071 { 4072 struct igc_q_vector *q_vector = data; 4073 4074 /* Write the ITR value calculated from the previous interrupt. */ 4075 igc_write_itr(q_vector); 4076 4077 napi_schedule(&q_vector->napi); 4078 4079 return IRQ_HANDLED; 4080 } 4081 4082 /** 4083 * igc_request_msix - Initialize MSI-X interrupts 4084 * @adapter: Pointer to adapter structure 4085 * 4086 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 4087 * kernel. 4088 */ 4089 static int igc_request_msix(struct igc_adapter *adapter) 4090 { 4091 int i = 0, err = 0, vector = 0, free_vector = 0; 4092 struct net_device *netdev = adapter->netdev; 4093 4094 err = request_irq(adapter->msix_entries[vector].vector, 4095 &igc_msix_other, 0, netdev->name, adapter); 4096 if (err) 4097 goto err_out; 4098 4099 for (i = 0; i < adapter->num_q_vectors; i++) { 4100 struct igc_q_vector *q_vector = adapter->q_vector[i]; 4101 4102 vector++; 4103 4104 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 4105 4106 if (q_vector->rx.ring && q_vector->tx.ring) 4107 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 4108 q_vector->rx.ring->queue_index); 4109 else if (q_vector->tx.ring) 4110 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 4111 q_vector->tx.ring->queue_index); 4112 else if (q_vector->rx.ring) 4113 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 4114 q_vector->rx.ring->queue_index); 4115 else 4116 sprintf(q_vector->name, "%s-unused", netdev->name); 4117 4118 err = request_irq(adapter->msix_entries[vector].vector, 4119 igc_msix_ring, 0, q_vector->name, 4120 q_vector); 4121 if (err) 4122 goto err_free; 4123 } 4124 4125 igc_configure_msix(adapter); 4126 return 0; 4127 4128 err_free: 4129 /* free already assigned IRQs */ 4130 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 4131 4132 vector--; 4133 for (i = 0; i < vector; i++) { 4134 free_irq(adapter->msix_entries[free_vector++].vector, 4135 adapter->q_vector[i]); 4136 } 4137 err_out: 4138 return err; 4139 } 4140 4141 /** 4142 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 4143 * @adapter: Pointer to adapter structure 4144 * 4145 * This function resets the device so that it has 0 rx queues, tx queues, and 4146 * MSI-X interrupts allocated. 4147 */ 4148 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 4149 { 4150 igc_free_q_vectors(adapter); 4151 igc_reset_interrupt_capability(adapter); 4152 } 4153 4154 /* Need to wait a few seconds after link up to get diagnostic information from 4155 * the phy 4156 */ 4157 static void igc_update_phy_info(struct timer_list *t) 4158 { 4159 struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); 4160 4161 igc_get_phy_info(&adapter->hw); 4162 } 4163 4164 /** 4165 * igc_has_link - check shared code for link and determine up/down 4166 * @adapter: pointer to driver private info 4167 */ 4168 bool igc_has_link(struct igc_adapter *adapter) 4169 { 4170 struct igc_hw *hw = &adapter->hw; 4171 bool link_active = false; 4172 4173 /* get_link_status is set on LSC (link status) interrupt or 4174 * rx sequence error interrupt. get_link_status will stay 4175 * false until the igc_check_for_link establishes link 4176 * for copper adapters ONLY 4177 */ 4178 switch (hw->phy.media_type) { 4179 case igc_media_type_copper: 4180 if (!hw->mac.get_link_status) 4181 return true; 4182 hw->mac.ops.check_for_link(hw); 4183 link_active = !hw->mac.get_link_status; 4184 break; 4185 default: 4186 case igc_media_type_unknown: 4187 break; 4188 } 4189 4190 if (hw->mac.type == igc_i225 && 4191 hw->phy.id == I225_I_PHY_ID) { 4192 if (!netif_carrier_ok(adapter->netdev)) { 4193 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4194 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 4195 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 4196 adapter->link_check_timeout = jiffies; 4197 } 4198 } 4199 4200 return link_active; 4201 } 4202 4203 /** 4204 * igc_watchdog - Timer Call-back 4205 * @t: timer for the watchdog 4206 */ 4207 static void igc_watchdog(struct timer_list *t) 4208 { 4209 struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); 4210 /* Do the rest outside of interrupt context */ 4211 schedule_work(&adapter->watchdog_task); 4212 } 4213 4214 static void igc_watchdog_task(struct work_struct *work) 4215 { 4216 struct igc_adapter *adapter = container_of(work, 4217 struct igc_adapter, 4218 watchdog_task); 4219 struct net_device *netdev = adapter->netdev; 4220 struct igc_hw *hw = &adapter->hw; 4221 struct igc_phy_info *phy = &hw->phy; 4222 u16 phy_data, retry_count = 20; 4223 u32 link; 4224 int i; 4225 4226 link = igc_has_link(adapter); 4227 4228 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 4229 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 4230 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4231 else 4232 link = false; 4233 } 4234 4235 if (link) { 4236 /* Cancel scheduled suspend requests. */ 4237 pm_runtime_resume(netdev->dev.parent); 4238 4239 if (!netif_carrier_ok(netdev)) { 4240 u32 ctrl; 4241 4242 hw->mac.ops.get_speed_and_duplex(hw, 4243 &adapter->link_speed, 4244 &adapter->link_duplex); 4245 4246 ctrl = rd32(IGC_CTRL); 4247 /* Link status message must follow this format */ 4248 netdev_info(netdev, 4249 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 4250 adapter->link_speed, 4251 adapter->link_duplex == FULL_DUPLEX ? 4252 "Full" : "Half", 4253 (ctrl & IGC_CTRL_TFCE) && 4254 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 4255 (ctrl & IGC_CTRL_RFCE) ? "RX" : 4256 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 4257 4258 /* check if SmartSpeed worked */ 4259 igc_check_downshift(hw); 4260 if (phy->speed_downgraded) 4261 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 4262 4263 /* adjust timeout factor according to speed/duplex */ 4264 adapter->tx_timeout_factor = 1; 4265 switch (adapter->link_speed) { 4266 case SPEED_10: 4267 adapter->tx_timeout_factor = 14; 4268 break; 4269 case SPEED_100: 4270 /* maybe add some timeout factor ? */ 4271 break; 4272 } 4273 4274 if (adapter->link_speed != SPEED_1000) 4275 goto no_wait; 4276 4277 /* wait for Remote receiver status OK */ 4278 retry_read_status: 4279 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 4280 &phy_data)) { 4281 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 4282 retry_count) { 4283 msleep(100); 4284 retry_count--; 4285 goto retry_read_status; 4286 } else if (!retry_count) { 4287 netdev_err(netdev, "exceed max 2 second\n"); 4288 } 4289 } else { 4290 netdev_err(netdev, "read 1000Base-T Status Reg\n"); 4291 } 4292 no_wait: 4293 netif_carrier_on(netdev); 4294 4295 /* link state has changed, schedule phy info update */ 4296 if (!test_bit(__IGC_DOWN, &adapter->state)) 4297 mod_timer(&adapter->phy_info_timer, 4298 round_jiffies(jiffies + 2 * HZ)); 4299 } 4300 } else { 4301 if (netif_carrier_ok(netdev)) { 4302 adapter->link_speed = 0; 4303 adapter->link_duplex = 0; 4304 4305 /* Links status message must follow this format */ 4306 netdev_info(netdev, "NIC Link is Down\n"); 4307 netif_carrier_off(netdev); 4308 4309 /* link state has changed, schedule phy info update */ 4310 if (!test_bit(__IGC_DOWN, &adapter->state)) 4311 mod_timer(&adapter->phy_info_timer, 4312 round_jiffies(jiffies + 2 * HZ)); 4313 4314 /* link is down, time to check for alternate media */ 4315 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 4316 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4317 schedule_work(&adapter->reset_task); 4318 /* return immediately */ 4319 return; 4320 } 4321 } 4322 pm_schedule_suspend(netdev->dev.parent, 4323 MSEC_PER_SEC * 5); 4324 4325 /* also check for alternate media here */ 4326 } else if (!netif_carrier_ok(netdev) && 4327 (adapter->flags & IGC_FLAG_MAS_ENABLE)) { 4328 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4329 schedule_work(&adapter->reset_task); 4330 /* return immediately */ 4331 return; 4332 } 4333 } 4334 } 4335 4336 spin_lock(&adapter->stats64_lock); 4337 igc_update_stats(adapter); 4338 spin_unlock(&adapter->stats64_lock); 4339 4340 for (i = 0; i < adapter->num_tx_queues; i++) { 4341 struct igc_ring *tx_ring = adapter->tx_ring[i]; 4342 4343 if (!netif_carrier_ok(netdev)) { 4344 /* We've lost link, so the controller stops DMA, 4345 * but we've got queued Tx work that's never going 4346 * to get done, so reset controller to flush Tx. 4347 * (Do the reset outside of interrupt context). 4348 */ 4349 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 4350 adapter->tx_timeout_count++; 4351 schedule_work(&adapter->reset_task); 4352 /* return immediately since reset is imminent */ 4353 return; 4354 } 4355 } 4356 4357 /* Force detection of hung controller every watchdog period */ 4358 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 4359 } 4360 4361 /* Cause software interrupt to ensure Rx ring is cleaned */ 4362 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4363 u32 eics = 0; 4364 4365 for (i = 0; i < adapter->num_q_vectors; i++) 4366 eics |= adapter->q_vector[i]->eims_value; 4367 wr32(IGC_EICS, eics); 4368 } else { 4369 wr32(IGC_ICS, IGC_ICS_RXDMT0); 4370 } 4371 4372 igc_ptp_tx_hang(adapter); 4373 4374 /* Reset the timer */ 4375 if (!test_bit(__IGC_DOWN, &adapter->state)) { 4376 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 4377 mod_timer(&adapter->watchdog_timer, 4378 round_jiffies(jiffies + HZ)); 4379 else 4380 mod_timer(&adapter->watchdog_timer, 4381 round_jiffies(jiffies + 2 * HZ)); 4382 } 4383 } 4384 4385 /** 4386 * igc_intr_msi - Interrupt Handler 4387 * @irq: interrupt number 4388 * @data: pointer to a network interface device structure 4389 */ 4390 static irqreturn_t igc_intr_msi(int irq, void *data) 4391 { 4392 struct igc_adapter *adapter = data; 4393 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4394 struct igc_hw *hw = &adapter->hw; 4395 /* read ICR disables interrupts using IAM */ 4396 u32 icr = rd32(IGC_ICR); 4397 4398 igc_write_itr(q_vector); 4399 4400 if (icr & IGC_ICR_DRSTA) 4401 schedule_work(&adapter->reset_task); 4402 4403 if (icr & IGC_ICR_DOUTSYNC) { 4404 /* HW is reporting DMA is out of sync */ 4405 adapter->stats.doosync++; 4406 } 4407 4408 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4409 hw->mac.get_link_status = 1; 4410 if (!test_bit(__IGC_DOWN, &adapter->state)) 4411 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4412 } 4413 4414 napi_schedule(&q_vector->napi); 4415 4416 return IRQ_HANDLED; 4417 } 4418 4419 /** 4420 * igc_intr - Legacy Interrupt Handler 4421 * @irq: interrupt number 4422 * @data: pointer to a network interface device structure 4423 */ 4424 static irqreturn_t igc_intr(int irq, void *data) 4425 { 4426 struct igc_adapter *adapter = data; 4427 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4428 struct igc_hw *hw = &adapter->hw; 4429 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 4430 * need for the IMC write 4431 */ 4432 u32 icr = rd32(IGC_ICR); 4433 4434 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 4435 * not set, then the adapter didn't send an interrupt 4436 */ 4437 if (!(icr & IGC_ICR_INT_ASSERTED)) 4438 return IRQ_NONE; 4439 4440 igc_write_itr(q_vector); 4441 4442 if (icr & IGC_ICR_DRSTA) 4443 schedule_work(&adapter->reset_task); 4444 4445 if (icr & IGC_ICR_DOUTSYNC) { 4446 /* HW is reporting DMA is out of sync */ 4447 adapter->stats.doosync++; 4448 } 4449 4450 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4451 hw->mac.get_link_status = 1; 4452 /* guard against interrupt when we're going down */ 4453 if (!test_bit(__IGC_DOWN, &adapter->state)) 4454 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4455 } 4456 4457 napi_schedule(&q_vector->napi); 4458 4459 return IRQ_HANDLED; 4460 } 4461 4462 static void igc_free_irq(struct igc_adapter *adapter) 4463 { 4464 if (adapter->msix_entries) { 4465 int vector = 0, i; 4466 4467 free_irq(adapter->msix_entries[vector++].vector, adapter); 4468 4469 for (i = 0; i < adapter->num_q_vectors; i++) 4470 free_irq(adapter->msix_entries[vector++].vector, 4471 adapter->q_vector[i]); 4472 } else { 4473 free_irq(adapter->pdev->irq, adapter); 4474 } 4475 } 4476 4477 /** 4478 * igc_request_irq - initialize interrupts 4479 * @adapter: Pointer to adapter structure 4480 * 4481 * Attempts to configure interrupts using the best available 4482 * capabilities of the hardware and kernel. 4483 */ 4484 static int igc_request_irq(struct igc_adapter *adapter) 4485 { 4486 struct net_device *netdev = adapter->netdev; 4487 struct pci_dev *pdev = adapter->pdev; 4488 int err = 0; 4489 4490 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4491 err = igc_request_msix(adapter); 4492 if (!err) 4493 goto request_done; 4494 /* fall back to MSI */ 4495 igc_free_all_tx_resources(adapter); 4496 igc_free_all_rx_resources(adapter); 4497 4498 igc_clear_interrupt_scheme(adapter); 4499 err = igc_init_interrupt_scheme(adapter, false); 4500 if (err) 4501 goto request_done; 4502 igc_setup_all_tx_resources(adapter); 4503 igc_setup_all_rx_resources(adapter); 4504 igc_configure(adapter); 4505 } 4506 4507 igc_assign_vector(adapter->q_vector[0], 0); 4508 4509 if (adapter->flags & IGC_FLAG_HAS_MSI) { 4510 err = request_irq(pdev->irq, &igc_intr_msi, 0, 4511 netdev->name, adapter); 4512 if (!err) 4513 goto request_done; 4514 4515 /* fall back to legacy interrupts */ 4516 igc_reset_interrupt_capability(adapter); 4517 adapter->flags &= ~IGC_FLAG_HAS_MSI; 4518 } 4519 4520 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 4521 netdev->name, adapter); 4522 4523 if (err) 4524 netdev_err(netdev, "Error %d getting interrupt\n", err); 4525 4526 request_done: 4527 return err; 4528 } 4529 4530 /** 4531 * __igc_open - Called when a network interface is made active 4532 * @netdev: network interface device structure 4533 * @resuming: boolean indicating if the device is resuming 4534 * 4535 * Returns 0 on success, negative value on failure 4536 * 4537 * The open entry point is called when a network interface is made 4538 * active by the system (IFF_UP). At this point all resources needed 4539 * for transmit and receive operations are allocated, the interrupt 4540 * handler is registered with the OS, the watchdog timer is started, 4541 * and the stack is notified that the interface is ready. 4542 */ 4543 static int __igc_open(struct net_device *netdev, bool resuming) 4544 { 4545 struct igc_adapter *adapter = netdev_priv(netdev); 4546 struct pci_dev *pdev = adapter->pdev; 4547 struct igc_hw *hw = &adapter->hw; 4548 int err = 0; 4549 int i = 0; 4550 4551 /* disallow open during test */ 4552 4553 if (test_bit(__IGC_TESTING, &adapter->state)) { 4554 WARN_ON(resuming); 4555 return -EBUSY; 4556 } 4557 4558 if (!resuming) 4559 pm_runtime_get_sync(&pdev->dev); 4560 4561 netif_carrier_off(netdev); 4562 4563 /* allocate transmit descriptors */ 4564 err = igc_setup_all_tx_resources(adapter); 4565 if (err) 4566 goto err_setup_tx; 4567 4568 /* allocate receive descriptors */ 4569 err = igc_setup_all_rx_resources(adapter); 4570 if (err) 4571 goto err_setup_rx; 4572 4573 igc_power_up_link(adapter); 4574 4575 igc_configure(adapter); 4576 4577 err = igc_request_irq(adapter); 4578 if (err) 4579 goto err_req_irq; 4580 4581 /* Notify the stack of the actual queue counts. */ 4582 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 4583 if (err) 4584 goto err_set_queues; 4585 4586 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 4587 if (err) 4588 goto err_set_queues; 4589 4590 clear_bit(__IGC_DOWN, &adapter->state); 4591 4592 for (i = 0; i < adapter->num_q_vectors; i++) 4593 napi_enable(&adapter->q_vector[i]->napi); 4594 4595 /* Clear any pending interrupts. */ 4596 rd32(IGC_ICR); 4597 igc_irq_enable(adapter); 4598 4599 if (!resuming) 4600 pm_runtime_put(&pdev->dev); 4601 4602 netif_tx_start_all_queues(netdev); 4603 4604 /* start the watchdog. */ 4605 hw->mac.get_link_status = 1; 4606 schedule_work(&adapter->watchdog_task); 4607 4608 return IGC_SUCCESS; 4609 4610 err_set_queues: 4611 igc_free_irq(adapter); 4612 err_req_irq: 4613 igc_release_hw_control(adapter); 4614 igc_power_down_link(adapter); 4615 igc_free_all_rx_resources(adapter); 4616 err_setup_rx: 4617 igc_free_all_tx_resources(adapter); 4618 err_setup_tx: 4619 igc_reset(adapter); 4620 if (!resuming) 4621 pm_runtime_put(&pdev->dev); 4622 4623 return err; 4624 } 4625 4626 int igc_open(struct net_device *netdev) 4627 { 4628 return __igc_open(netdev, false); 4629 } 4630 4631 /** 4632 * __igc_close - Disables a network interface 4633 * @netdev: network interface device structure 4634 * @suspending: boolean indicating the device is suspending 4635 * 4636 * Returns 0, this is not allowed to fail 4637 * 4638 * The close entry point is called when an interface is de-activated 4639 * by the OS. The hardware is still under the driver's control, but 4640 * needs to be disabled. A global MAC reset is issued to stop the 4641 * hardware, and all transmit and receive resources are freed. 4642 */ 4643 static int __igc_close(struct net_device *netdev, bool suspending) 4644 { 4645 struct igc_adapter *adapter = netdev_priv(netdev); 4646 struct pci_dev *pdev = adapter->pdev; 4647 4648 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 4649 4650 if (!suspending) 4651 pm_runtime_get_sync(&pdev->dev); 4652 4653 igc_down(adapter); 4654 4655 igc_release_hw_control(adapter); 4656 4657 igc_free_irq(adapter); 4658 4659 igc_free_all_tx_resources(adapter); 4660 igc_free_all_rx_resources(adapter); 4661 4662 if (!suspending) 4663 pm_runtime_put_sync(&pdev->dev); 4664 4665 return 0; 4666 } 4667 4668 int igc_close(struct net_device *netdev) 4669 { 4670 if (netif_device_present(netdev) || netdev->dismantle) 4671 return __igc_close(netdev, false); 4672 return 0; 4673 } 4674 4675 /** 4676 * igc_ioctl - Access the hwtstamp interface 4677 * @netdev: network interface device structure 4678 * @ifreq: interface request data 4679 * @cmd: ioctl command 4680 **/ 4681 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 4682 { 4683 switch (cmd) { 4684 case SIOCGHWTSTAMP: 4685 return igc_ptp_get_ts_config(netdev, ifr); 4686 case SIOCSHWTSTAMP: 4687 return igc_ptp_set_ts_config(netdev, ifr); 4688 default: 4689 return -EOPNOTSUPP; 4690 } 4691 } 4692 4693 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, 4694 bool enable) 4695 { 4696 struct igc_ring *ring; 4697 int i; 4698 4699 if (queue < 0 || queue >= adapter->num_tx_queues) 4700 return -EINVAL; 4701 4702 ring = adapter->tx_ring[queue]; 4703 ring->launchtime_enable = enable; 4704 4705 if (adapter->base_time) 4706 return 0; 4707 4708 adapter->cycle_time = NSEC_PER_SEC; 4709 4710 for (i = 0; i < adapter->num_tx_queues; i++) { 4711 ring = adapter->tx_ring[i]; 4712 ring->start_time = 0; 4713 ring->end_time = NSEC_PER_SEC; 4714 } 4715 4716 return 0; 4717 } 4718 4719 static bool validate_schedule(const struct tc_taprio_qopt_offload *qopt) 4720 { 4721 int queue_uses[IGC_MAX_TX_QUEUES] = { }; 4722 size_t n; 4723 4724 if (qopt->cycle_time_extension) 4725 return false; 4726 4727 for (n = 0; n < qopt->num_entries; n++) { 4728 const struct tc_taprio_sched_entry *e; 4729 int i; 4730 4731 e = &qopt->entries[n]; 4732 4733 /* i225 only supports "global" frame preemption 4734 * settings. 4735 */ 4736 if (e->command != TC_TAPRIO_CMD_SET_GATES) 4737 return false; 4738 4739 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4740 if (e->gate_mask & BIT(i)) 4741 queue_uses[i]++; 4742 4743 if (queue_uses[i] > 1) 4744 return false; 4745 } 4746 } 4747 4748 return true; 4749 } 4750 4751 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, 4752 struct tc_etf_qopt_offload *qopt) 4753 { 4754 struct igc_hw *hw = &adapter->hw; 4755 int err; 4756 4757 if (hw->mac.type != igc_i225) 4758 return -EOPNOTSUPP; 4759 4760 err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable); 4761 if (err) 4762 return err; 4763 4764 return igc_tsn_offload_apply(adapter); 4765 } 4766 4767 static int igc_save_qbv_schedule(struct igc_adapter *adapter, 4768 struct tc_taprio_qopt_offload *qopt) 4769 { 4770 u32 start_time = 0, end_time = 0; 4771 size_t n; 4772 4773 if (!qopt->enable) { 4774 adapter->base_time = 0; 4775 return 0; 4776 } 4777 4778 if (adapter->base_time) 4779 return -EALREADY; 4780 4781 if (!validate_schedule(qopt)) 4782 return -EINVAL; 4783 4784 adapter->cycle_time = qopt->cycle_time; 4785 adapter->base_time = qopt->base_time; 4786 4787 /* FIXME: be a little smarter about cases when the gate for a 4788 * queue stays open for more than one entry. 4789 */ 4790 for (n = 0; n < qopt->num_entries; n++) { 4791 struct tc_taprio_sched_entry *e = &qopt->entries[n]; 4792 int i; 4793 4794 end_time += e->interval; 4795 4796 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4797 struct igc_ring *ring = adapter->tx_ring[i]; 4798 4799 if (!(e->gate_mask & BIT(i))) 4800 continue; 4801 4802 ring->start_time = start_time; 4803 ring->end_time = end_time; 4804 } 4805 4806 start_time += e->interval; 4807 } 4808 4809 return 0; 4810 } 4811 4812 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, 4813 struct tc_taprio_qopt_offload *qopt) 4814 { 4815 struct igc_hw *hw = &adapter->hw; 4816 int err; 4817 4818 if (hw->mac.type != igc_i225) 4819 return -EOPNOTSUPP; 4820 4821 err = igc_save_qbv_schedule(adapter, qopt); 4822 if (err) 4823 return err; 4824 4825 return igc_tsn_offload_apply(adapter); 4826 } 4827 4828 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, 4829 void *type_data) 4830 { 4831 struct igc_adapter *adapter = netdev_priv(dev); 4832 4833 switch (type) { 4834 case TC_SETUP_QDISC_TAPRIO: 4835 return igc_tsn_enable_qbv_scheduling(adapter, type_data); 4836 4837 case TC_SETUP_QDISC_ETF: 4838 return igc_tsn_enable_launchtime(adapter, type_data); 4839 4840 default: 4841 return -EOPNOTSUPP; 4842 } 4843 } 4844 4845 static const struct net_device_ops igc_netdev_ops = { 4846 .ndo_open = igc_open, 4847 .ndo_stop = igc_close, 4848 .ndo_start_xmit = igc_xmit_frame, 4849 .ndo_set_rx_mode = igc_set_rx_mode, 4850 .ndo_set_mac_address = igc_set_mac, 4851 .ndo_change_mtu = igc_change_mtu, 4852 .ndo_get_stats = igc_get_stats, 4853 .ndo_fix_features = igc_fix_features, 4854 .ndo_set_features = igc_set_features, 4855 .ndo_features_check = igc_features_check, 4856 .ndo_do_ioctl = igc_ioctl, 4857 .ndo_setup_tc = igc_setup_tc, 4858 }; 4859 4860 /* PCIe configuration access */ 4861 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4862 { 4863 struct igc_adapter *adapter = hw->back; 4864 4865 pci_read_config_word(adapter->pdev, reg, value); 4866 } 4867 4868 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4869 { 4870 struct igc_adapter *adapter = hw->back; 4871 4872 pci_write_config_word(adapter->pdev, reg, *value); 4873 } 4874 4875 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4876 { 4877 struct igc_adapter *adapter = hw->back; 4878 4879 if (!pci_is_pcie(adapter->pdev)) 4880 return -IGC_ERR_CONFIG; 4881 4882 pcie_capability_read_word(adapter->pdev, reg, value); 4883 4884 return IGC_SUCCESS; 4885 } 4886 4887 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4888 { 4889 struct igc_adapter *adapter = hw->back; 4890 4891 if (!pci_is_pcie(adapter->pdev)) 4892 return -IGC_ERR_CONFIG; 4893 4894 pcie_capability_write_word(adapter->pdev, reg, *value); 4895 4896 return IGC_SUCCESS; 4897 } 4898 4899 u32 igc_rd32(struct igc_hw *hw, u32 reg) 4900 { 4901 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 4902 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 4903 u32 value = 0; 4904 4905 value = readl(&hw_addr[reg]); 4906 4907 /* reads should not return all F's */ 4908 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 4909 struct net_device *netdev = igc->netdev; 4910 4911 hw->hw_addr = NULL; 4912 netif_device_detach(netdev); 4913 netdev_err(netdev, "PCIe link lost, device now detached\n"); 4914 WARN(pci_device_is_present(igc->pdev), 4915 "igc: Failed to read reg 0x%x!\n", reg); 4916 } 4917 4918 return value; 4919 } 4920 4921 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) 4922 { 4923 struct igc_mac_info *mac = &adapter->hw.mac; 4924 4925 mac->autoneg = 0; 4926 4927 /* Make sure dplx is at most 1 bit and lsb of speed is not set 4928 * for the switch() below to work 4929 */ 4930 if ((spd & 1) || (dplx & ~1)) 4931 goto err_inval; 4932 4933 switch (spd + dplx) { 4934 case SPEED_10 + DUPLEX_HALF: 4935 mac->forced_speed_duplex = ADVERTISE_10_HALF; 4936 break; 4937 case SPEED_10 + DUPLEX_FULL: 4938 mac->forced_speed_duplex = ADVERTISE_10_FULL; 4939 break; 4940 case SPEED_100 + DUPLEX_HALF: 4941 mac->forced_speed_duplex = ADVERTISE_100_HALF; 4942 break; 4943 case SPEED_100 + DUPLEX_FULL: 4944 mac->forced_speed_duplex = ADVERTISE_100_FULL; 4945 break; 4946 case SPEED_1000 + DUPLEX_FULL: 4947 mac->autoneg = 1; 4948 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 4949 break; 4950 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 4951 goto err_inval; 4952 case SPEED_2500 + DUPLEX_FULL: 4953 mac->autoneg = 1; 4954 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 4955 break; 4956 case SPEED_2500 + DUPLEX_HALF: /* not supported */ 4957 default: 4958 goto err_inval; 4959 } 4960 4961 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 4962 adapter->hw.phy.mdix = AUTO_ALL_MODES; 4963 4964 return 0; 4965 4966 err_inval: 4967 netdev_err(adapter->netdev, "Unsupported Speed/Duplex configuration\n"); 4968 return -EINVAL; 4969 } 4970 4971 /** 4972 * igc_probe - Device Initialization Routine 4973 * @pdev: PCI device information struct 4974 * @ent: entry in igc_pci_tbl 4975 * 4976 * Returns 0 on success, negative on failure 4977 * 4978 * igc_probe initializes an adapter identified by a pci_dev structure. 4979 * The OS initialization, configuring the adapter private structure, 4980 * and a hardware reset occur. 4981 */ 4982 static int igc_probe(struct pci_dev *pdev, 4983 const struct pci_device_id *ent) 4984 { 4985 struct igc_adapter *adapter; 4986 struct net_device *netdev; 4987 struct igc_hw *hw; 4988 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 4989 int err, pci_using_dac; 4990 4991 err = pci_enable_device_mem(pdev); 4992 if (err) 4993 return err; 4994 4995 pci_using_dac = 0; 4996 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4997 if (!err) { 4998 pci_using_dac = 1; 4999 } else { 5000 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 5001 if (err) { 5002 dev_err(&pdev->dev, 5003 "No usable DMA configuration, aborting\n"); 5004 goto err_dma; 5005 } 5006 } 5007 5008 err = pci_request_mem_regions(pdev, igc_driver_name); 5009 if (err) 5010 goto err_pci_reg; 5011 5012 pci_enable_pcie_error_reporting(pdev); 5013 5014 pci_set_master(pdev); 5015 5016 err = -ENOMEM; 5017 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 5018 IGC_MAX_TX_QUEUES); 5019 5020 if (!netdev) 5021 goto err_alloc_etherdev; 5022 5023 SET_NETDEV_DEV(netdev, &pdev->dev); 5024 5025 pci_set_drvdata(pdev, netdev); 5026 adapter = netdev_priv(netdev); 5027 adapter->netdev = netdev; 5028 adapter->pdev = pdev; 5029 hw = &adapter->hw; 5030 hw->back = adapter; 5031 adapter->port_num = hw->bus.func; 5032 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 5033 5034 err = pci_save_state(pdev); 5035 if (err) 5036 goto err_ioremap; 5037 5038 err = -EIO; 5039 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 5040 pci_resource_len(pdev, 0)); 5041 if (!adapter->io_addr) 5042 goto err_ioremap; 5043 5044 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 5045 hw->hw_addr = adapter->io_addr; 5046 5047 netdev->netdev_ops = &igc_netdev_ops; 5048 igc_ethtool_set_ops(netdev); 5049 netdev->watchdog_timeo = 5 * HZ; 5050 5051 netdev->mem_start = pci_resource_start(pdev, 0); 5052 netdev->mem_end = pci_resource_end(pdev, 0); 5053 5054 /* PCI config space info */ 5055 hw->vendor_id = pdev->vendor; 5056 hw->device_id = pdev->device; 5057 hw->revision_id = pdev->revision; 5058 hw->subsystem_vendor_id = pdev->subsystem_vendor; 5059 hw->subsystem_device_id = pdev->subsystem_device; 5060 5061 /* Copy the default MAC and PHY function pointers */ 5062 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 5063 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 5064 5065 /* Initialize skew-specific constants */ 5066 err = ei->get_invariants(hw); 5067 if (err) 5068 goto err_sw_init; 5069 5070 /* Add supported features to the features list*/ 5071 netdev->features |= NETIF_F_SG; 5072 netdev->features |= NETIF_F_TSO; 5073 netdev->features |= NETIF_F_TSO6; 5074 netdev->features |= NETIF_F_TSO_ECN; 5075 netdev->features |= NETIF_F_RXCSUM; 5076 netdev->features |= NETIF_F_HW_CSUM; 5077 netdev->features |= NETIF_F_SCTP_CRC; 5078 netdev->features |= NETIF_F_HW_TC; 5079 5080 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 5081 NETIF_F_GSO_GRE_CSUM | \ 5082 NETIF_F_GSO_IPXIP4 | \ 5083 NETIF_F_GSO_IPXIP6 | \ 5084 NETIF_F_GSO_UDP_TUNNEL | \ 5085 NETIF_F_GSO_UDP_TUNNEL_CSUM) 5086 5087 netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; 5088 netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; 5089 5090 /* setup the private structure */ 5091 err = igc_sw_init(adapter); 5092 if (err) 5093 goto err_sw_init; 5094 5095 /* copy netdev features into list of user selectable features */ 5096 netdev->hw_features |= NETIF_F_NTUPLE; 5097 netdev->hw_features |= netdev->features; 5098 5099 if (pci_using_dac) 5100 netdev->features |= NETIF_F_HIGHDMA; 5101 5102 /* MTU range: 68 - 9216 */ 5103 netdev->min_mtu = ETH_MIN_MTU; 5104 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 5105 5106 /* before reading the NVM, reset the controller to put the device in a 5107 * known good starting state 5108 */ 5109 hw->mac.ops.reset_hw(hw); 5110 5111 if (igc_get_flash_presence_i225(hw)) { 5112 if (hw->nvm.ops.validate(hw) < 0) { 5113 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 5114 err = -EIO; 5115 goto err_eeprom; 5116 } 5117 } 5118 5119 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 5120 /* copy the MAC address out of the NVM */ 5121 if (hw->mac.ops.read_mac_addr(hw)) 5122 dev_err(&pdev->dev, "NVM Read Error\n"); 5123 } 5124 5125 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 5126 5127 if (!is_valid_ether_addr(netdev->dev_addr)) { 5128 dev_err(&pdev->dev, "Invalid MAC Address\n"); 5129 err = -EIO; 5130 goto err_eeprom; 5131 } 5132 5133 /* configure RXPBSIZE and TXPBSIZE */ 5134 wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT); 5135 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); 5136 5137 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 5138 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 5139 5140 INIT_WORK(&adapter->reset_task, igc_reset_task); 5141 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 5142 5143 /* Initialize link properties that are user-changeable */ 5144 adapter->fc_autoneg = true; 5145 hw->mac.autoneg = true; 5146 hw->phy.autoneg_advertised = 0xaf; 5147 5148 hw->fc.requested_mode = igc_fc_default; 5149 hw->fc.current_mode = igc_fc_default; 5150 5151 /* By default, support wake on port A */ 5152 adapter->flags |= IGC_FLAG_WOL_SUPPORTED; 5153 5154 /* initialize the wol settings based on the eeprom settings */ 5155 if (adapter->flags & IGC_FLAG_WOL_SUPPORTED) 5156 adapter->wol |= IGC_WUFC_MAG; 5157 5158 device_set_wakeup_enable(&adapter->pdev->dev, 5159 adapter->flags & IGC_FLAG_WOL_SUPPORTED); 5160 5161 /* reset the hardware with the new settings */ 5162 igc_reset(adapter); 5163 5164 /* let the f/w know that the h/w is now under the control of the 5165 * driver. 5166 */ 5167 igc_get_hw_control(adapter); 5168 5169 strncpy(netdev->name, "eth%d", IFNAMSIZ); 5170 err = register_netdev(netdev); 5171 if (err) 5172 goto err_register; 5173 5174 /* carrier off reporting is important to ethtool even BEFORE open */ 5175 netif_carrier_off(netdev); 5176 5177 /* do hw tstamp init after resetting */ 5178 igc_ptp_init(adapter); 5179 5180 /* Check if Media Autosense is enabled */ 5181 adapter->ei = *ei; 5182 5183 /* print pcie link status and MAC address */ 5184 pcie_print_link_status(pdev); 5185 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 5186 5187 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); 5188 5189 pm_runtime_put_noidle(&pdev->dev); 5190 5191 return 0; 5192 5193 err_register: 5194 igc_release_hw_control(adapter); 5195 err_eeprom: 5196 if (!igc_check_reset_block(hw)) 5197 igc_reset_phy(hw); 5198 err_sw_init: 5199 igc_clear_interrupt_scheme(adapter); 5200 iounmap(adapter->io_addr); 5201 err_ioremap: 5202 free_netdev(netdev); 5203 err_alloc_etherdev: 5204 pci_release_mem_regions(pdev); 5205 err_pci_reg: 5206 err_dma: 5207 pci_disable_device(pdev); 5208 return err; 5209 } 5210 5211 /** 5212 * igc_remove - Device Removal Routine 5213 * @pdev: PCI device information struct 5214 * 5215 * igc_remove is called by the PCI subsystem to alert the driver 5216 * that it should release a PCI device. This could be caused by a 5217 * Hot-Plug event, or because the driver is going to be removed from 5218 * memory. 5219 */ 5220 static void igc_remove(struct pci_dev *pdev) 5221 { 5222 struct net_device *netdev = pci_get_drvdata(pdev); 5223 struct igc_adapter *adapter = netdev_priv(netdev); 5224 5225 pm_runtime_get_noresume(&pdev->dev); 5226 5227 igc_flush_nfc_rules(adapter); 5228 5229 igc_ptp_stop(adapter); 5230 5231 set_bit(__IGC_DOWN, &adapter->state); 5232 5233 del_timer_sync(&adapter->watchdog_timer); 5234 del_timer_sync(&adapter->phy_info_timer); 5235 5236 cancel_work_sync(&adapter->reset_task); 5237 cancel_work_sync(&adapter->watchdog_task); 5238 5239 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5240 * would have already happened in close and is redundant. 5241 */ 5242 igc_release_hw_control(adapter); 5243 unregister_netdev(netdev); 5244 5245 igc_clear_interrupt_scheme(adapter); 5246 pci_iounmap(pdev, adapter->io_addr); 5247 pci_release_mem_regions(pdev); 5248 5249 free_netdev(netdev); 5250 5251 pci_disable_pcie_error_reporting(pdev); 5252 5253 pci_disable_device(pdev); 5254 } 5255 5256 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, 5257 bool runtime) 5258 { 5259 struct net_device *netdev = pci_get_drvdata(pdev); 5260 struct igc_adapter *adapter = netdev_priv(netdev); 5261 u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol; 5262 struct igc_hw *hw = &adapter->hw; 5263 u32 ctrl, rctl, status; 5264 bool wake; 5265 5266 rtnl_lock(); 5267 netif_device_detach(netdev); 5268 5269 if (netif_running(netdev)) 5270 __igc_close(netdev, true); 5271 5272 igc_ptp_suspend(adapter); 5273 5274 igc_clear_interrupt_scheme(adapter); 5275 rtnl_unlock(); 5276 5277 status = rd32(IGC_STATUS); 5278 if (status & IGC_STATUS_LU) 5279 wufc &= ~IGC_WUFC_LNKC; 5280 5281 if (wufc) { 5282 igc_setup_rctl(adapter); 5283 igc_set_rx_mode(netdev); 5284 5285 /* turn on all-multi mode if wake on multicast is enabled */ 5286 if (wufc & IGC_WUFC_MC) { 5287 rctl = rd32(IGC_RCTL); 5288 rctl |= IGC_RCTL_MPE; 5289 wr32(IGC_RCTL, rctl); 5290 } 5291 5292 ctrl = rd32(IGC_CTRL); 5293 ctrl |= IGC_CTRL_ADVD3WUC; 5294 wr32(IGC_CTRL, ctrl); 5295 5296 /* Allow time for pending master requests to run */ 5297 igc_disable_pcie_master(hw); 5298 5299 wr32(IGC_WUC, IGC_WUC_PME_EN); 5300 wr32(IGC_WUFC, wufc); 5301 } else { 5302 wr32(IGC_WUC, 0); 5303 wr32(IGC_WUFC, 0); 5304 } 5305 5306 wake = wufc || adapter->en_mng_pt; 5307 if (!wake) 5308 igc_power_down_link(adapter); 5309 else 5310 igc_power_up_link(adapter); 5311 5312 if (enable_wake) 5313 *enable_wake = wake; 5314 5315 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5316 * would have already happened in close and is redundant. 5317 */ 5318 igc_release_hw_control(adapter); 5319 5320 pci_disable_device(pdev); 5321 5322 return 0; 5323 } 5324 5325 #ifdef CONFIG_PM 5326 static int __maybe_unused igc_runtime_suspend(struct device *dev) 5327 { 5328 return __igc_shutdown(to_pci_dev(dev), NULL, 1); 5329 } 5330 5331 static void igc_deliver_wake_packet(struct net_device *netdev) 5332 { 5333 struct igc_adapter *adapter = netdev_priv(netdev); 5334 struct igc_hw *hw = &adapter->hw; 5335 struct sk_buff *skb; 5336 u32 wupl; 5337 5338 wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK; 5339 5340 /* WUPM stores only the first 128 bytes of the wake packet. 5341 * Read the packet only if we have the whole thing. 5342 */ 5343 if (wupl == 0 || wupl > IGC_WUPM_BYTES) 5344 return; 5345 5346 skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES); 5347 if (!skb) 5348 return; 5349 5350 skb_put(skb, wupl); 5351 5352 /* Ensure reads are 32-bit aligned */ 5353 wupl = roundup(wupl, 4); 5354 5355 memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl); 5356 5357 skb->protocol = eth_type_trans(skb, netdev); 5358 netif_rx(skb); 5359 } 5360 5361 static int __maybe_unused igc_resume(struct device *dev) 5362 { 5363 struct pci_dev *pdev = to_pci_dev(dev); 5364 struct net_device *netdev = pci_get_drvdata(pdev); 5365 struct igc_adapter *adapter = netdev_priv(netdev); 5366 struct igc_hw *hw = &adapter->hw; 5367 u32 err, val; 5368 5369 pci_set_power_state(pdev, PCI_D0); 5370 pci_restore_state(pdev); 5371 pci_save_state(pdev); 5372 5373 if (!pci_device_is_present(pdev)) 5374 return -ENODEV; 5375 err = pci_enable_device_mem(pdev); 5376 if (err) { 5377 netdev_err(netdev, "Cannot enable PCI device from suspend\n"); 5378 return err; 5379 } 5380 pci_set_master(pdev); 5381 5382 pci_enable_wake(pdev, PCI_D3hot, 0); 5383 pci_enable_wake(pdev, PCI_D3cold, 0); 5384 5385 if (igc_init_interrupt_scheme(adapter, true)) { 5386 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5387 return -ENOMEM; 5388 } 5389 5390 igc_reset(adapter); 5391 5392 /* let the f/w know that the h/w is now under the control of the 5393 * driver. 5394 */ 5395 igc_get_hw_control(adapter); 5396 5397 val = rd32(IGC_WUS); 5398 if (val & WAKE_PKT_WUS) 5399 igc_deliver_wake_packet(netdev); 5400 5401 wr32(IGC_WUS, ~0); 5402 5403 rtnl_lock(); 5404 if (!err && netif_running(netdev)) 5405 err = __igc_open(netdev, true); 5406 5407 if (!err) 5408 netif_device_attach(netdev); 5409 rtnl_unlock(); 5410 5411 return err; 5412 } 5413 5414 static int __maybe_unused igc_runtime_resume(struct device *dev) 5415 { 5416 return igc_resume(dev); 5417 } 5418 5419 static int __maybe_unused igc_suspend(struct device *dev) 5420 { 5421 return __igc_shutdown(to_pci_dev(dev), NULL, 0); 5422 } 5423 5424 static int __maybe_unused igc_runtime_idle(struct device *dev) 5425 { 5426 struct net_device *netdev = dev_get_drvdata(dev); 5427 struct igc_adapter *adapter = netdev_priv(netdev); 5428 5429 if (!igc_has_link(adapter)) 5430 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 5431 5432 return -EBUSY; 5433 } 5434 #endif /* CONFIG_PM */ 5435 5436 static void igc_shutdown(struct pci_dev *pdev) 5437 { 5438 bool wake; 5439 5440 __igc_shutdown(pdev, &wake, 0); 5441 5442 if (system_state == SYSTEM_POWER_OFF) { 5443 pci_wake_from_d3(pdev, wake); 5444 pci_set_power_state(pdev, PCI_D3hot); 5445 } 5446 } 5447 5448 /** 5449 * igc_io_error_detected - called when PCI error is detected 5450 * @pdev: Pointer to PCI device 5451 * @state: The current PCI connection state 5452 * 5453 * This function is called after a PCI bus error affecting 5454 * this device has been detected. 5455 **/ 5456 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, 5457 pci_channel_state_t state) 5458 { 5459 struct net_device *netdev = pci_get_drvdata(pdev); 5460 struct igc_adapter *adapter = netdev_priv(netdev); 5461 5462 netif_device_detach(netdev); 5463 5464 if (state == pci_channel_io_perm_failure) 5465 return PCI_ERS_RESULT_DISCONNECT; 5466 5467 if (netif_running(netdev)) 5468 igc_down(adapter); 5469 pci_disable_device(pdev); 5470 5471 /* Request a slot reset. */ 5472 return PCI_ERS_RESULT_NEED_RESET; 5473 } 5474 5475 /** 5476 * igc_io_slot_reset - called after the PCI bus has been reset. 5477 * @pdev: Pointer to PCI device 5478 * 5479 * Restart the card from scratch, as if from a cold-boot. Implementation 5480 * resembles the first-half of the igc_resume routine. 5481 **/ 5482 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) 5483 { 5484 struct net_device *netdev = pci_get_drvdata(pdev); 5485 struct igc_adapter *adapter = netdev_priv(netdev); 5486 struct igc_hw *hw = &adapter->hw; 5487 pci_ers_result_t result; 5488 5489 if (pci_enable_device_mem(pdev)) { 5490 netdev_err(netdev, "Could not re-enable PCI device after reset\n"); 5491 result = PCI_ERS_RESULT_DISCONNECT; 5492 } else { 5493 pci_set_master(pdev); 5494 pci_restore_state(pdev); 5495 pci_save_state(pdev); 5496 5497 pci_enable_wake(pdev, PCI_D3hot, 0); 5498 pci_enable_wake(pdev, PCI_D3cold, 0); 5499 5500 /* In case of PCI error, adapter loses its HW address 5501 * so we should re-assign it here. 5502 */ 5503 hw->hw_addr = adapter->io_addr; 5504 5505 igc_reset(adapter); 5506 wr32(IGC_WUS, ~0); 5507 result = PCI_ERS_RESULT_RECOVERED; 5508 } 5509 5510 return result; 5511 } 5512 5513 /** 5514 * igc_io_resume - called when traffic can start to flow again. 5515 * @pdev: Pointer to PCI device 5516 * 5517 * This callback is called when the error recovery driver tells us that 5518 * its OK to resume normal operation. Implementation resembles the 5519 * second-half of the igc_resume routine. 5520 */ 5521 static void igc_io_resume(struct pci_dev *pdev) 5522 { 5523 struct net_device *netdev = pci_get_drvdata(pdev); 5524 struct igc_adapter *adapter = netdev_priv(netdev); 5525 5526 rtnl_lock(); 5527 if (netif_running(netdev)) { 5528 if (igc_open(netdev)) { 5529 netdev_err(netdev, "igc_open failed after reset\n"); 5530 return; 5531 } 5532 } 5533 5534 netif_device_attach(netdev); 5535 5536 /* let the f/w know that the h/w is now under the control of the 5537 * driver. 5538 */ 5539 igc_get_hw_control(adapter); 5540 rtnl_unlock(); 5541 } 5542 5543 static const struct pci_error_handlers igc_err_handler = { 5544 .error_detected = igc_io_error_detected, 5545 .slot_reset = igc_io_slot_reset, 5546 .resume = igc_io_resume, 5547 }; 5548 5549 #ifdef CONFIG_PM 5550 static const struct dev_pm_ops igc_pm_ops = { 5551 SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume) 5552 SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume, 5553 igc_runtime_idle) 5554 }; 5555 #endif 5556 5557 static struct pci_driver igc_driver = { 5558 .name = igc_driver_name, 5559 .id_table = igc_pci_tbl, 5560 .probe = igc_probe, 5561 .remove = igc_remove, 5562 #ifdef CONFIG_PM 5563 .driver.pm = &igc_pm_ops, 5564 #endif 5565 .shutdown = igc_shutdown, 5566 .err_handler = &igc_err_handler, 5567 }; 5568 5569 /** 5570 * igc_reinit_queues - return error 5571 * @adapter: pointer to adapter structure 5572 */ 5573 int igc_reinit_queues(struct igc_adapter *adapter) 5574 { 5575 struct net_device *netdev = adapter->netdev; 5576 int err = 0; 5577 5578 if (netif_running(netdev)) 5579 igc_close(netdev); 5580 5581 igc_reset_interrupt_capability(adapter); 5582 5583 if (igc_init_interrupt_scheme(adapter, true)) { 5584 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5585 return -ENOMEM; 5586 } 5587 5588 if (netif_running(netdev)) 5589 err = igc_open(netdev); 5590 5591 return err; 5592 } 5593 5594 /** 5595 * igc_get_hw_dev - return device 5596 * @hw: pointer to hardware structure 5597 * 5598 * used by hardware layer to print debugging information 5599 */ 5600 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 5601 { 5602 struct igc_adapter *adapter = hw->back; 5603 5604 return adapter->netdev; 5605 } 5606 5607 /** 5608 * igc_init_module - Driver Registration Routine 5609 * 5610 * igc_init_module is the first routine called when the driver is 5611 * loaded. All it does is register with the PCI subsystem. 5612 */ 5613 static int __init igc_init_module(void) 5614 { 5615 int ret; 5616 5617 pr_info("%s - version %s\n", 5618 igc_driver_string, igc_driver_version); 5619 5620 pr_info("%s\n", igc_copyright); 5621 5622 ret = pci_register_driver(&igc_driver); 5623 return ret; 5624 } 5625 5626 module_init(igc_init_module); 5627 5628 /** 5629 * igc_exit_module - Driver Exit Cleanup Routine 5630 * 5631 * igc_exit_module is called just before the driver is removed 5632 * from memory. 5633 */ 5634 static void __exit igc_exit_module(void) 5635 { 5636 pci_unregister_driver(&igc_driver); 5637 } 5638 5639 module_exit(igc_exit_module); 5640 /* igc_main.c */ 5641