1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/aer.h> 8 #include <linux/tcp.h> 9 #include <linux/udp.h> 10 #include <linux/ip.h> 11 #include <linux/pm_runtime.h> 12 #include <net/pkt_sched.h> 13 14 #include <net/ipv6.h> 15 16 #include "igc.h" 17 #include "igc_hw.h" 18 #include "igc_tsn.h" 19 20 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 21 22 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 23 24 static int debug = -1; 25 26 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 27 MODULE_DESCRIPTION(DRV_SUMMARY); 28 MODULE_LICENSE("GPL v2"); 29 module_param(debug, int, 0); 30 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 31 32 char igc_driver_name[] = "igc"; 33 static const char igc_driver_string[] = DRV_SUMMARY; 34 static const char igc_copyright[] = 35 "Copyright(c) 2018 Intel Corporation."; 36 37 static const struct igc_info *igc_info_tbl[] = { 38 [board_base] = &igc_base_info, 39 }; 40 41 static const struct pci_device_id igc_pci_tbl[] = { 42 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base }, 43 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base }, 44 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, 45 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, 46 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, 47 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base }, 48 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base }, 49 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base }, 50 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, 51 /* required last entry */ 52 {0, } 53 }; 54 55 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 56 57 enum latency_range { 58 lowest_latency = 0, 59 low_latency = 1, 60 bulk_latency = 2, 61 latency_invalid = 255 62 }; 63 64 void igc_reset(struct igc_adapter *adapter) 65 { 66 struct net_device *dev = adapter->netdev; 67 struct igc_hw *hw = &adapter->hw; 68 struct igc_fc_info *fc = &hw->fc; 69 u32 pba, hwm; 70 71 /* Repartition PBA for greater than 9k MTU if required */ 72 pba = IGC_PBA_34K; 73 74 /* flow control settings 75 * The high water mark must be low enough to fit one full frame 76 * after transmitting the pause frame. As such we must have enough 77 * space to allow for us to complete our current transmit and then 78 * receive the frame that is in progress from the link partner. 79 * Set it to: 80 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 81 */ 82 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 83 84 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 85 fc->low_water = fc->high_water - 16; 86 fc->pause_time = 0xFFFF; 87 fc->send_xon = 1; 88 fc->current_mode = fc->requested_mode; 89 90 hw->mac.ops.reset_hw(hw); 91 92 if (hw->mac.ops.init_hw(hw)) 93 netdev_err(dev, "Error on hardware initialization\n"); 94 95 /* Re-establish EEE setting */ 96 igc_set_eee_i225(hw, true, true, true); 97 98 if (!netif_running(adapter->netdev)) 99 igc_power_down_phy_copper_base(&adapter->hw); 100 101 /* Re-enable PTP, where applicable. */ 102 igc_ptp_reset(adapter); 103 104 /* Re-enable TSN offloading, where applicable. */ 105 igc_tsn_offload_apply(adapter); 106 107 igc_get_phy_info(hw); 108 } 109 110 /** 111 * igc_power_up_link - Power up the phy link 112 * @adapter: address of board private structure 113 */ 114 static void igc_power_up_link(struct igc_adapter *adapter) 115 { 116 igc_reset_phy(&adapter->hw); 117 118 igc_power_up_phy_copper(&adapter->hw); 119 120 igc_setup_link(&adapter->hw); 121 } 122 123 /** 124 * igc_release_hw_control - release control of the h/w to f/w 125 * @adapter: address of board private structure 126 * 127 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 128 * For ASF and Pass Through versions of f/w this means that the 129 * driver is no longer loaded. 130 */ 131 static void igc_release_hw_control(struct igc_adapter *adapter) 132 { 133 struct igc_hw *hw = &adapter->hw; 134 u32 ctrl_ext; 135 136 /* Let firmware take over control of h/w */ 137 ctrl_ext = rd32(IGC_CTRL_EXT); 138 wr32(IGC_CTRL_EXT, 139 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 140 } 141 142 /** 143 * igc_get_hw_control - get control of the h/w from f/w 144 * @adapter: address of board private structure 145 * 146 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 147 * For ASF and Pass Through versions of f/w this means that 148 * the driver is loaded. 149 */ 150 static void igc_get_hw_control(struct igc_adapter *adapter) 151 { 152 struct igc_hw *hw = &adapter->hw; 153 u32 ctrl_ext; 154 155 /* Let firmware know the driver has taken over */ 156 ctrl_ext = rd32(IGC_CTRL_EXT); 157 wr32(IGC_CTRL_EXT, 158 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 159 } 160 161 /** 162 * igc_clean_tx_ring - Free Tx Buffers 163 * @tx_ring: ring to be cleaned 164 */ 165 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 166 { 167 u16 i = tx_ring->next_to_clean; 168 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 169 170 while (i != tx_ring->next_to_use) { 171 union igc_adv_tx_desc *eop_desc, *tx_desc; 172 173 /* Free all the Tx ring sk_buffs */ 174 dev_kfree_skb_any(tx_buffer->skb); 175 176 /* unmap skb header data */ 177 dma_unmap_single(tx_ring->dev, 178 dma_unmap_addr(tx_buffer, dma), 179 dma_unmap_len(tx_buffer, len), 180 DMA_TO_DEVICE); 181 182 /* check for eop_desc to determine the end of the packet */ 183 eop_desc = tx_buffer->next_to_watch; 184 tx_desc = IGC_TX_DESC(tx_ring, i); 185 186 /* unmap remaining buffers */ 187 while (tx_desc != eop_desc) { 188 tx_buffer++; 189 tx_desc++; 190 i++; 191 if (unlikely(i == tx_ring->count)) { 192 i = 0; 193 tx_buffer = tx_ring->tx_buffer_info; 194 tx_desc = IGC_TX_DESC(tx_ring, 0); 195 } 196 197 /* unmap any remaining paged data */ 198 if (dma_unmap_len(tx_buffer, len)) 199 dma_unmap_page(tx_ring->dev, 200 dma_unmap_addr(tx_buffer, dma), 201 dma_unmap_len(tx_buffer, len), 202 DMA_TO_DEVICE); 203 } 204 205 /* move us one more past the eop_desc for start of next pkt */ 206 tx_buffer++; 207 i++; 208 if (unlikely(i == tx_ring->count)) { 209 i = 0; 210 tx_buffer = tx_ring->tx_buffer_info; 211 } 212 } 213 214 /* reset BQL for queue */ 215 netdev_tx_reset_queue(txring_txq(tx_ring)); 216 217 /* reset next_to_use and next_to_clean */ 218 tx_ring->next_to_use = 0; 219 tx_ring->next_to_clean = 0; 220 } 221 222 /** 223 * igc_free_tx_resources - Free Tx Resources per Queue 224 * @tx_ring: Tx descriptor ring for a specific queue 225 * 226 * Free all transmit software resources 227 */ 228 void igc_free_tx_resources(struct igc_ring *tx_ring) 229 { 230 igc_clean_tx_ring(tx_ring); 231 232 vfree(tx_ring->tx_buffer_info); 233 tx_ring->tx_buffer_info = NULL; 234 235 /* if not set, then don't free */ 236 if (!tx_ring->desc) 237 return; 238 239 dma_free_coherent(tx_ring->dev, tx_ring->size, 240 tx_ring->desc, tx_ring->dma); 241 242 tx_ring->desc = NULL; 243 } 244 245 /** 246 * igc_free_all_tx_resources - Free Tx Resources for All Queues 247 * @adapter: board private structure 248 * 249 * Free all transmit software resources 250 */ 251 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 252 { 253 int i; 254 255 for (i = 0; i < adapter->num_tx_queues; i++) 256 igc_free_tx_resources(adapter->tx_ring[i]); 257 } 258 259 /** 260 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 261 * @adapter: board private structure 262 */ 263 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 264 { 265 int i; 266 267 for (i = 0; i < adapter->num_tx_queues; i++) 268 if (adapter->tx_ring[i]) 269 igc_clean_tx_ring(adapter->tx_ring[i]); 270 } 271 272 /** 273 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 274 * @tx_ring: tx descriptor ring (for a specific queue) to setup 275 * 276 * Return 0 on success, negative on failure 277 */ 278 int igc_setup_tx_resources(struct igc_ring *tx_ring) 279 { 280 struct net_device *ndev = tx_ring->netdev; 281 struct device *dev = tx_ring->dev; 282 int size = 0; 283 284 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 285 tx_ring->tx_buffer_info = vzalloc(size); 286 if (!tx_ring->tx_buffer_info) 287 goto err; 288 289 /* round up to nearest 4K */ 290 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 291 tx_ring->size = ALIGN(tx_ring->size, 4096); 292 293 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 294 &tx_ring->dma, GFP_KERNEL); 295 296 if (!tx_ring->desc) 297 goto err; 298 299 tx_ring->next_to_use = 0; 300 tx_ring->next_to_clean = 0; 301 302 return 0; 303 304 err: 305 vfree(tx_ring->tx_buffer_info); 306 netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n"); 307 return -ENOMEM; 308 } 309 310 /** 311 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 312 * @adapter: board private structure 313 * 314 * Return 0 on success, negative on failure 315 */ 316 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 317 { 318 struct net_device *dev = adapter->netdev; 319 int i, err = 0; 320 321 for (i = 0; i < adapter->num_tx_queues; i++) { 322 err = igc_setup_tx_resources(adapter->tx_ring[i]); 323 if (err) { 324 netdev_err(dev, "Error on Tx queue %u setup\n", i); 325 for (i--; i >= 0; i--) 326 igc_free_tx_resources(adapter->tx_ring[i]); 327 break; 328 } 329 } 330 331 return err; 332 } 333 334 /** 335 * igc_clean_rx_ring - Free Rx Buffers per Queue 336 * @rx_ring: ring to free buffers from 337 */ 338 static void igc_clean_rx_ring(struct igc_ring *rx_ring) 339 { 340 u16 i = rx_ring->next_to_clean; 341 342 dev_kfree_skb(rx_ring->skb); 343 rx_ring->skb = NULL; 344 345 /* Free all the Rx ring sk_buffs */ 346 while (i != rx_ring->next_to_alloc) { 347 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 348 349 /* Invalidate cache lines that may have been written to by 350 * device so that we avoid corrupting memory. 351 */ 352 dma_sync_single_range_for_cpu(rx_ring->dev, 353 buffer_info->dma, 354 buffer_info->page_offset, 355 igc_rx_bufsz(rx_ring), 356 DMA_FROM_DEVICE); 357 358 /* free resources associated with mapping */ 359 dma_unmap_page_attrs(rx_ring->dev, 360 buffer_info->dma, 361 igc_rx_pg_size(rx_ring), 362 DMA_FROM_DEVICE, 363 IGC_RX_DMA_ATTR); 364 __page_frag_cache_drain(buffer_info->page, 365 buffer_info->pagecnt_bias); 366 367 i++; 368 if (i == rx_ring->count) 369 i = 0; 370 } 371 372 rx_ring->next_to_alloc = 0; 373 rx_ring->next_to_clean = 0; 374 rx_ring->next_to_use = 0; 375 } 376 377 /** 378 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 379 * @adapter: board private structure 380 */ 381 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 382 { 383 int i; 384 385 for (i = 0; i < adapter->num_rx_queues; i++) 386 if (adapter->rx_ring[i]) 387 igc_clean_rx_ring(adapter->rx_ring[i]); 388 } 389 390 /** 391 * igc_free_rx_resources - Free Rx Resources 392 * @rx_ring: ring to clean the resources from 393 * 394 * Free all receive software resources 395 */ 396 void igc_free_rx_resources(struct igc_ring *rx_ring) 397 { 398 igc_clean_rx_ring(rx_ring); 399 400 vfree(rx_ring->rx_buffer_info); 401 rx_ring->rx_buffer_info = NULL; 402 403 /* if not set, then don't free */ 404 if (!rx_ring->desc) 405 return; 406 407 dma_free_coherent(rx_ring->dev, rx_ring->size, 408 rx_ring->desc, rx_ring->dma); 409 410 rx_ring->desc = NULL; 411 } 412 413 /** 414 * igc_free_all_rx_resources - Free Rx Resources for All Queues 415 * @adapter: board private structure 416 * 417 * Free all receive software resources 418 */ 419 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 420 { 421 int i; 422 423 for (i = 0; i < adapter->num_rx_queues; i++) 424 igc_free_rx_resources(adapter->rx_ring[i]); 425 } 426 427 /** 428 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 429 * @rx_ring: rx descriptor ring (for a specific queue) to setup 430 * 431 * Returns 0 on success, negative on failure 432 */ 433 int igc_setup_rx_resources(struct igc_ring *rx_ring) 434 { 435 struct net_device *ndev = rx_ring->netdev; 436 struct device *dev = rx_ring->dev; 437 int size, desc_len; 438 439 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 440 rx_ring->rx_buffer_info = vzalloc(size); 441 if (!rx_ring->rx_buffer_info) 442 goto err; 443 444 desc_len = sizeof(union igc_adv_rx_desc); 445 446 /* Round up to nearest 4K */ 447 rx_ring->size = rx_ring->count * desc_len; 448 rx_ring->size = ALIGN(rx_ring->size, 4096); 449 450 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 451 &rx_ring->dma, GFP_KERNEL); 452 453 if (!rx_ring->desc) 454 goto err; 455 456 rx_ring->next_to_alloc = 0; 457 rx_ring->next_to_clean = 0; 458 rx_ring->next_to_use = 0; 459 460 return 0; 461 462 err: 463 vfree(rx_ring->rx_buffer_info); 464 rx_ring->rx_buffer_info = NULL; 465 netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n"); 466 return -ENOMEM; 467 } 468 469 /** 470 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 471 * (Descriptors) for all queues 472 * @adapter: board private structure 473 * 474 * Return 0 on success, negative on failure 475 */ 476 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 477 { 478 struct net_device *dev = adapter->netdev; 479 int i, err = 0; 480 481 for (i = 0; i < adapter->num_rx_queues; i++) { 482 err = igc_setup_rx_resources(adapter->rx_ring[i]); 483 if (err) { 484 netdev_err(dev, "Error on Rx queue %u setup\n", i); 485 for (i--; i >= 0; i--) 486 igc_free_rx_resources(adapter->rx_ring[i]); 487 break; 488 } 489 } 490 491 return err; 492 } 493 494 /** 495 * igc_configure_rx_ring - Configure a receive ring after Reset 496 * @adapter: board private structure 497 * @ring: receive ring to be configured 498 * 499 * Configure the Rx unit of the MAC after a reset. 500 */ 501 static void igc_configure_rx_ring(struct igc_adapter *adapter, 502 struct igc_ring *ring) 503 { 504 struct igc_hw *hw = &adapter->hw; 505 union igc_adv_rx_desc *rx_desc; 506 int reg_idx = ring->reg_idx; 507 u32 srrctl = 0, rxdctl = 0; 508 u64 rdba = ring->dma; 509 510 /* disable the queue */ 511 wr32(IGC_RXDCTL(reg_idx), 0); 512 513 /* Set DMA base address registers */ 514 wr32(IGC_RDBAL(reg_idx), 515 rdba & 0x00000000ffffffffULL); 516 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 517 wr32(IGC_RDLEN(reg_idx), 518 ring->count * sizeof(union igc_adv_rx_desc)); 519 520 /* initialize head and tail */ 521 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 522 wr32(IGC_RDH(reg_idx), 0); 523 writel(0, ring->tail); 524 525 /* reset next-to- use/clean to place SW in sync with hardware */ 526 ring->next_to_clean = 0; 527 ring->next_to_use = 0; 528 529 /* set descriptor configuration */ 530 srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; 531 if (ring_uses_large_buffer(ring)) 532 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 533 else 534 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 535 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 536 537 wr32(IGC_SRRCTL(reg_idx), srrctl); 538 539 rxdctl |= IGC_RX_PTHRESH; 540 rxdctl |= IGC_RX_HTHRESH << 8; 541 rxdctl |= IGC_RX_WTHRESH << 16; 542 543 /* initialize rx_buffer_info */ 544 memset(ring->rx_buffer_info, 0, 545 sizeof(struct igc_rx_buffer) * ring->count); 546 547 /* initialize Rx descriptor 0 */ 548 rx_desc = IGC_RX_DESC(ring, 0); 549 rx_desc->wb.upper.length = 0; 550 551 /* enable receive descriptor fetching */ 552 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 553 554 wr32(IGC_RXDCTL(reg_idx), rxdctl); 555 } 556 557 /** 558 * igc_configure_rx - Configure receive Unit after Reset 559 * @adapter: board private structure 560 * 561 * Configure the Rx unit of the MAC after a reset. 562 */ 563 static void igc_configure_rx(struct igc_adapter *adapter) 564 { 565 int i; 566 567 /* Setup the HW Rx Head and Tail Descriptor Pointers and 568 * the Base and Length of the Rx Descriptor Ring 569 */ 570 for (i = 0; i < adapter->num_rx_queues; i++) 571 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 572 } 573 574 /** 575 * igc_configure_tx_ring - Configure transmit ring after Reset 576 * @adapter: board private structure 577 * @ring: tx ring to configure 578 * 579 * Configure a transmit ring after a reset. 580 */ 581 static void igc_configure_tx_ring(struct igc_adapter *adapter, 582 struct igc_ring *ring) 583 { 584 struct igc_hw *hw = &adapter->hw; 585 int reg_idx = ring->reg_idx; 586 u64 tdba = ring->dma; 587 u32 txdctl = 0; 588 589 /* disable the queue */ 590 wr32(IGC_TXDCTL(reg_idx), 0); 591 wrfl(); 592 mdelay(10); 593 594 wr32(IGC_TDLEN(reg_idx), 595 ring->count * sizeof(union igc_adv_tx_desc)); 596 wr32(IGC_TDBAL(reg_idx), 597 tdba & 0x00000000ffffffffULL); 598 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 599 600 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 601 wr32(IGC_TDH(reg_idx), 0); 602 writel(0, ring->tail); 603 604 txdctl |= IGC_TX_PTHRESH; 605 txdctl |= IGC_TX_HTHRESH << 8; 606 txdctl |= IGC_TX_WTHRESH << 16; 607 608 txdctl |= IGC_TXDCTL_QUEUE_ENABLE; 609 wr32(IGC_TXDCTL(reg_idx), txdctl); 610 } 611 612 /** 613 * igc_configure_tx - Configure transmit Unit after Reset 614 * @adapter: board private structure 615 * 616 * Configure the Tx unit of the MAC after a reset. 617 */ 618 static void igc_configure_tx(struct igc_adapter *adapter) 619 { 620 int i; 621 622 for (i = 0; i < adapter->num_tx_queues; i++) 623 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 624 } 625 626 /** 627 * igc_setup_mrqc - configure the multiple receive queue control registers 628 * @adapter: Board private structure 629 */ 630 static void igc_setup_mrqc(struct igc_adapter *adapter) 631 { 632 struct igc_hw *hw = &adapter->hw; 633 u32 j, num_rx_queues; 634 u32 mrqc, rxcsum; 635 u32 rss_key[10]; 636 637 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 638 for (j = 0; j < 10; j++) 639 wr32(IGC_RSSRK(j), rss_key[j]); 640 641 num_rx_queues = adapter->rss_queues; 642 643 if (adapter->rss_indir_tbl_init != num_rx_queues) { 644 for (j = 0; j < IGC_RETA_SIZE; j++) 645 adapter->rss_indir_tbl[j] = 646 (j * num_rx_queues) / IGC_RETA_SIZE; 647 adapter->rss_indir_tbl_init = num_rx_queues; 648 } 649 igc_write_rss_indir_tbl(adapter); 650 651 /* Disable raw packet checksumming so that RSS hash is placed in 652 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 653 * offloads as they are enabled by default 654 */ 655 rxcsum = rd32(IGC_RXCSUM); 656 rxcsum |= IGC_RXCSUM_PCSD; 657 658 /* Enable Receive Checksum Offload for SCTP */ 659 rxcsum |= IGC_RXCSUM_CRCOFL; 660 661 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 662 wr32(IGC_RXCSUM, rxcsum); 663 664 /* Generate RSS hash based on packet types, TCP/UDP 665 * port numbers and/or IPv4/v6 src and dst addresses 666 */ 667 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 668 IGC_MRQC_RSS_FIELD_IPV4_TCP | 669 IGC_MRQC_RSS_FIELD_IPV6 | 670 IGC_MRQC_RSS_FIELD_IPV6_TCP | 671 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 672 673 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 674 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 675 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 676 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 677 678 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 679 680 wr32(IGC_MRQC, mrqc); 681 } 682 683 /** 684 * igc_setup_rctl - configure the receive control registers 685 * @adapter: Board private structure 686 */ 687 static void igc_setup_rctl(struct igc_adapter *adapter) 688 { 689 struct igc_hw *hw = &adapter->hw; 690 u32 rctl; 691 692 rctl = rd32(IGC_RCTL); 693 694 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 695 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 696 697 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 698 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 699 700 /* enable stripping of CRC. Newer features require 701 * that the HW strips the CRC. 702 */ 703 rctl |= IGC_RCTL_SECRC; 704 705 /* disable store bad packets and clear size bits. */ 706 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 707 708 /* enable LPE to allow for reception of jumbo frames */ 709 rctl |= IGC_RCTL_LPE; 710 711 /* disable queue 0 to prevent tail write w/o re-config */ 712 wr32(IGC_RXDCTL(0), 0); 713 714 /* This is useful for sniffing bad packets. */ 715 if (adapter->netdev->features & NETIF_F_RXALL) { 716 /* UPE and MPE will be handled by normal PROMISC logic 717 * in set_rx_mode 718 */ 719 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 720 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 721 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 722 723 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 724 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 725 } 726 727 wr32(IGC_RCTL, rctl); 728 } 729 730 /** 731 * igc_setup_tctl - configure the transmit control registers 732 * @adapter: Board private structure 733 */ 734 static void igc_setup_tctl(struct igc_adapter *adapter) 735 { 736 struct igc_hw *hw = &adapter->hw; 737 u32 tctl; 738 739 /* disable queue 0 which icould be enabled by default */ 740 wr32(IGC_TXDCTL(0), 0); 741 742 /* Program the Transmit Control Register */ 743 tctl = rd32(IGC_TCTL); 744 tctl &= ~IGC_TCTL_CT; 745 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 746 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 747 748 /* Enable transmits */ 749 tctl |= IGC_TCTL_EN; 750 751 wr32(IGC_TCTL, tctl); 752 } 753 754 /** 755 * igc_set_mac_filter_hw() - Set MAC address filter in hardware 756 * @adapter: Pointer to adapter where the filter should be set 757 * @index: Filter index 758 * @type: MAC address filter type (source or destination) 759 * @addr: MAC address 760 * @queue: If non-negative, queue assignment feature is enabled and frames 761 * matching the filter are enqueued onto 'queue'. Otherwise, queue 762 * assignment is disabled. 763 */ 764 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, 765 enum igc_mac_filter_type type, 766 const u8 *addr, int queue) 767 { 768 struct net_device *dev = adapter->netdev; 769 struct igc_hw *hw = &adapter->hw; 770 u32 ral, rah; 771 772 if (WARN_ON(index >= hw->mac.rar_entry_count)) 773 return; 774 775 ral = le32_to_cpup((__le32 *)(addr)); 776 rah = le16_to_cpup((__le16 *)(addr + 4)); 777 778 if (type == IGC_MAC_FILTER_TYPE_SRC) { 779 rah &= ~IGC_RAH_ASEL_MASK; 780 rah |= IGC_RAH_ASEL_SRC_ADDR; 781 } 782 783 if (queue >= 0) { 784 rah &= ~IGC_RAH_QSEL_MASK; 785 rah |= (queue << IGC_RAH_QSEL_SHIFT); 786 rah |= IGC_RAH_QSEL_ENABLE; 787 } 788 789 rah |= IGC_RAH_AV; 790 791 wr32(IGC_RAL(index), ral); 792 wr32(IGC_RAH(index), rah); 793 794 netdev_dbg(dev, "MAC address filter set in HW: index %d", index); 795 } 796 797 /** 798 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware 799 * @adapter: Pointer to adapter where the filter should be cleared 800 * @index: Filter index 801 */ 802 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) 803 { 804 struct net_device *dev = adapter->netdev; 805 struct igc_hw *hw = &adapter->hw; 806 807 if (WARN_ON(index >= hw->mac.rar_entry_count)) 808 return; 809 810 wr32(IGC_RAL(index), 0); 811 wr32(IGC_RAH(index), 0); 812 813 netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index); 814 } 815 816 /* Set default MAC address for the PF in the first RAR entry */ 817 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 818 { 819 struct net_device *dev = adapter->netdev; 820 u8 *addr = adapter->hw.mac.addr; 821 822 netdev_dbg(dev, "Set default MAC address filter: address %pM", addr); 823 824 igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1); 825 } 826 827 /** 828 * igc_set_mac - Change the Ethernet Address of the NIC 829 * @netdev: network interface device structure 830 * @p: pointer to an address structure 831 * 832 * Returns 0 on success, negative on failure 833 */ 834 static int igc_set_mac(struct net_device *netdev, void *p) 835 { 836 struct igc_adapter *adapter = netdev_priv(netdev); 837 struct igc_hw *hw = &adapter->hw; 838 struct sockaddr *addr = p; 839 840 if (!is_valid_ether_addr(addr->sa_data)) 841 return -EADDRNOTAVAIL; 842 843 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 844 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 845 846 /* set the correct pool for the new PF MAC address in entry 0 */ 847 igc_set_default_mac_filter(adapter); 848 849 return 0; 850 } 851 852 /** 853 * igc_write_mc_addr_list - write multicast addresses to MTA 854 * @netdev: network interface device structure 855 * 856 * Writes multicast address list to the MTA hash table. 857 * Returns: -ENOMEM on failure 858 * 0 on no addresses written 859 * X on writing X addresses to MTA 860 **/ 861 static int igc_write_mc_addr_list(struct net_device *netdev) 862 { 863 struct igc_adapter *adapter = netdev_priv(netdev); 864 struct igc_hw *hw = &adapter->hw; 865 struct netdev_hw_addr *ha; 866 u8 *mta_list; 867 int i; 868 869 if (netdev_mc_empty(netdev)) { 870 /* nothing to program, so clear mc list */ 871 igc_update_mc_addr_list(hw, NULL, 0); 872 return 0; 873 } 874 875 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 876 if (!mta_list) 877 return -ENOMEM; 878 879 /* The shared function expects a packed array of only addresses. */ 880 i = 0; 881 netdev_for_each_mc_addr(ha, netdev) 882 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 883 884 igc_update_mc_addr_list(hw, mta_list, i); 885 kfree(mta_list); 886 887 return netdev_mc_count(netdev); 888 } 889 890 static __le32 igc_tx_launchtime(struct igc_adapter *adapter, ktime_t txtime) 891 { 892 ktime_t cycle_time = adapter->cycle_time; 893 ktime_t base_time = adapter->base_time; 894 u32 launchtime; 895 896 /* FIXME: when using ETF together with taprio, we may have a 897 * case where 'delta' is larger than the cycle_time, this may 898 * cause problems if we don't read the current value of 899 * IGC_BASET, as the value writen into the launchtime 900 * descriptor field may be misinterpreted. 901 */ 902 div_s64_rem(ktime_sub_ns(txtime, base_time), cycle_time, &launchtime); 903 904 return cpu_to_le32(launchtime); 905 } 906 907 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, 908 struct igc_tx_buffer *first, 909 u32 vlan_macip_lens, u32 type_tucmd, 910 u32 mss_l4len_idx) 911 { 912 struct igc_adv_tx_context_desc *context_desc; 913 u16 i = tx_ring->next_to_use; 914 915 context_desc = IGC_TX_CTXTDESC(tx_ring, i); 916 917 i++; 918 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 919 920 /* set bits to identify this as an advanced context descriptor */ 921 type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 922 923 /* For i225, context index must be unique per ring. */ 924 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 925 mss_l4len_idx |= tx_ring->reg_idx << 4; 926 927 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 928 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 929 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 930 931 /* We assume there is always a valid Tx time available. Invalid times 932 * should have been handled by the upper layers. 933 */ 934 if (tx_ring->launchtime_enable) { 935 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 936 ktime_t txtime = first->skb->tstamp; 937 938 first->skb->tstamp = ktime_set(0, 0); 939 context_desc->launch_time = igc_tx_launchtime(adapter, 940 txtime); 941 } else { 942 context_desc->launch_time = 0; 943 } 944 } 945 946 static inline bool igc_ipv6_csum_is_sctp(struct sk_buff *skb) 947 { 948 unsigned int offset = 0; 949 950 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 951 952 return offset == skb_checksum_start_offset(skb); 953 } 954 955 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first) 956 { 957 struct sk_buff *skb = first->skb; 958 u32 vlan_macip_lens = 0; 959 u32 type_tucmd = 0; 960 961 if (skb->ip_summed != CHECKSUM_PARTIAL) { 962 csum_failed: 963 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && 964 !tx_ring->launchtime_enable) 965 return; 966 goto no_csum; 967 } 968 969 switch (skb->csum_offset) { 970 case offsetof(struct tcphdr, check): 971 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 972 fallthrough; 973 case offsetof(struct udphdr, check): 974 break; 975 case offsetof(struct sctphdr, checksum): 976 /* validate that this is actually an SCTP request */ 977 if ((first->protocol == htons(ETH_P_IP) && 978 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 979 (first->protocol == htons(ETH_P_IPV6) && 980 igc_ipv6_csum_is_sctp(skb))) { 981 type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; 982 break; 983 } 984 fallthrough; 985 default: 986 skb_checksum_help(skb); 987 goto csum_failed; 988 } 989 990 /* update TX checksum flag */ 991 first->tx_flags |= IGC_TX_FLAGS_CSUM; 992 vlan_macip_lens = skb_checksum_start_offset(skb) - 993 skb_network_offset(skb); 994 no_csum: 995 vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; 996 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 997 998 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0); 999 } 1000 1001 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1002 { 1003 struct net_device *netdev = tx_ring->netdev; 1004 1005 netif_stop_subqueue(netdev, tx_ring->queue_index); 1006 1007 /* memory barriier comment */ 1008 smp_mb(); 1009 1010 /* We need to check again in a case another CPU has just 1011 * made room available. 1012 */ 1013 if (igc_desc_unused(tx_ring) < size) 1014 return -EBUSY; 1015 1016 /* A reprieve! */ 1017 netif_wake_subqueue(netdev, tx_ring->queue_index); 1018 1019 u64_stats_update_begin(&tx_ring->tx_syncp2); 1020 tx_ring->tx_stats.restart_queue2++; 1021 u64_stats_update_end(&tx_ring->tx_syncp2); 1022 1023 return 0; 1024 } 1025 1026 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1027 { 1028 if (igc_desc_unused(tx_ring) >= size) 1029 return 0; 1030 return __igc_maybe_stop_tx(tx_ring, size); 1031 } 1032 1033 #define IGC_SET_FLAG(_input, _flag, _result) \ 1034 (((_flag) <= (_result)) ? \ 1035 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ 1036 ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) 1037 1038 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 1039 { 1040 /* set type for advanced descriptor with frame checksum insertion */ 1041 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 1042 IGC_ADVTXD_DCMD_DEXT | 1043 IGC_ADVTXD_DCMD_IFCS; 1044 1045 /* set segmentation bits for TSO */ 1046 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, 1047 (IGC_ADVTXD_DCMD_TSE)); 1048 1049 /* set timestamp bit if present */ 1050 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, 1051 (IGC_ADVTXD_MAC_TSTAMP)); 1052 1053 return cmd_type; 1054 } 1055 1056 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 1057 union igc_adv_tx_desc *tx_desc, 1058 u32 tx_flags, unsigned int paylen) 1059 { 1060 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 1061 1062 /* insert L4 checksum */ 1063 olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) * 1064 ((IGC_TXD_POPTS_TXSM << 8) / 1065 IGC_TX_FLAGS_CSUM); 1066 1067 /* insert IPv4 checksum */ 1068 olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) * 1069 (((IGC_TXD_POPTS_IXSM << 8)) / 1070 IGC_TX_FLAGS_IPV4); 1071 1072 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 1073 } 1074 1075 static int igc_tx_map(struct igc_ring *tx_ring, 1076 struct igc_tx_buffer *first, 1077 const u8 hdr_len) 1078 { 1079 struct sk_buff *skb = first->skb; 1080 struct igc_tx_buffer *tx_buffer; 1081 union igc_adv_tx_desc *tx_desc; 1082 u32 tx_flags = first->tx_flags; 1083 skb_frag_t *frag; 1084 u16 i = tx_ring->next_to_use; 1085 unsigned int data_len, size; 1086 dma_addr_t dma; 1087 u32 cmd_type = igc_tx_cmd_type(skb, tx_flags); 1088 1089 tx_desc = IGC_TX_DESC(tx_ring, i); 1090 1091 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 1092 1093 size = skb_headlen(skb); 1094 data_len = skb->data_len; 1095 1096 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1097 1098 tx_buffer = first; 1099 1100 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1101 if (dma_mapping_error(tx_ring->dev, dma)) 1102 goto dma_error; 1103 1104 /* record length, and DMA address */ 1105 dma_unmap_len_set(tx_buffer, len, size); 1106 dma_unmap_addr_set(tx_buffer, dma, dma); 1107 1108 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1109 1110 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 1111 tx_desc->read.cmd_type_len = 1112 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 1113 1114 i++; 1115 tx_desc++; 1116 if (i == tx_ring->count) { 1117 tx_desc = IGC_TX_DESC(tx_ring, 0); 1118 i = 0; 1119 } 1120 tx_desc->read.olinfo_status = 0; 1121 1122 dma += IGC_MAX_DATA_PER_TXD; 1123 size -= IGC_MAX_DATA_PER_TXD; 1124 1125 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1126 } 1127 1128 if (likely(!data_len)) 1129 break; 1130 1131 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 1132 1133 i++; 1134 tx_desc++; 1135 if (i == tx_ring->count) { 1136 tx_desc = IGC_TX_DESC(tx_ring, 0); 1137 i = 0; 1138 } 1139 tx_desc->read.olinfo_status = 0; 1140 1141 size = skb_frag_size(frag); 1142 data_len -= size; 1143 1144 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 1145 size, DMA_TO_DEVICE); 1146 1147 tx_buffer = &tx_ring->tx_buffer_info[i]; 1148 } 1149 1150 /* write last descriptor with RS and EOP bits */ 1151 cmd_type |= size | IGC_TXD_DCMD; 1152 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1153 1154 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1155 1156 /* set the timestamp */ 1157 first->time_stamp = jiffies; 1158 1159 skb_tx_timestamp(skb); 1160 1161 /* Force memory writes to complete before letting h/w know there 1162 * are new descriptors to fetch. (Only applicable for weak-ordered 1163 * memory model archs, such as IA-64). 1164 * 1165 * We also need this memory barrier to make certain all of the 1166 * status bits have been updated before next_to_watch is written. 1167 */ 1168 wmb(); 1169 1170 /* set next_to_watch value indicating a packet is present */ 1171 first->next_to_watch = tx_desc; 1172 1173 i++; 1174 if (i == tx_ring->count) 1175 i = 0; 1176 1177 tx_ring->next_to_use = i; 1178 1179 /* Make sure there is space in the ring for the next send. */ 1180 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 1181 1182 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1183 writel(i, tx_ring->tail); 1184 } 1185 1186 return 0; 1187 dma_error: 1188 netdev_err(tx_ring->netdev, "TX DMA map failed\n"); 1189 tx_buffer = &tx_ring->tx_buffer_info[i]; 1190 1191 /* clear dma mappings for failed tx_buffer_info map */ 1192 while (tx_buffer != first) { 1193 if (dma_unmap_len(tx_buffer, len)) 1194 dma_unmap_page(tx_ring->dev, 1195 dma_unmap_addr(tx_buffer, dma), 1196 dma_unmap_len(tx_buffer, len), 1197 DMA_TO_DEVICE); 1198 dma_unmap_len_set(tx_buffer, len, 0); 1199 1200 if (i-- == 0) 1201 i += tx_ring->count; 1202 tx_buffer = &tx_ring->tx_buffer_info[i]; 1203 } 1204 1205 if (dma_unmap_len(tx_buffer, len)) 1206 dma_unmap_single(tx_ring->dev, 1207 dma_unmap_addr(tx_buffer, dma), 1208 dma_unmap_len(tx_buffer, len), 1209 DMA_TO_DEVICE); 1210 dma_unmap_len_set(tx_buffer, len, 0); 1211 1212 dev_kfree_skb_any(tx_buffer->skb); 1213 tx_buffer->skb = NULL; 1214 1215 tx_ring->next_to_use = i; 1216 1217 return -1; 1218 } 1219 1220 static int igc_tso(struct igc_ring *tx_ring, 1221 struct igc_tx_buffer *first, 1222 u8 *hdr_len) 1223 { 1224 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 1225 struct sk_buff *skb = first->skb; 1226 union { 1227 struct iphdr *v4; 1228 struct ipv6hdr *v6; 1229 unsigned char *hdr; 1230 } ip; 1231 union { 1232 struct tcphdr *tcp; 1233 struct udphdr *udp; 1234 unsigned char *hdr; 1235 } l4; 1236 u32 paylen, l4_offset; 1237 int err; 1238 1239 if (skb->ip_summed != CHECKSUM_PARTIAL) 1240 return 0; 1241 1242 if (!skb_is_gso(skb)) 1243 return 0; 1244 1245 err = skb_cow_head(skb, 0); 1246 if (err < 0) 1247 return err; 1248 1249 ip.hdr = skb_network_header(skb); 1250 l4.hdr = skb_checksum_start(skb); 1251 1252 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 1253 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 1254 1255 /* initialize outer IP header fields */ 1256 if (ip.v4->version == 4) { 1257 unsigned char *csum_start = skb_checksum_start(skb); 1258 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 1259 1260 /* IP header will have to cancel out any data that 1261 * is not a part of the outer IP header 1262 */ 1263 ip.v4->check = csum_fold(csum_partial(trans_start, 1264 csum_start - trans_start, 1265 0)); 1266 type_tucmd |= IGC_ADVTXD_TUCMD_IPV4; 1267 1268 ip.v4->tot_len = 0; 1269 first->tx_flags |= IGC_TX_FLAGS_TSO | 1270 IGC_TX_FLAGS_CSUM | 1271 IGC_TX_FLAGS_IPV4; 1272 } else { 1273 ip.v6->payload_len = 0; 1274 first->tx_flags |= IGC_TX_FLAGS_TSO | 1275 IGC_TX_FLAGS_CSUM; 1276 } 1277 1278 /* determine offset of inner transport header */ 1279 l4_offset = l4.hdr - skb->data; 1280 1281 /* remove payload length from inner checksum */ 1282 paylen = skb->len - l4_offset; 1283 if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) { 1284 /* compute length of segmentation header */ 1285 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 1286 csum_replace_by_diff(&l4.tcp->check, 1287 (__force __wsum)htonl(paylen)); 1288 } else { 1289 /* compute length of segmentation header */ 1290 *hdr_len = sizeof(*l4.udp) + l4_offset; 1291 csum_replace_by_diff(&l4.udp->check, 1292 (__force __wsum)htonl(paylen)); 1293 } 1294 1295 /* update gso size and bytecount with header size */ 1296 first->gso_segs = skb_shinfo(skb)->gso_segs; 1297 first->bytecount += (first->gso_segs - 1) * *hdr_len; 1298 1299 /* MSS L4LEN IDX */ 1300 mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT; 1301 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT; 1302 1303 /* VLAN MACLEN IPLEN */ 1304 vlan_macip_lens = l4.hdr - ip.hdr; 1305 vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT; 1306 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1307 1308 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, 1309 type_tucmd, mss_l4len_idx); 1310 1311 return 1; 1312 } 1313 1314 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1315 struct igc_ring *tx_ring) 1316 { 1317 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1318 __be16 protocol = vlan_get_protocol(skb); 1319 struct igc_tx_buffer *first; 1320 u32 tx_flags = 0; 1321 unsigned short f; 1322 u8 hdr_len = 0; 1323 int tso = 0; 1324 1325 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1326 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1327 * + 2 desc gap to keep tail from touching head, 1328 * + 1 desc for context descriptor, 1329 * otherwise try next time 1330 */ 1331 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1332 count += TXD_USE_COUNT(skb_frag_size( 1333 &skb_shinfo(skb)->frags[f])); 1334 1335 if (igc_maybe_stop_tx(tx_ring, count + 3)) { 1336 /* this is a hard error */ 1337 return NETDEV_TX_BUSY; 1338 } 1339 1340 /* record the location of the first descriptor for this packet */ 1341 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1342 first->skb = skb; 1343 first->bytecount = skb->len; 1344 first->gso_segs = 1; 1345 1346 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 1347 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 1348 1349 /* FIXME: add support for retrieving timestamps from 1350 * the other timer registers before skipping the 1351 * timestamping request. 1352 */ 1353 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && 1354 !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS, 1355 &adapter->state)) { 1356 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1357 tx_flags |= IGC_TX_FLAGS_TSTAMP; 1358 1359 adapter->ptp_tx_skb = skb_get(skb); 1360 adapter->ptp_tx_start = jiffies; 1361 } else { 1362 adapter->tx_hwtstamp_skipped++; 1363 } 1364 } 1365 1366 /* record initial flags and protocol */ 1367 first->tx_flags = tx_flags; 1368 first->protocol = protocol; 1369 1370 tso = igc_tso(tx_ring, first, &hdr_len); 1371 if (tso < 0) 1372 goto out_drop; 1373 else if (!tso) 1374 igc_tx_csum(tx_ring, first); 1375 1376 igc_tx_map(tx_ring, first, hdr_len); 1377 1378 return NETDEV_TX_OK; 1379 1380 out_drop: 1381 dev_kfree_skb_any(first->skb); 1382 first->skb = NULL; 1383 1384 return NETDEV_TX_OK; 1385 } 1386 1387 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1388 struct sk_buff *skb) 1389 { 1390 unsigned int r_idx = skb->queue_mapping; 1391 1392 if (r_idx >= adapter->num_tx_queues) 1393 r_idx = r_idx % adapter->num_tx_queues; 1394 1395 return adapter->tx_ring[r_idx]; 1396 } 1397 1398 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1399 struct net_device *netdev) 1400 { 1401 struct igc_adapter *adapter = netdev_priv(netdev); 1402 1403 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1404 * in order to meet this minimum size requirement. 1405 */ 1406 if (skb->len < 17) { 1407 if (skb_padto(skb, 17)) 1408 return NETDEV_TX_OK; 1409 skb->len = 17; 1410 } 1411 1412 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1413 } 1414 1415 static void igc_rx_checksum(struct igc_ring *ring, 1416 union igc_adv_rx_desc *rx_desc, 1417 struct sk_buff *skb) 1418 { 1419 skb_checksum_none_assert(skb); 1420 1421 /* Ignore Checksum bit is set */ 1422 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) 1423 return; 1424 1425 /* Rx checksum disabled via ethtool */ 1426 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1427 return; 1428 1429 /* TCP/UDP checksum error bit is set */ 1430 if (igc_test_staterr(rx_desc, 1431 IGC_RXDEXT_STATERR_TCPE | 1432 IGC_RXDEXT_STATERR_IPE)) { 1433 /* work around errata with sctp packets where the TCPE aka 1434 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 1435 * packets (aka let the stack check the crc32c) 1436 */ 1437 if (!(skb->len == 60 && 1438 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 1439 u64_stats_update_begin(&ring->rx_syncp); 1440 ring->rx_stats.csum_err++; 1441 u64_stats_update_end(&ring->rx_syncp); 1442 } 1443 /* let the stack verify checksum errors */ 1444 return; 1445 } 1446 /* It must be a TCP or UDP packet with a valid checksum */ 1447 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | 1448 IGC_RXD_STAT_UDPCS)) 1449 skb->ip_summed = CHECKSUM_UNNECESSARY; 1450 1451 netdev_dbg(ring->netdev, "cksum success: bits %08X\n", 1452 le32_to_cpu(rx_desc->wb.upper.status_error)); 1453 } 1454 1455 static inline void igc_rx_hash(struct igc_ring *ring, 1456 union igc_adv_rx_desc *rx_desc, 1457 struct sk_buff *skb) 1458 { 1459 if (ring->netdev->features & NETIF_F_RXHASH) 1460 skb_set_hash(skb, 1461 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 1462 PKT_HASH_TYPE_L3); 1463 } 1464 1465 /** 1466 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1467 * @rx_ring: rx descriptor ring packet is being transacted on 1468 * @rx_desc: pointer to the EOP Rx descriptor 1469 * @skb: pointer to current skb being populated 1470 * 1471 * This function checks the ring, descriptor, and packet information in order 1472 * to populate the hash, checksum, VLAN, protocol, and other fields within the 1473 * skb. 1474 */ 1475 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1476 union igc_adv_rx_desc *rx_desc, 1477 struct sk_buff *skb) 1478 { 1479 igc_rx_hash(rx_ring, rx_desc, skb); 1480 1481 igc_rx_checksum(rx_ring, rx_desc, skb); 1482 1483 skb_record_rx_queue(skb, rx_ring->queue_index); 1484 1485 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1486 } 1487 1488 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1489 const unsigned int size) 1490 { 1491 struct igc_rx_buffer *rx_buffer; 1492 1493 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1494 prefetchw(rx_buffer->page); 1495 1496 /* we are reusing so sync this buffer for CPU use */ 1497 dma_sync_single_range_for_cpu(rx_ring->dev, 1498 rx_buffer->dma, 1499 rx_buffer->page_offset, 1500 size, 1501 DMA_FROM_DEVICE); 1502 1503 rx_buffer->pagecnt_bias--; 1504 1505 return rx_buffer; 1506 } 1507 1508 /** 1509 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1510 * @rx_ring: rx descriptor ring to transact packets on 1511 * @rx_buffer: buffer containing page to add 1512 * @skb: sk_buff to place the data into 1513 * @size: size of buffer to be added 1514 * 1515 * This function will add the data contained in rx_buffer->page to the skb. 1516 */ 1517 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1518 struct igc_rx_buffer *rx_buffer, 1519 struct sk_buff *skb, 1520 unsigned int size) 1521 { 1522 #if (PAGE_SIZE < 8192) 1523 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1524 1525 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1526 rx_buffer->page_offset, size, truesize); 1527 rx_buffer->page_offset ^= truesize; 1528 #else 1529 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1530 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1531 SKB_DATA_ALIGN(size); 1532 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1533 rx_buffer->page_offset, size, truesize); 1534 rx_buffer->page_offset += truesize; 1535 #endif 1536 } 1537 1538 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1539 struct igc_rx_buffer *rx_buffer, 1540 union igc_adv_rx_desc *rx_desc, 1541 unsigned int size) 1542 { 1543 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1544 #if (PAGE_SIZE < 8192) 1545 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1546 #else 1547 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1548 SKB_DATA_ALIGN(IGC_SKB_PAD + size); 1549 #endif 1550 struct sk_buff *skb; 1551 1552 /* prefetch first cache line of first page */ 1553 prefetch(va); 1554 #if L1_CACHE_BYTES < 128 1555 prefetch(va + L1_CACHE_BYTES); 1556 #endif 1557 1558 /* build an skb around the page buffer */ 1559 skb = build_skb(va - IGC_SKB_PAD, truesize); 1560 if (unlikely(!skb)) 1561 return NULL; 1562 1563 /* update pointers within the skb to store the data */ 1564 skb_reserve(skb, IGC_SKB_PAD); 1565 __skb_put(skb, size); 1566 1567 /* update buffer offset */ 1568 #if (PAGE_SIZE < 8192) 1569 rx_buffer->page_offset ^= truesize; 1570 #else 1571 rx_buffer->page_offset += truesize; 1572 #endif 1573 1574 return skb; 1575 } 1576 1577 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1578 struct igc_rx_buffer *rx_buffer, 1579 union igc_adv_rx_desc *rx_desc, 1580 unsigned int size) 1581 { 1582 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1583 #if (PAGE_SIZE < 8192) 1584 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1585 #else 1586 unsigned int truesize = SKB_DATA_ALIGN(size); 1587 #endif 1588 unsigned int headlen; 1589 struct sk_buff *skb; 1590 1591 /* prefetch first cache line of first page */ 1592 prefetch(va); 1593 #if L1_CACHE_BYTES < 128 1594 prefetch(va + L1_CACHE_BYTES); 1595 #endif 1596 1597 /* allocate a skb to store the frags */ 1598 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); 1599 if (unlikely(!skb)) 1600 return NULL; 1601 1602 if (unlikely(igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP))) { 1603 igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb); 1604 va += IGC_TS_HDR_LEN; 1605 size -= IGC_TS_HDR_LEN; 1606 } 1607 1608 /* Determine available headroom for copy */ 1609 headlen = size; 1610 if (headlen > IGC_RX_HDR_LEN) 1611 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 1612 1613 /* align pull length to size of long to optimize memcpy performance */ 1614 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 1615 1616 /* update all of the pointers */ 1617 size -= headlen; 1618 if (size) { 1619 skb_add_rx_frag(skb, 0, rx_buffer->page, 1620 (va + headlen) - page_address(rx_buffer->page), 1621 size, truesize); 1622 #if (PAGE_SIZE < 8192) 1623 rx_buffer->page_offset ^= truesize; 1624 #else 1625 rx_buffer->page_offset += truesize; 1626 #endif 1627 } else { 1628 rx_buffer->pagecnt_bias++; 1629 } 1630 1631 return skb; 1632 } 1633 1634 /** 1635 * igc_reuse_rx_page - page flip buffer and store it back on the ring 1636 * @rx_ring: rx descriptor ring to store buffers on 1637 * @old_buff: donor buffer to have page reused 1638 * 1639 * Synchronizes page for reuse by the adapter 1640 */ 1641 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 1642 struct igc_rx_buffer *old_buff) 1643 { 1644 u16 nta = rx_ring->next_to_alloc; 1645 struct igc_rx_buffer *new_buff; 1646 1647 new_buff = &rx_ring->rx_buffer_info[nta]; 1648 1649 /* update, and store next to alloc */ 1650 nta++; 1651 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1652 1653 /* Transfer page from old buffer to new buffer. 1654 * Move each member individually to avoid possible store 1655 * forwarding stalls. 1656 */ 1657 new_buff->dma = old_buff->dma; 1658 new_buff->page = old_buff->page; 1659 new_buff->page_offset = old_buff->page_offset; 1660 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1661 } 1662 1663 static inline bool igc_page_is_reserved(struct page *page) 1664 { 1665 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 1666 } 1667 1668 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) 1669 { 1670 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1671 struct page *page = rx_buffer->page; 1672 1673 /* avoid re-using remote pages */ 1674 if (unlikely(igc_page_is_reserved(page))) 1675 return false; 1676 1677 #if (PAGE_SIZE < 8192) 1678 /* if we are only owner of page we can reuse it */ 1679 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 1680 return false; 1681 #else 1682 #define IGC_LAST_OFFSET \ 1683 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 1684 1685 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 1686 return false; 1687 #endif 1688 1689 /* If we have drained the page fragment pool we need to update 1690 * the pagecnt_bias and page count so that we fully restock the 1691 * number of references the driver holds. 1692 */ 1693 if (unlikely(!pagecnt_bias)) { 1694 page_ref_add(page, USHRT_MAX); 1695 rx_buffer->pagecnt_bias = USHRT_MAX; 1696 } 1697 1698 return true; 1699 } 1700 1701 /** 1702 * igc_is_non_eop - process handling of non-EOP buffers 1703 * @rx_ring: Rx ring being processed 1704 * @rx_desc: Rx descriptor for current buffer 1705 * 1706 * This function updates next to clean. If the buffer is an EOP buffer 1707 * this function exits returning false, otherwise it will place the 1708 * sk_buff in the next buffer to be chained and return true indicating 1709 * that this is in fact a non-EOP buffer. 1710 */ 1711 static bool igc_is_non_eop(struct igc_ring *rx_ring, 1712 union igc_adv_rx_desc *rx_desc) 1713 { 1714 u32 ntc = rx_ring->next_to_clean + 1; 1715 1716 /* fetch, update, and store next to clean */ 1717 ntc = (ntc < rx_ring->count) ? ntc : 0; 1718 rx_ring->next_to_clean = ntc; 1719 1720 prefetch(IGC_RX_DESC(rx_ring, ntc)); 1721 1722 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 1723 return false; 1724 1725 return true; 1726 } 1727 1728 /** 1729 * igc_cleanup_headers - Correct corrupted or empty headers 1730 * @rx_ring: rx descriptor ring packet is being transacted on 1731 * @rx_desc: pointer to the EOP Rx descriptor 1732 * @skb: pointer to current skb being fixed 1733 * 1734 * Address the case where we are pulling data in on pages only 1735 * and as such no data is present in the skb header. 1736 * 1737 * In addition if skb is not at least 60 bytes we need to pad it so that 1738 * it is large enough to qualify as a valid Ethernet frame. 1739 * 1740 * Returns true if an error was encountered and skb was freed. 1741 */ 1742 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 1743 union igc_adv_rx_desc *rx_desc, 1744 struct sk_buff *skb) 1745 { 1746 if (unlikely((igc_test_staterr(rx_desc, 1747 IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) { 1748 struct net_device *netdev = rx_ring->netdev; 1749 1750 if (!(netdev->features & NETIF_F_RXALL)) { 1751 dev_kfree_skb_any(skb); 1752 return true; 1753 } 1754 } 1755 1756 /* if eth_skb_pad returns an error the skb was freed */ 1757 if (eth_skb_pad(skb)) 1758 return true; 1759 1760 return false; 1761 } 1762 1763 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 1764 struct igc_rx_buffer *rx_buffer) 1765 { 1766 if (igc_can_reuse_rx_page(rx_buffer)) { 1767 /* hand second half of page back to the ring */ 1768 igc_reuse_rx_page(rx_ring, rx_buffer); 1769 } else { 1770 /* We are not reusing the buffer so unmap it and free 1771 * any references we are holding to it 1772 */ 1773 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1774 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 1775 IGC_RX_DMA_ATTR); 1776 __page_frag_cache_drain(rx_buffer->page, 1777 rx_buffer->pagecnt_bias); 1778 } 1779 1780 /* clear contents of rx_buffer */ 1781 rx_buffer->page = NULL; 1782 } 1783 1784 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 1785 { 1786 return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; 1787 } 1788 1789 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 1790 struct igc_rx_buffer *bi) 1791 { 1792 struct page *page = bi->page; 1793 dma_addr_t dma; 1794 1795 /* since we are recycling buffers we should seldom need to alloc */ 1796 if (likely(page)) 1797 return true; 1798 1799 /* alloc new page for storage */ 1800 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 1801 if (unlikely(!page)) { 1802 rx_ring->rx_stats.alloc_failed++; 1803 return false; 1804 } 1805 1806 /* map page for use */ 1807 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1808 igc_rx_pg_size(rx_ring), 1809 DMA_FROM_DEVICE, 1810 IGC_RX_DMA_ATTR); 1811 1812 /* if mapping failed free memory back to system since 1813 * there isn't much point in holding memory we can't use 1814 */ 1815 if (dma_mapping_error(rx_ring->dev, dma)) { 1816 __free_page(page); 1817 1818 rx_ring->rx_stats.alloc_failed++; 1819 return false; 1820 } 1821 1822 bi->dma = dma; 1823 bi->page = page; 1824 bi->page_offset = igc_rx_offset(rx_ring); 1825 bi->pagecnt_bias = 1; 1826 1827 return true; 1828 } 1829 1830 /** 1831 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 1832 * @rx_ring: rx descriptor ring 1833 * @cleaned_count: number of buffers to clean 1834 */ 1835 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 1836 { 1837 union igc_adv_rx_desc *rx_desc; 1838 u16 i = rx_ring->next_to_use; 1839 struct igc_rx_buffer *bi; 1840 u16 bufsz; 1841 1842 /* nothing to do */ 1843 if (!cleaned_count) 1844 return; 1845 1846 rx_desc = IGC_RX_DESC(rx_ring, i); 1847 bi = &rx_ring->rx_buffer_info[i]; 1848 i -= rx_ring->count; 1849 1850 bufsz = igc_rx_bufsz(rx_ring); 1851 1852 do { 1853 if (!igc_alloc_mapped_page(rx_ring, bi)) 1854 break; 1855 1856 /* sync the buffer for use by the device */ 1857 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1858 bi->page_offset, bufsz, 1859 DMA_FROM_DEVICE); 1860 1861 /* Refresh the desc even if buffer_addrs didn't change 1862 * because each write-back erases this info. 1863 */ 1864 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1865 1866 rx_desc++; 1867 bi++; 1868 i++; 1869 if (unlikely(!i)) { 1870 rx_desc = IGC_RX_DESC(rx_ring, 0); 1871 bi = rx_ring->rx_buffer_info; 1872 i -= rx_ring->count; 1873 } 1874 1875 /* clear the length for the next_to_use descriptor */ 1876 rx_desc->wb.upper.length = 0; 1877 1878 cleaned_count--; 1879 } while (cleaned_count); 1880 1881 i += rx_ring->count; 1882 1883 if (rx_ring->next_to_use != i) { 1884 /* record the next descriptor to use */ 1885 rx_ring->next_to_use = i; 1886 1887 /* update next to alloc since we have filled the ring */ 1888 rx_ring->next_to_alloc = i; 1889 1890 /* Force memory writes to complete before letting h/w 1891 * know there are new descriptors to fetch. (Only 1892 * applicable for weak-ordered memory model archs, 1893 * such as IA-64). 1894 */ 1895 wmb(); 1896 writel(i, rx_ring->tail); 1897 } 1898 } 1899 1900 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 1901 { 1902 unsigned int total_bytes = 0, total_packets = 0; 1903 struct igc_ring *rx_ring = q_vector->rx.ring; 1904 struct sk_buff *skb = rx_ring->skb; 1905 u16 cleaned_count = igc_desc_unused(rx_ring); 1906 1907 while (likely(total_packets < budget)) { 1908 union igc_adv_rx_desc *rx_desc; 1909 struct igc_rx_buffer *rx_buffer; 1910 unsigned int size; 1911 1912 /* return some buffers to hardware, one at a time is too slow */ 1913 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 1914 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1915 cleaned_count = 0; 1916 } 1917 1918 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 1919 size = le16_to_cpu(rx_desc->wb.upper.length); 1920 if (!size) 1921 break; 1922 1923 /* This memory barrier is needed to keep us from reading 1924 * any other fields out of the rx_desc until we know the 1925 * descriptor has been written back 1926 */ 1927 dma_rmb(); 1928 1929 rx_buffer = igc_get_rx_buffer(rx_ring, size); 1930 1931 /* retrieve a buffer from the ring */ 1932 if (skb) 1933 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 1934 else if (ring_uses_build_skb(rx_ring)) 1935 skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); 1936 else 1937 skb = igc_construct_skb(rx_ring, rx_buffer, 1938 rx_desc, size); 1939 1940 /* exit if we failed to retrieve a buffer */ 1941 if (!skb) { 1942 rx_ring->rx_stats.alloc_failed++; 1943 rx_buffer->pagecnt_bias++; 1944 break; 1945 } 1946 1947 igc_put_rx_buffer(rx_ring, rx_buffer); 1948 cleaned_count++; 1949 1950 /* fetch next buffer in frame if non-eop */ 1951 if (igc_is_non_eop(rx_ring, rx_desc)) 1952 continue; 1953 1954 /* verify the packet layout is correct */ 1955 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) { 1956 skb = NULL; 1957 continue; 1958 } 1959 1960 /* probably a little skewed due to removing CRC */ 1961 total_bytes += skb->len; 1962 1963 /* populate checksum, VLAN, and protocol */ 1964 igc_process_skb_fields(rx_ring, rx_desc, skb); 1965 1966 napi_gro_receive(&q_vector->napi, skb); 1967 1968 /* reset skb pointer */ 1969 skb = NULL; 1970 1971 /* update budget accounting */ 1972 total_packets++; 1973 } 1974 1975 /* place incomplete frames back on ring for completion */ 1976 rx_ring->skb = skb; 1977 1978 u64_stats_update_begin(&rx_ring->rx_syncp); 1979 rx_ring->rx_stats.packets += total_packets; 1980 rx_ring->rx_stats.bytes += total_bytes; 1981 u64_stats_update_end(&rx_ring->rx_syncp); 1982 q_vector->rx.total_packets += total_packets; 1983 q_vector->rx.total_bytes += total_bytes; 1984 1985 if (cleaned_count) 1986 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1987 1988 return total_packets; 1989 } 1990 1991 /** 1992 * igc_clean_tx_irq - Reclaim resources after transmit completes 1993 * @q_vector: pointer to q_vector containing needed info 1994 * @napi_budget: Used to determine if we are in netpoll 1995 * 1996 * returns true if ring is completely cleaned 1997 */ 1998 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 1999 { 2000 struct igc_adapter *adapter = q_vector->adapter; 2001 unsigned int total_bytes = 0, total_packets = 0; 2002 unsigned int budget = q_vector->tx.work_limit; 2003 struct igc_ring *tx_ring = q_vector->tx.ring; 2004 unsigned int i = tx_ring->next_to_clean; 2005 struct igc_tx_buffer *tx_buffer; 2006 union igc_adv_tx_desc *tx_desc; 2007 2008 if (test_bit(__IGC_DOWN, &adapter->state)) 2009 return true; 2010 2011 tx_buffer = &tx_ring->tx_buffer_info[i]; 2012 tx_desc = IGC_TX_DESC(tx_ring, i); 2013 i -= tx_ring->count; 2014 2015 do { 2016 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 2017 2018 /* if next_to_watch is not set then there is no work pending */ 2019 if (!eop_desc) 2020 break; 2021 2022 /* prevent any other reads prior to eop_desc */ 2023 smp_rmb(); 2024 2025 /* if DD is not set pending work has not been completed */ 2026 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 2027 break; 2028 2029 /* clear next_to_watch to prevent false hangs */ 2030 tx_buffer->next_to_watch = NULL; 2031 2032 /* update the statistics for this packet */ 2033 total_bytes += tx_buffer->bytecount; 2034 total_packets += tx_buffer->gso_segs; 2035 2036 /* free the skb */ 2037 napi_consume_skb(tx_buffer->skb, napi_budget); 2038 2039 /* unmap skb header data */ 2040 dma_unmap_single(tx_ring->dev, 2041 dma_unmap_addr(tx_buffer, dma), 2042 dma_unmap_len(tx_buffer, len), 2043 DMA_TO_DEVICE); 2044 2045 /* clear tx_buffer data */ 2046 dma_unmap_len_set(tx_buffer, len, 0); 2047 2048 /* clear last DMA location and unmap remaining buffers */ 2049 while (tx_desc != eop_desc) { 2050 tx_buffer++; 2051 tx_desc++; 2052 i++; 2053 if (unlikely(!i)) { 2054 i -= tx_ring->count; 2055 tx_buffer = tx_ring->tx_buffer_info; 2056 tx_desc = IGC_TX_DESC(tx_ring, 0); 2057 } 2058 2059 /* unmap any remaining paged data */ 2060 if (dma_unmap_len(tx_buffer, len)) { 2061 dma_unmap_page(tx_ring->dev, 2062 dma_unmap_addr(tx_buffer, dma), 2063 dma_unmap_len(tx_buffer, len), 2064 DMA_TO_DEVICE); 2065 dma_unmap_len_set(tx_buffer, len, 0); 2066 } 2067 } 2068 2069 /* move us one more past the eop_desc for start of next pkt */ 2070 tx_buffer++; 2071 tx_desc++; 2072 i++; 2073 if (unlikely(!i)) { 2074 i -= tx_ring->count; 2075 tx_buffer = tx_ring->tx_buffer_info; 2076 tx_desc = IGC_TX_DESC(tx_ring, 0); 2077 } 2078 2079 /* issue prefetch for next Tx descriptor */ 2080 prefetch(tx_desc); 2081 2082 /* update budget accounting */ 2083 budget--; 2084 } while (likely(budget)); 2085 2086 netdev_tx_completed_queue(txring_txq(tx_ring), 2087 total_packets, total_bytes); 2088 2089 i += tx_ring->count; 2090 tx_ring->next_to_clean = i; 2091 u64_stats_update_begin(&tx_ring->tx_syncp); 2092 tx_ring->tx_stats.bytes += total_bytes; 2093 tx_ring->tx_stats.packets += total_packets; 2094 u64_stats_update_end(&tx_ring->tx_syncp); 2095 q_vector->tx.total_bytes += total_bytes; 2096 q_vector->tx.total_packets += total_packets; 2097 2098 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 2099 struct igc_hw *hw = &adapter->hw; 2100 2101 /* Detect a transmit hang in hardware, this serializes the 2102 * check with the clearing of time_stamp and movement of i 2103 */ 2104 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 2105 if (tx_buffer->next_to_watch && 2106 time_after(jiffies, tx_buffer->time_stamp + 2107 (adapter->tx_timeout_factor * HZ)) && 2108 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { 2109 /* detected Tx unit hang */ 2110 netdev_err(tx_ring->netdev, 2111 "Detected Tx Unit Hang\n" 2112 " Tx Queue <%d>\n" 2113 " TDH <%x>\n" 2114 " TDT <%x>\n" 2115 " next_to_use <%x>\n" 2116 " next_to_clean <%x>\n" 2117 "buffer_info[next_to_clean]\n" 2118 " time_stamp <%lx>\n" 2119 " next_to_watch <%p>\n" 2120 " jiffies <%lx>\n" 2121 " desc.status <%x>\n", 2122 tx_ring->queue_index, 2123 rd32(IGC_TDH(tx_ring->reg_idx)), 2124 readl(tx_ring->tail), 2125 tx_ring->next_to_use, 2126 tx_ring->next_to_clean, 2127 tx_buffer->time_stamp, 2128 tx_buffer->next_to_watch, 2129 jiffies, 2130 tx_buffer->next_to_watch->wb.status); 2131 netif_stop_subqueue(tx_ring->netdev, 2132 tx_ring->queue_index); 2133 2134 /* we are about to reset, no point in enabling stuff */ 2135 return true; 2136 } 2137 } 2138 2139 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 2140 if (unlikely(total_packets && 2141 netif_carrier_ok(tx_ring->netdev) && 2142 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 2143 /* Make sure that anybody stopping the queue after this 2144 * sees the new next_to_clean. 2145 */ 2146 smp_mb(); 2147 if (__netif_subqueue_stopped(tx_ring->netdev, 2148 tx_ring->queue_index) && 2149 !(test_bit(__IGC_DOWN, &adapter->state))) { 2150 netif_wake_subqueue(tx_ring->netdev, 2151 tx_ring->queue_index); 2152 2153 u64_stats_update_begin(&tx_ring->tx_syncp); 2154 tx_ring->tx_stats.restart_queue++; 2155 u64_stats_update_end(&tx_ring->tx_syncp); 2156 } 2157 } 2158 2159 return !!budget; 2160 } 2161 2162 static int igc_find_mac_filter(struct igc_adapter *adapter, 2163 enum igc_mac_filter_type type, const u8 *addr) 2164 { 2165 struct igc_hw *hw = &adapter->hw; 2166 int max_entries = hw->mac.rar_entry_count; 2167 u32 ral, rah; 2168 int i; 2169 2170 for (i = 0; i < max_entries; i++) { 2171 ral = rd32(IGC_RAL(i)); 2172 rah = rd32(IGC_RAH(i)); 2173 2174 if (!(rah & IGC_RAH_AV)) 2175 continue; 2176 if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type) 2177 continue; 2178 if ((rah & IGC_RAH_RAH_MASK) != 2179 le16_to_cpup((__le16 *)(addr + 4))) 2180 continue; 2181 if (ral != le32_to_cpup((__le32 *)(addr))) 2182 continue; 2183 2184 return i; 2185 } 2186 2187 return -1; 2188 } 2189 2190 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) 2191 { 2192 struct igc_hw *hw = &adapter->hw; 2193 int max_entries = hw->mac.rar_entry_count; 2194 u32 rah; 2195 int i; 2196 2197 for (i = 0; i < max_entries; i++) { 2198 rah = rd32(IGC_RAH(i)); 2199 2200 if (!(rah & IGC_RAH_AV)) 2201 return i; 2202 } 2203 2204 return -1; 2205 } 2206 2207 /** 2208 * igc_add_mac_filter() - Add MAC address filter 2209 * @adapter: Pointer to adapter where the filter should be added 2210 * @type: MAC address filter type (source or destination) 2211 * @addr: MAC address 2212 * @queue: If non-negative, queue assignment feature is enabled and frames 2213 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2214 * assignment is disabled. 2215 * 2216 * Return: 0 in case of success, negative errno code otherwise. 2217 */ 2218 static int igc_add_mac_filter(struct igc_adapter *adapter, 2219 enum igc_mac_filter_type type, const u8 *addr, 2220 int queue) 2221 { 2222 struct net_device *dev = adapter->netdev; 2223 int index; 2224 2225 index = igc_find_mac_filter(adapter, type, addr); 2226 if (index >= 0) 2227 goto update_filter; 2228 2229 index = igc_get_avail_mac_filter_slot(adapter); 2230 if (index < 0) 2231 return -ENOSPC; 2232 2233 netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n", 2234 index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2235 addr, queue); 2236 2237 update_filter: 2238 igc_set_mac_filter_hw(adapter, index, type, addr, queue); 2239 return 0; 2240 } 2241 2242 /** 2243 * igc_del_mac_filter() - Delete MAC address filter 2244 * @adapter: Pointer to adapter where the filter should be deleted from 2245 * @type: MAC address filter type (source or destination) 2246 * @addr: MAC address 2247 */ 2248 static void igc_del_mac_filter(struct igc_adapter *adapter, 2249 enum igc_mac_filter_type type, const u8 *addr) 2250 { 2251 struct net_device *dev = adapter->netdev; 2252 int index; 2253 2254 index = igc_find_mac_filter(adapter, type, addr); 2255 if (index < 0) 2256 return; 2257 2258 if (index == 0) { 2259 /* If this is the default filter, we don't actually delete it. 2260 * We just reset to its default value i.e. disable queue 2261 * assignment. 2262 */ 2263 netdev_dbg(dev, "Disable default MAC filter queue assignment"); 2264 2265 igc_set_mac_filter_hw(adapter, 0, type, addr, -1); 2266 } else { 2267 netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n", 2268 index, 2269 type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2270 addr); 2271 2272 igc_clear_mac_filter_hw(adapter, index); 2273 } 2274 } 2275 2276 /** 2277 * igc_add_vlan_prio_filter() - Add VLAN priority filter 2278 * @adapter: Pointer to adapter where the filter should be added 2279 * @prio: VLAN priority value 2280 * @queue: Queue number which matching frames are assigned to 2281 * 2282 * Return: 0 in case of success, negative errno code otherwise. 2283 */ 2284 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, 2285 int queue) 2286 { 2287 struct net_device *dev = adapter->netdev; 2288 struct igc_hw *hw = &adapter->hw; 2289 u32 vlanpqf; 2290 2291 vlanpqf = rd32(IGC_VLANPQF); 2292 2293 if (vlanpqf & IGC_VLANPQF_VALID(prio)) { 2294 netdev_dbg(dev, "VLAN priority filter already in use\n"); 2295 return -EEXIST; 2296 } 2297 2298 vlanpqf |= IGC_VLANPQF_QSEL(prio, queue); 2299 vlanpqf |= IGC_VLANPQF_VALID(prio); 2300 2301 wr32(IGC_VLANPQF, vlanpqf); 2302 2303 netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n", 2304 prio, queue); 2305 return 0; 2306 } 2307 2308 /** 2309 * igc_del_vlan_prio_filter() - Delete VLAN priority filter 2310 * @adapter: Pointer to adapter where the filter should be deleted from 2311 * @prio: VLAN priority value 2312 */ 2313 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) 2314 { 2315 struct igc_hw *hw = &adapter->hw; 2316 u32 vlanpqf; 2317 2318 vlanpqf = rd32(IGC_VLANPQF); 2319 2320 vlanpqf &= ~IGC_VLANPQF_VALID(prio); 2321 vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK); 2322 2323 wr32(IGC_VLANPQF, vlanpqf); 2324 2325 netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n", 2326 prio); 2327 } 2328 2329 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) 2330 { 2331 struct igc_hw *hw = &adapter->hw; 2332 int i; 2333 2334 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2335 u32 etqf = rd32(IGC_ETQF(i)); 2336 2337 if (!(etqf & IGC_ETQF_FILTER_ENABLE)) 2338 return i; 2339 } 2340 2341 return -1; 2342 } 2343 2344 /** 2345 * igc_add_etype_filter() - Add ethertype filter 2346 * @adapter: Pointer to adapter where the filter should be added 2347 * @etype: Ethertype value 2348 * @queue: If non-negative, queue assignment feature is enabled and frames 2349 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2350 * assignment is disabled. 2351 * 2352 * Return: 0 in case of success, negative errno code otherwise. 2353 */ 2354 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, 2355 int queue) 2356 { 2357 struct igc_hw *hw = &adapter->hw; 2358 int index; 2359 u32 etqf; 2360 2361 index = igc_get_avail_etype_filter_slot(adapter); 2362 if (index < 0) 2363 return -ENOSPC; 2364 2365 etqf = rd32(IGC_ETQF(index)); 2366 2367 etqf &= ~IGC_ETQF_ETYPE_MASK; 2368 etqf |= etype; 2369 2370 if (queue >= 0) { 2371 etqf &= ~IGC_ETQF_QUEUE_MASK; 2372 etqf |= (queue << IGC_ETQF_QUEUE_SHIFT); 2373 etqf |= IGC_ETQF_QUEUE_ENABLE; 2374 } 2375 2376 etqf |= IGC_ETQF_FILTER_ENABLE; 2377 2378 wr32(IGC_ETQF(index), etqf); 2379 2380 netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n", 2381 etype, queue); 2382 return 0; 2383 } 2384 2385 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) 2386 { 2387 struct igc_hw *hw = &adapter->hw; 2388 int i; 2389 2390 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2391 u32 etqf = rd32(IGC_ETQF(i)); 2392 2393 if ((etqf & IGC_ETQF_ETYPE_MASK) == etype) 2394 return i; 2395 } 2396 2397 return -1; 2398 } 2399 2400 /** 2401 * igc_del_etype_filter() - Delete ethertype filter 2402 * @adapter: Pointer to adapter where the filter should be deleted from 2403 * @etype: Ethertype value 2404 */ 2405 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) 2406 { 2407 struct igc_hw *hw = &adapter->hw; 2408 int index; 2409 2410 index = igc_find_etype_filter(adapter, etype); 2411 if (index < 0) 2412 return; 2413 2414 wr32(IGC_ETQF(index), 0); 2415 2416 netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n", 2417 etype); 2418 } 2419 2420 static int igc_enable_nfc_rule(struct igc_adapter *adapter, 2421 const struct igc_nfc_rule *rule) 2422 { 2423 int err; 2424 2425 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { 2426 err = igc_add_etype_filter(adapter, rule->filter.etype, 2427 rule->action); 2428 if (err) 2429 return err; 2430 } 2431 2432 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { 2433 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2434 rule->filter.src_addr, rule->action); 2435 if (err) 2436 return err; 2437 } 2438 2439 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { 2440 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2441 rule->filter.dst_addr, rule->action); 2442 if (err) 2443 return err; 2444 } 2445 2446 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2447 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2448 VLAN_PRIO_SHIFT; 2449 2450 err = igc_add_vlan_prio_filter(adapter, prio, rule->action); 2451 if (err) 2452 return err; 2453 } 2454 2455 return 0; 2456 } 2457 2458 static void igc_disable_nfc_rule(struct igc_adapter *adapter, 2459 const struct igc_nfc_rule *rule) 2460 { 2461 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) 2462 igc_del_etype_filter(adapter, rule->filter.etype); 2463 2464 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2465 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2466 VLAN_PRIO_SHIFT; 2467 2468 igc_del_vlan_prio_filter(adapter, prio); 2469 } 2470 2471 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) 2472 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2473 rule->filter.src_addr); 2474 2475 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) 2476 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2477 rule->filter.dst_addr); 2478 } 2479 2480 /** 2481 * igc_get_nfc_rule() - Get NFC rule 2482 * @adapter: Pointer to adapter 2483 * @location: Rule location 2484 * 2485 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2486 * 2487 * Return: Pointer to NFC rule at @location. If not found, NULL. 2488 */ 2489 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, 2490 u32 location) 2491 { 2492 struct igc_nfc_rule *rule; 2493 2494 list_for_each_entry(rule, &adapter->nfc_rule_list, list) { 2495 if (rule->location == location) 2496 return rule; 2497 if (rule->location > location) 2498 break; 2499 } 2500 2501 return NULL; 2502 } 2503 2504 /** 2505 * igc_del_nfc_rule() - Delete NFC rule 2506 * @adapter: Pointer to adapter 2507 * @rule: Pointer to rule to be deleted 2508 * 2509 * Disable NFC rule in hardware and delete it from adapter. 2510 * 2511 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2512 */ 2513 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2514 { 2515 igc_disable_nfc_rule(adapter, rule); 2516 2517 list_del(&rule->list); 2518 adapter->nfc_rule_count--; 2519 2520 kfree(rule); 2521 } 2522 2523 static void igc_flush_nfc_rules(struct igc_adapter *adapter) 2524 { 2525 struct igc_nfc_rule *rule, *tmp; 2526 2527 mutex_lock(&adapter->nfc_rule_lock); 2528 2529 list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list) 2530 igc_del_nfc_rule(adapter, rule); 2531 2532 mutex_unlock(&adapter->nfc_rule_lock); 2533 } 2534 2535 /** 2536 * igc_add_nfc_rule() - Add NFC rule 2537 * @adapter: Pointer to adapter 2538 * @rule: Pointer to rule to be added 2539 * 2540 * Enable NFC rule in hardware and add it to adapter. 2541 * 2542 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2543 * 2544 * Return: 0 on success, negative errno on failure. 2545 */ 2546 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2547 { 2548 struct igc_nfc_rule *pred, *cur; 2549 int err; 2550 2551 err = igc_enable_nfc_rule(adapter, rule); 2552 if (err) 2553 return err; 2554 2555 pred = NULL; 2556 list_for_each_entry(cur, &adapter->nfc_rule_list, list) { 2557 if (cur->location >= rule->location) 2558 break; 2559 pred = cur; 2560 } 2561 2562 list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list); 2563 adapter->nfc_rule_count++; 2564 return 0; 2565 } 2566 2567 static void igc_restore_nfc_rules(struct igc_adapter *adapter) 2568 { 2569 struct igc_nfc_rule *rule; 2570 2571 mutex_lock(&adapter->nfc_rule_lock); 2572 2573 list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list) 2574 igc_enable_nfc_rule(adapter, rule); 2575 2576 mutex_unlock(&adapter->nfc_rule_lock); 2577 } 2578 2579 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) 2580 { 2581 struct igc_adapter *adapter = netdev_priv(netdev); 2582 2583 return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1); 2584 } 2585 2586 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) 2587 { 2588 struct igc_adapter *adapter = netdev_priv(netdev); 2589 2590 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr); 2591 return 0; 2592 } 2593 2594 /** 2595 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 2596 * @netdev: network interface device structure 2597 * 2598 * The set_rx_mode entry point is called whenever the unicast or multicast 2599 * address lists or the network interface flags are updated. This routine is 2600 * responsible for configuring the hardware for proper unicast, multicast, 2601 * promiscuous mode, and all-multi behavior. 2602 */ 2603 static void igc_set_rx_mode(struct net_device *netdev) 2604 { 2605 struct igc_adapter *adapter = netdev_priv(netdev); 2606 struct igc_hw *hw = &adapter->hw; 2607 u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 2608 int count; 2609 2610 /* Check for Promiscuous and All Multicast modes */ 2611 if (netdev->flags & IFF_PROMISC) { 2612 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; 2613 } else { 2614 if (netdev->flags & IFF_ALLMULTI) { 2615 rctl |= IGC_RCTL_MPE; 2616 } else { 2617 /* Write addresses to the MTA, if the attempt fails 2618 * then we should just turn on promiscuous mode so 2619 * that we can at least receive multicast traffic 2620 */ 2621 count = igc_write_mc_addr_list(netdev); 2622 if (count < 0) 2623 rctl |= IGC_RCTL_MPE; 2624 } 2625 } 2626 2627 /* Write addresses to available RAR registers, if there is not 2628 * sufficient space to store all the addresses then enable 2629 * unicast promiscuous mode 2630 */ 2631 if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) 2632 rctl |= IGC_RCTL_UPE; 2633 2634 /* update state of unicast and multicast */ 2635 rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 2636 wr32(IGC_RCTL, rctl); 2637 2638 #if (PAGE_SIZE < 8192) 2639 if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) 2640 rlpml = IGC_MAX_FRAME_BUILD_SKB; 2641 #endif 2642 wr32(IGC_RLPML, rlpml); 2643 } 2644 2645 /** 2646 * igc_configure - configure the hardware for RX and TX 2647 * @adapter: private board structure 2648 */ 2649 static void igc_configure(struct igc_adapter *adapter) 2650 { 2651 struct net_device *netdev = adapter->netdev; 2652 int i = 0; 2653 2654 igc_get_hw_control(adapter); 2655 igc_set_rx_mode(netdev); 2656 2657 igc_setup_tctl(adapter); 2658 igc_setup_mrqc(adapter); 2659 igc_setup_rctl(adapter); 2660 2661 igc_set_default_mac_filter(adapter); 2662 igc_restore_nfc_rules(adapter); 2663 2664 igc_configure_tx(adapter); 2665 igc_configure_rx(adapter); 2666 2667 igc_rx_fifo_flush_base(&adapter->hw); 2668 2669 /* call igc_desc_unused which always leaves 2670 * at least 1 descriptor unused to make sure 2671 * next_to_use != next_to_clean 2672 */ 2673 for (i = 0; i < adapter->num_rx_queues; i++) { 2674 struct igc_ring *ring = adapter->rx_ring[i]; 2675 2676 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 2677 } 2678 } 2679 2680 /** 2681 * igc_write_ivar - configure ivar for given MSI-X vector 2682 * @hw: pointer to the HW structure 2683 * @msix_vector: vector number we are allocating to a given ring 2684 * @index: row index of IVAR register to write within IVAR table 2685 * @offset: column offset of in IVAR, should be multiple of 8 2686 * 2687 * The IVAR table consists of 2 columns, 2688 * each containing an cause allocation for an Rx and Tx ring, and a 2689 * variable number of rows depending on the number of queues supported. 2690 */ 2691 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 2692 int index, int offset) 2693 { 2694 u32 ivar = array_rd32(IGC_IVAR0, index); 2695 2696 /* clear any bits that are currently set */ 2697 ivar &= ~((u32)0xFF << offset); 2698 2699 /* write vector and valid bit */ 2700 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 2701 2702 array_wr32(IGC_IVAR0, index, ivar); 2703 } 2704 2705 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 2706 { 2707 struct igc_adapter *adapter = q_vector->adapter; 2708 struct igc_hw *hw = &adapter->hw; 2709 int rx_queue = IGC_N0_QUEUE; 2710 int tx_queue = IGC_N0_QUEUE; 2711 2712 if (q_vector->rx.ring) 2713 rx_queue = q_vector->rx.ring->reg_idx; 2714 if (q_vector->tx.ring) 2715 tx_queue = q_vector->tx.ring->reg_idx; 2716 2717 switch (hw->mac.type) { 2718 case igc_i225: 2719 if (rx_queue > IGC_N0_QUEUE) 2720 igc_write_ivar(hw, msix_vector, 2721 rx_queue >> 1, 2722 (rx_queue & 0x1) << 4); 2723 if (tx_queue > IGC_N0_QUEUE) 2724 igc_write_ivar(hw, msix_vector, 2725 tx_queue >> 1, 2726 ((tx_queue & 0x1) << 4) + 8); 2727 q_vector->eims_value = BIT(msix_vector); 2728 break; 2729 default: 2730 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 2731 break; 2732 } 2733 2734 /* add q_vector eims value to global eims_enable_mask */ 2735 adapter->eims_enable_mask |= q_vector->eims_value; 2736 2737 /* configure q_vector to set itr on first interrupt */ 2738 q_vector->set_itr = 1; 2739 } 2740 2741 /** 2742 * igc_configure_msix - Configure MSI-X hardware 2743 * @adapter: Pointer to adapter structure 2744 * 2745 * igc_configure_msix sets up the hardware to properly 2746 * generate MSI-X interrupts. 2747 */ 2748 static void igc_configure_msix(struct igc_adapter *adapter) 2749 { 2750 struct igc_hw *hw = &adapter->hw; 2751 int i, vector = 0; 2752 u32 tmp; 2753 2754 adapter->eims_enable_mask = 0; 2755 2756 /* set vector for other causes, i.e. link changes */ 2757 switch (hw->mac.type) { 2758 case igc_i225: 2759 /* Turn on MSI-X capability first, or our settings 2760 * won't stick. And it will take days to debug. 2761 */ 2762 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 2763 IGC_GPIE_PBA | IGC_GPIE_EIAME | 2764 IGC_GPIE_NSICR); 2765 2766 /* enable msix_other interrupt */ 2767 adapter->eims_other = BIT(vector); 2768 tmp = (vector++ | IGC_IVAR_VALID) << 8; 2769 2770 wr32(IGC_IVAR_MISC, tmp); 2771 break; 2772 default: 2773 /* do nothing, since nothing else supports MSI-X */ 2774 break; 2775 } /* switch (hw->mac.type) */ 2776 2777 adapter->eims_enable_mask |= adapter->eims_other; 2778 2779 for (i = 0; i < adapter->num_q_vectors; i++) 2780 igc_assign_vector(adapter->q_vector[i], vector++); 2781 2782 wrfl(); 2783 } 2784 2785 /** 2786 * igc_irq_enable - Enable default interrupt generation settings 2787 * @adapter: board private structure 2788 */ 2789 static void igc_irq_enable(struct igc_adapter *adapter) 2790 { 2791 struct igc_hw *hw = &adapter->hw; 2792 2793 if (adapter->msix_entries) { 2794 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 2795 u32 regval = rd32(IGC_EIAC); 2796 2797 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 2798 regval = rd32(IGC_EIAM); 2799 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 2800 wr32(IGC_EIMS, adapter->eims_enable_mask); 2801 wr32(IGC_IMS, ims); 2802 } else { 2803 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2804 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2805 } 2806 } 2807 2808 /** 2809 * igc_irq_disable - Mask off interrupt generation on the NIC 2810 * @adapter: board private structure 2811 */ 2812 static void igc_irq_disable(struct igc_adapter *adapter) 2813 { 2814 struct igc_hw *hw = &adapter->hw; 2815 2816 if (adapter->msix_entries) { 2817 u32 regval = rd32(IGC_EIAM); 2818 2819 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 2820 wr32(IGC_EIMC, adapter->eims_enable_mask); 2821 regval = rd32(IGC_EIAC); 2822 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 2823 } 2824 2825 wr32(IGC_IAM, 0); 2826 wr32(IGC_IMC, ~0); 2827 wrfl(); 2828 2829 if (adapter->msix_entries) { 2830 int vector = 0, i; 2831 2832 synchronize_irq(adapter->msix_entries[vector++].vector); 2833 2834 for (i = 0; i < adapter->num_q_vectors; i++) 2835 synchronize_irq(adapter->msix_entries[vector++].vector); 2836 } else { 2837 synchronize_irq(adapter->pdev->irq); 2838 } 2839 } 2840 2841 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 2842 const u32 max_rss_queues) 2843 { 2844 /* Determine if we need to pair queues. */ 2845 /* If rss_queues > half of max_rss_queues, pair the queues in 2846 * order to conserve interrupts due to limited supply. 2847 */ 2848 if (adapter->rss_queues > (max_rss_queues / 2)) 2849 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 2850 else 2851 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 2852 } 2853 2854 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 2855 { 2856 return IGC_MAX_RX_QUEUES; 2857 } 2858 2859 static void igc_init_queue_configuration(struct igc_adapter *adapter) 2860 { 2861 u32 max_rss_queues; 2862 2863 max_rss_queues = igc_get_max_rss_queues(adapter); 2864 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 2865 2866 igc_set_flag_queue_pairs(adapter, max_rss_queues); 2867 } 2868 2869 /** 2870 * igc_reset_q_vector - Reset config for interrupt vector 2871 * @adapter: board private structure to initialize 2872 * @v_idx: Index of vector to be reset 2873 * 2874 * If NAPI is enabled it will delete any references to the 2875 * NAPI struct. This is preparation for igc_free_q_vector. 2876 */ 2877 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 2878 { 2879 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2880 2881 /* if we're coming from igc_set_interrupt_capability, the vectors are 2882 * not yet allocated 2883 */ 2884 if (!q_vector) 2885 return; 2886 2887 if (q_vector->tx.ring) 2888 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 2889 2890 if (q_vector->rx.ring) 2891 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 2892 2893 netif_napi_del(&q_vector->napi); 2894 } 2895 2896 /** 2897 * igc_free_q_vector - Free memory allocated for specific interrupt vector 2898 * @adapter: board private structure to initialize 2899 * @v_idx: Index of vector to be freed 2900 * 2901 * This function frees the memory allocated to the q_vector. 2902 */ 2903 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 2904 { 2905 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2906 2907 adapter->q_vector[v_idx] = NULL; 2908 2909 /* igc_get_stats64() might access the rings on this vector, 2910 * we must wait a grace period before freeing it. 2911 */ 2912 if (q_vector) 2913 kfree_rcu(q_vector, rcu); 2914 } 2915 2916 /** 2917 * igc_free_q_vectors - Free memory allocated for interrupt vectors 2918 * @adapter: board private structure to initialize 2919 * 2920 * This function frees the memory allocated to the q_vectors. In addition if 2921 * NAPI is enabled it will delete any references to the NAPI struct prior 2922 * to freeing the q_vector. 2923 */ 2924 static void igc_free_q_vectors(struct igc_adapter *adapter) 2925 { 2926 int v_idx = adapter->num_q_vectors; 2927 2928 adapter->num_tx_queues = 0; 2929 adapter->num_rx_queues = 0; 2930 adapter->num_q_vectors = 0; 2931 2932 while (v_idx--) { 2933 igc_reset_q_vector(adapter, v_idx); 2934 igc_free_q_vector(adapter, v_idx); 2935 } 2936 } 2937 2938 /** 2939 * igc_update_itr - update the dynamic ITR value based on statistics 2940 * @q_vector: pointer to q_vector 2941 * @ring_container: ring info to update the itr for 2942 * 2943 * Stores a new ITR value based on packets and byte 2944 * counts during the last interrupt. The advantage of per interrupt 2945 * computation is faster updates and more accurate ITR for the current 2946 * traffic pattern. Constants in this function were computed 2947 * based on theoretical maximum wire speed and thresholds were set based 2948 * on testing data as well as attempting to minimize response time 2949 * while increasing bulk throughput. 2950 * NOTE: These calculations are only valid when operating in a single- 2951 * queue environment. 2952 */ 2953 static void igc_update_itr(struct igc_q_vector *q_vector, 2954 struct igc_ring_container *ring_container) 2955 { 2956 unsigned int packets = ring_container->total_packets; 2957 unsigned int bytes = ring_container->total_bytes; 2958 u8 itrval = ring_container->itr; 2959 2960 /* no packets, exit with status unchanged */ 2961 if (packets == 0) 2962 return; 2963 2964 switch (itrval) { 2965 case lowest_latency: 2966 /* handle TSO and jumbo frames */ 2967 if (bytes / packets > 8000) 2968 itrval = bulk_latency; 2969 else if ((packets < 5) && (bytes > 512)) 2970 itrval = low_latency; 2971 break; 2972 case low_latency: /* 50 usec aka 20000 ints/s */ 2973 if (bytes > 10000) { 2974 /* this if handles the TSO accounting */ 2975 if (bytes / packets > 8000) 2976 itrval = bulk_latency; 2977 else if ((packets < 10) || ((bytes / packets) > 1200)) 2978 itrval = bulk_latency; 2979 else if ((packets > 35)) 2980 itrval = lowest_latency; 2981 } else if (bytes / packets > 2000) { 2982 itrval = bulk_latency; 2983 } else if (packets <= 2 && bytes < 512) { 2984 itrval = lowest_latency; 2985 } 2986 break; 2987 case bulk_latency: /* 250 usec aka 4000 ints/s */ 2988 if (bytes > 25000) { 2989 if (packets > 35) 2990 itrval = low_latency; 2991 } else if (bytes < 1500) { 2992 itrval = low_latency; 2993 } 2994 break; 2995 } 2996 2997 /* clear work counters since we have the values we need */ 2998 ring_container->total_bytes = 0; 2999 ring_container->total_packets = 0; 3000 3001 /* write updated itr to ring container */ 3002 ring_container->itr = itrval; 3003 } 3004 3005 static void igc_set_itr(struct igc_q_vector *q_vector) 3006 { 3007 struct igc_adapter *adapter = q_vector->adapter; 3008 u32 new_itr = q_vector->itr_val; 3009 u8 current_itr = 0; 3010 3011 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 3012 switch (adapter->link_speed) { 3013 case SPEED_10: 3014 case SPEED_100: 3015 current_itr = 0; 3016 new_itr = IGC_4K_ITR; 3017 goto set_itr_now; 3018 default: 3019 break; 3020 } 3021 3022 igc_update_itr(q_vector, &q_vector->tx); 3023 igc_update_itr(q_vector, &q_vector->rx); 3024 3025 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 3026 3027 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3028 if (current_itr == lowest_latency && 3029 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3030 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3031 current_itr = low_latency; 3032 3033 switch (current_itr) { 3034 /* counts and packets in update_itr are dependent on these numbers */ 3035 case lowest_latency: 3036 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 3037 break; 3038 case low_latency: 3039 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 3040 break; 3041 case bulk_latency: 3042 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 3043 break; 3044 default: 3045 break; 3046 } 3047 3048 set_itr_now: 3049 if (new_itr != q_vector->itr_val) { 3050 /* this attempts to bias the interrupt rate towards Bulk 3051 * by adding intermediate steps when interrupt rate is 3052 * increasing 3053 */ 3054 new_itr = new_itr > q_vector->itr_val ? 3055 max((new_itr * q_vector->itr_val) / 3056 (new_itr + (q_vector->itr_val >> 2)), 3057 new_itr) : new_itr; 3058 /* Don't write the value here; it resets the adapter's 3059 * internal timer, and causes us to delay far longer than 3060 * we should between interrupts. Instead, we write the ITR 3061 * value at the beginning of the next interrupt so the timing 3062 * ends up being correct. 3063 */ 3064 q_vector->itr_val = new_itr; 3065 q_vector->set_itr = 1; 3066 } 3067 } 3068 3069 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 3070 { 3071 int v_idx = adapter->num_q_vectors; 3072 3073 if (adapter->msix_entries) { 3074 pci_disable_msix(adapter->pdev); 3075 kfree(adapter->msix_entries); 3076 adapter->msix_entries = NULL; 3077 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 3078 pci_disable_msi(adapter->pdev); 3079 } 3080 3081 while (v_idx--) 3082 igc_reset_q_vector(adapter, v_idx); 3083 } 3084 3085 /** 3086 * igc_set_interrupt_capability - set MSI or MSI-X if supported 3087 * @adapter: Pointer to adapter structure 3088 * @msix: boolean value for MSI-X capability 3089 * 3090 * Attempt to configure interrupts using the best available 3091 * capabilities of the hardware and kernel. 3092 */ 3093 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 3094 bool msix) 3095 { 3096 int numvecs, i; 3097 int err; 3098 3099 if (!msix) 3100 goto msi_only; 3101 adapter->flags |= IGC_FLAG_HAS_MSIX; 3102 3103 /* Number of supported queues. */ 3104 adapter->num_rx_queues = adapter->rss_queues; 3105 3106 adapter->num_tx_queues = adapter->rss_queues; 3107 3108 /* start with one vector for every Rx queue */ 3109 numvecs = adapter->num_rx_queues; 3110 3111 /* if Tx handler is separate add 1 for every Tx queue */ 3112 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 3113 numvecs += adapter->num_tx_queues; 3114 3115 /* store the number of vectors reserved for queues */ 3116 adapter->num_q_vectors = numvecs; 3117 3118 /* add 1 vector for link status interrupts */ 3119 numvecs++; 3120 3121 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), 3122 GFP_KERNEL); 3123 3124 if (!adapter->msix_entries) 3125 return; 3126 3127 /* populate entry values */ 3128 for (i = 0; i < numvecs; i++) 3129 adapter->msix_entries[i].entry = i; 3130 3131 err = pci_enable_msix_range(adapter->pdev, 3132 adapter->msix_entries, 3133 numvecs, 3134 numvecs); 3135 if (err > 0) 3136 return; 3137 3138 kfree(adapter->msix_entries); 3139 adapter->msix_entries = NULL; 3140 3141 igc_reset_interrupt_capability(adapter); 3142 3143 msi_only: 3144 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 3145 3146 adapter->rss_queues = 1; 3147 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 3148 adapter->num_rx_queues = 1; 3149 adapter->num_tx_queues = 1; 3150 adapter->num_q_vectors = 1; 3151 if (!pci_enable_msi(adapter->pdev)) 3152 adapter->flags |= IGC_FLAG_HAS_MSI; 3153 } 3154 3155 /** 3156 * igc_update_ring_itr - update the dynamic ITR value based on packet size 3157 * @q_vector: pointer to q_vector 3158 * 3159 * Stores a new ITR value based on strictly on packet size. This 3160 * algorithm is less sophisticated than that used in igc_update_itr, 3161 * due to the difficulty of synchronizing statistics across multiple 3162 * receive rings. The divisors and thresholds used by this function 3163 * were determined based on theoretical maximum wire speed and testing 3164 * data, in order to minimize response time while increasing bulk 3165 * throughput. 3166 * NOTE: This function is called only when operating in a multiqueue 3167 * receive environment. 3168 */ 3169 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 3170 { 3171 struct igc_adapter *adapter = q_vector->adapter; 3172 int new_val = q_vector->itr_val; 3173 int avg_wire_size = 0; 3174 unsigned int packets; 3175 3176 /* For non-gigabit speeds, just fix the interrupt rate at 4000 3177 * ints/sec - ITR timer value of 120 ticks. 3178 */ 3179 switch (adapter->link_speed) { 3180 case SPEED_10: 3181 case SPEED_100: 3182 new_val = IGC_4K_ITR; 3183 goto set_itr_val; 3184 default: 3185 break; 3186 } 3187 3188 packets = q_vector->rx.total_packets; 3189 if (packets) 3190 avg_wire_size = q_vector->rx.total_bytes / packets; 3191 3192 packets = q_vector->tx.total_packets; 3193 if (packets) 3194 avg_wire_size = max_t(u32, avg_wire_size, 3195 q_vector->tx.total_bytes / packets); 3196 3197 /* if avg_wire_size isn't set no work was done */ 3198 if (!avg_wire_size) 3199 goto clear_counts; 3200 3201 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 3202 avg_wire_size += 24; 3203 3204 /* Don't starve jumbo frames */ 3205 avg_wire_size = min(avg_wire_size, 3000); 3206 3207 /* Give a little boost to mid-size frames */ 3208 if (avg_wire_size > 300 && avg_wire_size < 1200) 3209 new_val = avg_wire_size / 3; 3210 else 3211 new_val = avg_wire_size / 2; 3212 3213 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3214 if (new_val < IGC_20K_ITR && 3215 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3216 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3217 new_val = IGC_20K_ITR; 3218 3219 set_itr_val: 3220 if (new_val != q_vector->itr_val) { 3221 q_vector->itr_val = new_val; 3222 q_vector->set_itr = 1; 3223 } 3224 clear_counts: 3225 q_vector->rx.total_bytes = 0; 3226 q_vector->rx.total_packets = 0; 3227 q_vector->tx.total_bytes = 0; 3228 q_vector->tx.total_packets = 0; 3229 } 3230 3231 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 3232 { 3233 struct igc_adapter *adapter = q_vector->adapter; 3234 struct igc_hw *hw = &adapter->hw; 3235 3236 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 3237 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 3238 if (adapter->num_q_vectors == 1) 3239 igc_set_itr(q_vector); 3240 else 3241 igc_update_ring_itr(q_vector); 3242 } 3243 3244 if (!test_bit(__IGC_DOWN, &adapter->state)) { 3245 if (adapter->msix_entries) 3246 wr32(IGC_EIMS, q_vector->eims_value); 3247 else 3248 igc_irq_enable(adapter); 3249 } 3250 } 3251 3252 static void igc_add_ring(struct igc_ring *ring, 3253 struct igc_ring_container *head) 3254 { 3255 head->ring = ring; 3256 head->count++; 3257 } 3258 3259 /** 3260 * igc_cache_ring_register - Descriptor ring to register mapping 3261 * @adapter: board private structure to initialize 3262 * 3263 * Once we know the feature-set enabled for the device, we'll cache 3264 * the register offset the descriptor ring is assigned to. 3265 */ 3266 static void igc_cache_ring_register(struct igc_adapter *adapter) 3267 { 3268 int i = 0, j = 0; 3269 3270 switch (adapter->hw.mac.type) { 3271 case igc_i225: 3272 default: 3273 for (; i < adapter->num_rx_queues; i++) 3274 adapter->rx_ring[i]->reg_idx = i; 3275 for (; j < adapter->num_tx_queues; j++) 3276 adapter->tx_ring[j]->reg_idx = j; 3277 break; 3278 } 3279 } 3280 3281 /** 3282 * igc_poll - NAPI Rx polling callback 3283 * @napi: napi polling structure 3284 * @budget: count of how many packets we should handle 3285 */ 3286 static int igc_poll(struct napi_struct *napi, int budget) 3287 { 3288 struct igc_q_vector *q_vector = container_of(napi, 3289 struct igc_q_vector, 3290 napi); 3291 bool clean_complete = true; 3292 int work_done = 0; 3293 3294 if (q_vector->tx.ring) 3295 clean_complete = igc_clean_tx_irq(q_vector, budget); 3296 3297 if (q_vector->rx.ring) { 3298 int cleaned = igc_clean_rx_irq(q_vector, budget); 3299 3300 work_done += cleaned; 3301 if (cleaned >= budget) 3302 clean_complete = false; 3303 } 3304 3305 /* If all work not completed, return budget and keep polling */ 3306 if (!clean_complete) 3307 return budget; 3308 3309 /* Exit the polling mode, but don't re-enable interrupts if stack might 3310 * poll us due to busy-polling 3311 */ 3312 if (likely(napi_complete_done(napi, work_done))) 3313 igc_ring_irq_enable(q_vector); 3314 3315 return min(work_done, budget - 1); 3316 } 3317 3318 /** 3319 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 3320 * @adapter: board private structure to initialize 3321 * @v_count: q_vectors allocated on adapter, used for ring interleaving 3322 * @v_idx: index of vector in adapter struct 3323 * @txr_count: total number of Tx rings to allocate 3324 * @txr_idx: index of first Tx ring to allocate 3325 * @rxr_count: total number of Rx rings to allocate 3326 * @rxr_idx: index of first Rx ring to allocate 3327 * 3328 * We allocate one q_vector. If allocation fails we return -ENOMEM. 3329 */ 3330 static int igc_alloc_q_vector(struct igc_adapter *adapter, 3331 unsigned int v_count, unsigned int v_idx, 3332 unsigned int txr_count, unsigned int txr_idx, 3333 unsigned int rxr_count, unsigned int rxr_idx) 3334 { 3335 struct igc_q_vector *q_vector; 3336 struct igc_ring *ring; 3337 int ring_count; 3338 3339 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 3340 if (txr_count > 1 || rxr_count > 1) 3341 return -ENOMEM; 3342 3343 ring_count = txr_count + rxr_count; 3344 3345 /* allocate q_vector and rings */ 3346 q_vector = adapter->q_vector[v_idx]; 3347 if (!q_vector) 3348 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), 3349 GFP_KERNEL); 3350 else 3351 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 3352 if (!q_vector) 3353 return -ENOMEM; 3354 3355 /* initialize NAPI */ 3356 netif_napi_add(adapter->netdev, &q_vector->napi, 3357 igc_poll, 64); 3358 3359 /* tie q_vector and adapter together */ 3360 adapter->q_vector[v_idx] = q_vector; 3361 q_vector->adapter = adapter; 3362 3363 /* initialize work limits */ 3364 q_vector->tx.work_limit = adapter->tx_work_limit; 3365 3366 /* initialize ITR configuration */ 3367 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 3368 q_vector->itr_val = IGC_START_ITR; 3369 3370 /* initialize pointer to rings */ 3371 ring = q_vector->ring; 3372 3373 /* initialize ITR */ 3374 if (rxr_count) { 3375 /* rx or rx/tx vector */ 3376 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 3377 q_vector->itr_val = adapter->rx_itr_setting; 3378 } else { 3379 /* tx only vector */ 3380 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 3381 q_vector->itr_val = adapter->tx_itr_setting; 3382 } 3383 3384 if (txr_count) { 3385 /* assign generic ring traits */ 3386 ring->dev = &adapter->pdev->dev; 3387 ring->netdev = adapter->netdev; 3388 3389 /* configure backlink on ring */ 3390 ring->q_vector = q_vector; 3391 3392 /* update q_vector Tx values */ 3393 igc_add_ring(ring, &q_vector->tx); 3394 3395 /* apply Tx specific ring traits */ 3396 ring->count = adapter->tx_ring_count; 3397 ring->queue_index = txr_idx; 3398 3399 /* assign ring to adapter */ 3400 adapter->tx_ring[txr_idx] = ring; 3401 3402 /* push pointer to next ring */ 3403 ring++; 3404 } 3405 3406 if (rxr_count) { 3407 /* assign generic ring traits */ 3408 ring->dev = &adapter->pdev->dev; 3409 ring->netdev = adapter->netdev; 3410 3411 /* configure backlink on ring */ 3412 ring->q_vector = q_vector; 3413 3414 /* update q_vector Rx values */ 3415 igc_add_ring(ring, &q_vector->rx); 3416 3417 /* apply Rx specific ring traits */ 3418 ring->count = adapter->rx_ring_count; 3419 ring->queue_index = rxr_idx; 3420 3421 /* assign ring to adapter */ 3422 adapter->rx_ring[rxr_idx] = ring; 3423 } 3424 3425 return 0; 3426 } 3427 3428 /** 3429 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 3430 * @adapter: board private structure to initialize 3431 * 3432 * We allocate one q_vector per queue interrupt. If allocation fails we 3433 * return -ENOMEM. 3434 */ 3435 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 3436 { 3437 int rxr_remaining = adapter->num_rx_queues; 3438 int txr_remaining = adapter->num_tx_queues; 3439 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 3440 int q_vectors = adapter->num_q_vectors; 3441 int err; 3442 3443 if (q_vectors >= (rxr_remaining + txr_remaining)) { 3444 for (; rxr_remaining; v_idx++) { 3445 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3446 0, 0, 1, rxr_idx); 3447 3448 if (err) 3449 goto err_out; 3450 3451 /* update counts and index */ 3452 rxr_remaining--; 3453 rxr_idx++; 3454 } 3455 } 3456 3457 for (; v_idx < q_vectors; v_idx++) { 3458 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 3459 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 3460 3461 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3462 tqpv, txr_idx, rqpv, rxr_idx); 3463 3464 if (err) 3465 goto err_out; 3466 3467 /* update counts and index */ 3468 rxr_remaining -= rqpv; 3469 txr_remaining -= tqpv; 3470 rxr_idx++; 3471 txr_idx++; 3472 } 3473 3474 return 0; 3475 3476 err_out: 3477 adapter->num_tx_queues = 0; 3478 adapter->num_rx_queues = 0; 3479 adapter->num_q_vectors = 0; 3480 3481 while (v_idx--) 3482 igc_free_q_vector(adapter, v_idx); 3483 3484 return -ENOMEM; 3485 } 3486 3487 /** 3488 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 3489 * @adapter: Pointer to adapter structure 3490 * @msix: boolean for MSI-X capability 3491 * 3492 * This function initializes the interrupts and allocates all of the queues. 3493 */ 3494 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 3495 { 3496 struct net_device *dev = adapter->netdev; 3497 int err = 0; 3498 3499 igc_set_interrupt_capability(adapter, msix); 3500 3501 err = igc_alloc_q_vectors(adapter); 3502 if (err) { 3503 netdev_err(dev, "Unable to allocate memory for vectors\n"); 3504 goto err_alloc_q_vectors; 3505 } 3506 3507 igc_cache_ring_register(adapter); 3508 3509 return 0; 3510 3511 err_alloc_q_vectors: 3512 igc_reset_interrupt_capability(adapter); 3513 return err; 3514 } 3515 3516 /** 3517 * igc_sw_init - Initialize general software structures (struct igc_adapter) 3518 * @adapter: board private structure to initialize 3519 * 3520 * igc_sw_init initializes the Adapter private data structure. 3521 * Fields are initialized based on PCI device information and 3522 * OS network device settings (MTU size). 3523 */ 3524 static int igc_sw_init(struct igc_adapter *adapter) 3525 { 3526 struct net_device *netdev = adapter->netdev; 3527 struct pci_dev *pdev = adapter->pdev; 3528 struct igc_hw *hw = &adapter->hw; 3529 3530 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 3531 3532 /* set default ring sizes */ 3533 adapter->tx_ring_count = IGC_DEFAULT_TXD; 3534 adapter->rx_ring_count = IGC_DEFAULT_RXD; 3535 3536 /* set default ITR values */ 3537 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 3538 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 3539 3540 /* set default work limits */ 3541 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 3542 3543 /* adjust max frame to be at least the size of a standard frame */ 3544 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 3545 VLAN_HLEN; 3546 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 3547 3548 mutex_init(&adapter->nfc_rule_lock); 3549 INIT_LIST_HEAD(&adapter->nfc_rule_list); 3550 adapter->nfc_rule_count = 0; 3551 3552 spin_lock_init(&adapter->stats64_lock); 3553 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 3554 adapter->flags |= IGC_FLAG_HAS_MSIX; 3555 3556 igc_init_queue_configuration(adapter); 3557 3558 /* This call may decrease the number of queues */ 3559 if (igc_init_interrupt_scheme(adapter, true)) { 3560 netdev_err(netdev, "Unable to allocate memory for queues\n"); 3561 return -ENOMEM; 3562 } 3563 3564 /* Explicitly disable IRQ since the NIC can be in any state. */ 3565 igc_irq_disable(adapter); 3566 3567 set_bit(__IGC_DOWN, &adapter->state); 3568 3569 return 0; 3570 } 3571 3572 /** 3573 * igc_up - Open the interface and prepare it to handle traffic 3574 * @adapter: board private structure 3575 */ 3576 void igc_up(struct igc_adapter *adapter) 3577 { 3578 struct igc_hw *hw = &adapter->hw; 3579 int i = 0; 3580 3581 /* hardware has been reset, we need to reload some things */ 3582 igc_configure(adapter); 3583 3584 clear_bit(__IGC_DOWN, &adapter->state); 3585 3586 for (i = 0; i < adapter->num_q_vectors; i++) 3587 napi_enable(&adapter->q_vector[i]->napi); 3588 3589 if (adapter->msix_entries) 3590 igc_configure_msix(adapter); 3591 else 3592 igc_assign_vector(adapter->q_vector[0], 0); 3593 3594 /* Clear any pending interrupts. */ 3595 rd32(IGC_ICR); 3596 igc_irq_enable(adapter); 3597 3598 netif_tx_start_all_queues(adapter->netdev); 3599 3600 /* start the watchdog. */ 3601 hw->mac.get_link_status = 1; 3602 schedule_work(&adapter->watchdog_task); 3603 } 3604 3605 /** 3606 * igc_update_stats - Update the board statistics counters 3607 * @adapter: board private structure 3608 */ 3609 void igc_update_stats(struct igc_adapter *adapter) 3610 { 3611 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 3612 struct pci_dev *pdev = adapter->pdev; 3613 struct igc_hw *hw = &adapter->hw; 3614 u64 _bytes, _packets; 3615 u64 bytes, packets; 3616 unsigned int start; 3617 u32 mpc; 3618 int i; 3619 3620 /* Prevent stats update while adapter is being reset, or if the pci 3621 * connection is down. 3622 */ 3623 if (adapter->link_speed == 0) 3624 return; 3625 if (pci_channel_offline(pdev)) 3626 return; 3627 3628 packets = 0; 3629 bytes = 0; 3630 3631 rcu_read_lock(); 3632 for (i = 0; i < adapter->num_rx_queues; i++) { 3633 struct igc_ring *ring = adapter->rx_ring[i]; 3634 u32 rqdpc = rd32(IGC_RQDPC(i)); 3635 3636 if (hw->mac.type >= igc_i225) 3637 wr32(IGC_RQDPC(i), 0); 3638 3639 if (rqdpc) { 3640 ring->rx_stats.drops += rqdpc; 3641 net_stats->rx_fifo_errors += rqdpc; 3642 } 3643 3644 do { 3645 start = u64_stats_fetch_begin_irq(&ring->rx_syncp); 3646 _bytes = ring->rx_stats.bytes; 3647 _packets = ring->rx_stats.packets; 3648 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); 3649 bytes += _bytes; 3650 packets += _packets; 3651 } 3652 3653 net_stats->rx_bytes = bytes; 3654 net_stats->rx_packets = packets; 3655 3656 packets = 0; 3657 bytes = 0; 3658 for (i = 0; i < adapter->num_tx_queues; i++) { 3659 struct igc_ring *ring = adapter->tx_ring[i]; 3660 3661 do { 3662 start = u64_stats_fetch_begin_irq(&ring->tx_syncp); 3663 _bytes = ring->tx_stats.bytes; 3664 _packets = ring->tx_stats.packets; 3665 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); 3666 bytes += _bytes; 3667 packets += _packets; 3668 } 3669 net_stats->tx_bytes = bytes; 3670 net_stats->tx_packets = packets; 3671 rcu_read_unlock(); 3672 3673 /* read stats registers */ 3674 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 3675 adapter->stats.gprc += rd32(IGC_GPRC); 3676 adapter->stats.gorc += rd32(IGC_GORCL); 3677 rd32(IGC_GORCH); /* clear GORCL */ 3678 adapter->stats.bprc += rd32(IGC_BPRC); 3679 adapter->stats.mprc += rd32(IGC_MPRC); 3680 adapter->stats.roc += rd32(IGC_ROC); 3681 3682 adapter->stats.prc64 += rd32(IGC_PRC64); 3683 adapter->stats.prc127 += rd32(IGC_PRC127); 3684 adapter->stats.prc255 += rd32(IGC_PRC255); 3685 adapter->stats.prc511 += rd32(IGC_PRC511); 3686 adapter->stats.prc1023 += rd32(IGC_PRC1023); 3687 adapter->stats.prc1522 += rd32(IGC_PRC1522); 3688 3689 mpc = rd32(IGC_MPC); 3690 adapter->stats.mpc += mpc; 3691 net_stats->rx_fifo_errors += mpc; 3692 adapter->stats.scc += rd32(IGC_SCC); 3693 adapter->stats.ecol += rd32(IGC_ECOL); 3694 adapter->stats.mcc += rd32(IGC_MCC); 3695 adapter->stats.latecol += rd32(IGC_LATECOL); 3696 adapter->stats.dc += rd32(IGC_DC); 3697 adapter->stats.rlec += rd32(IGC_RLEC); 3698 adapter->stats.xonrxc += rd32(IGC_XONRXC); 3699 adapter->stats.xontxc += rd32(IGC_XONTXC); 3700 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 3701 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 3702 adapter->stats.fcruc += rd32(IGC_FCRUC); 3703 adapter->stats.gptc += rd32(IGC_GPTC); 3704 adapter->stats.gotc += rd32(IGC_GOTCL); 3705 rd32(IGC_GOTCH); /* clear GOTCL */ 3706 adapter->stats.rnbc += rd32(IGC_RNBC); 3707 adapter->stats.ruc += rd32(IGC_RUC); 3708 adapter->stats.rfc += rd32(IGC_RFC); 3709 adapter->stats.rjc += rd32(IGC_RJC); 3710 adapter->stats.tor += rd32(IGC_TORH); 3711 adapter->stats.tot += rd32(IGC_TOTH); 3712 adapter->stats.tpr += rd32(IGC_TPR); 3713 3714 adapter->stats.ptc64 += rd32(IGC_PTC64); 3715 adapter->stats.ptc127 += rd32(IGC_PTC127); 3716 adapter->stats.ptc255 += rd32(IGC_PTC255); 3717 adapter->stats.ptc511 += rd32(IGC_PTC511); 3718 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 3719 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 3720 3721 adapter->stats.mptc += rd32(IGC_MPTC); 3722 adapter->stats.bptc += rd32(IGC_BPTC); 3723 3724 adapter->stats.tpt += rd32(IGC_TPT); 3725 adapter->stats.colc += rd32(IGC_COLC); 3726 adapter->stats.colc += rd32(IGC_RERC); 3727 3728 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 3729 3730 adapter->stats.tsctc += rd32(IGC_TSCTC); 3731 3732 adapter->stats.iac += rd32(IGC_IAC); 3733 3734 /* Fill out the OS statistics structure */ 3735 net_stats->multicast = adapter->stats.mprc; 3736 net_stats->collisions = adapter->stats.colc; 3737 3738 /* Rx Errors */ 3739 3740 /* RLEC on some newer hardware can be incorrect so build 3741 * our own version based on RUC and ROC 3742 */ 3743 net_stats->rx_errors = adapter->stats.rxerrc + 3744 adapter->stats.crcerrs + adapter->stats.algnerrc + 3745 adapter->stats.ruc + adapter->stats.roc + 3746 adapter->stats.cexterr; 3747 net_stats->rx_length_errors = adapter->stats.ruc + 3748 adapter->stats.roc; 3749 net_stats->rx_crc_errors = adapter->stats.crcerrs; 3750 net_stats->rx_frame_errors = adapter->stats.algnerrc; 3751 net_stats->rx_missed_errors = adapter->stats.mpc; 3752 3753 /* Tx Errors */ 3754 net_stats->tx_errors = adapter->stats.ecol + 3755 adapter->stats.latecol; 3756 net_stats->tx_aborted_errors = adapter->stats.ecol; 3757 net_stats->tx_window_errors = adapter->stats.latecol; 3758 net_stats->tx_carrier_errors = adapter->stats.tncrs; 3759 3760 /* Tx Dropped needs to be maintained elsewhere */ 3761 3762 /* Management Stats */ 3763 adapter->stats.mgptc += rd32(IGC_MGTPTC); 3764 adapter->stats.mgprc += rd32(IGC_MGTPRC); 3765 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 3766 } 3767 3768 /** 3769 * igc_down - Close the interface 3770 * @adapter: board private structure 3771 */ 3772 void igc_down(struct igc_adapter *adapter) 3773 { 3774 struct net_device *netdev = adapter->netdev; 3775 struct igc_hw *hw = &adapter->hw; 3776 u32 tctl, rctl; 3777 int i = 0; 3778 3779 set_bit(__IGC_DOWN, &adapter->state); 3780 3781 /* disable receives in the hardware */ 3782 rctl = rd32(IGC_RCTL); 3783 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 3784 /* flush and sleep below */ 3785 3786 /* set trans_start so we don't get spurious watchdogs during reset */ 3787 netif_trans_update(netdev); 3788 3789 netif_carrier_off(netdev); 3790 netif_tx_stop_all_queues(netdev); 3791 3792 /* disable transmits in the hardware */ 3793 tctl = rd32(IGC_TCTL); 3794 tctl &= ~IGC_TCTL_EN; 3795 wr32(IGC_TCTL, tctl); 3796 /* flush both disables and wait for them to finish */ 3797 wrfl(); 3798 usleep_range(10000, 20000); 3799 3800 igc_irq_disable(adapter); 3801 3802 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3803 3804 for (i = 0; i < adapter->num_q_vectors; i++) { 3805 if (adapter->q_vector[i]) { 3806 napi_synchronize(&adapter->q_vector[i]->napi); 3807 napi_disable(&adapter->q_vector[i]->napi); 3808 } 3809 } 3810 3811 del_timer_sync(&adapter->watchdog_timer); 3812 del_timer_sync(&adapter->phy_info_timer); 3813 3814 /* record the stats before reset*/ 3815 spin_lock(&adapter->stats64_lock); 3816 igc_update_stats(adapter); 3817 spin_unlock(&adapter->stats64_lock); 3818 3819 adapter->link_speed = 0; 3820 adapter->link_duplex = 0; 3821 3822 if (!pci_channel_offline(adapter->pdev)) 3823 igc_reset(adapter); 3824 3825 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 3826 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 3827 3828 igc_clean_all_tx_rings(adapter); 3829 igc_clean_all_rx_rings(adapter); 3830 } 3831 3832 void igc_reinit_locked(struct igc_adapter *adapter) 3833 { 3834 WARN_ON(in_interrupt()); 3835 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3836 usleep_range(1000, 2000); 3837 igc_down(adapter); 3838 igc_up(adapter); 3839 clear_bit(__IGC_RESETTING, &adapter->state); 3840 } 3841 3842 static void igc_reset_task(struct work_struct *work) 3843 { 3844 struct igc_adapter *adapter; 3845 3846 adapter = container_of(work, struct igc_adapter, reset_task); 3847 3848 igc_rings_dump(adapter); 3849 igc_regs_dump(adapter); 3850 netdev_err(adapter->netdev, "Reset adapter\n"); 3851 igc_reinit_locked(adapter); 3852 } 3853 3854 /** 3855 * igc_change_mtu - Change the Maximum Transfer Unit 3856 * @netdev: network interface device structure 3857 * @new_mtu: new value for maximum frame size 3858 * 3859 * Returns 0 on success, negative on failure 3860 */ 3861 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 3862 { 3863 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 3864 struct igc_adapter *adapter = netdev_priv(netdev); 3865 3866 /* adjust max frame to be at least the size of a standard frame */ 3867 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 3868 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 3869 3870 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3871 usleep_range(1000, 2000); 3872 3873 /* igc_down has a dependency on max_frame_size */ 3874 adapter->max_frame_size = max_frame; 3875 3876 if (netif_running(netdev)) 3877 igc_down(adapter); 3878 3879 netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 3880 netdev->mtu = new_mtu; 3881 3882 if (netif_running(netdev)) 3883 igc_up(adapter); 3884 else 3885 igc_reset(adapter); 3886 3887 clear_bit(__IGC_RESETTING, &adapter->state); 3888 3889 return 0; 3890 } 3891 3892 /** 3893 * igc_get_stats - Get System Network Statistics 3894 * @netdev: network interface device structure 3895 * 3896 * Returns the address of the device statistics structure. 3897 * The statistics are updated here and also from the timer callback. 3898 */ 3899 static struct net_device_stats *igc_get_stats(struct net_device *netdev) 3900 { 3901 struct igc_adapter *adapter = netdev_priv(netdev); 3902 3903 if (!test_bit(__IGC_RESETTING, &adapter->state)) 3904 igc_update_stats(adapter); 3905 3906 /* only return the current stats */ 3907 return &netdev->stats; 3908 } 3909 3910 static netdev_features_t igc_fix_features(struct net_device *netdev, 3911 netdev_features_t features) 3912 { 3913 /* Since there is no support for separate Rx/Tx vlan accel 3914 * enable/disable make sure Tx flag is always in same state as Rx. 3915 */ 3916 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3917 features |= NETIF_F_HW_VLAN_CTAG_TX; 3918 else 3919 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 3920 3921 return features; 3922 } 3923 3924 static int igc_set_features(struct net_device *netdev, 3925 netdev_features_t features) 3926 { 3927 netdev_features_t changed = netdev->features ^ features; 3928 struct igc_adapter *adapter = netdev_priv(netdev); 3929 3930 /* Add VLAN support */ 3931 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 3932 return 0; 3933 3934 if (!(features & NETIF_F_NTUPLE)) 3935 igc_flush_nfc_rules(adapter); 3936 3937 netdev->features = features; 3938 3939 if (netif_running(netdev)) 3940 igc_reinit_locked(adapter); 3941 else 3942 igc_reset(adapter); 3943 3944 return 1; 3945 } 3946 3947 static netdev_features_t 3948 igc_features_check(struct sk_buff *skb, struct net_device *dev, 3949 netdev_features_t features) 3950 { 3951 unsigned int network_hdr_len, mac_hdr_len; 3952 3953 /* Make certain the headers can be described by a context descriptor */ 3954 mac_hdr_len = skb_network_header(skb) - skb->data; 3955 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 3956 return features & ~(NETIF_F_HW_CSUM | 3957 NETIF_F_SCTP_CRC | 3958 NETIF_F_HW_VLAN_CTAG_TX | 3959 NETIF_F_TSO | 3960 NETIF_F_TSO6); 3961 3962 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 3963 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 3964 return features & ~(NETIF_F_HW_CSUM | 3965 NETIF_F_SCTP_CRC | 3966 NETIF_F_TSO | 3967 NETIF_F_TSO6); 3968 3969 /* We can only support IPv4 TSO in tunnels if we can mangle the 3970 * inner IP ID field, so strip TSO if MANGLEID is not supported. 3971 */ 3972 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 3973 features &= ~NETIF_F_TSO; 3974 3975 return features; 3976 } 3977 3978 static void igc_tsync_interrupt(struct igc_adapter *adapter) 3979 { 3980 struct igc_hw *hw = &adapter->hw; 3981 u32 tsicr = rd32(IGC_TSICR); 3982 u32 ack = 0; 3983 3984 if (tsicr & IGC_TSICR_TXTS) { 3985 /* retrieve hardware timestamp */ 3986 schedule_work(&adapter->ptp_tx_work); 3987 ack |= IGC_TSICR_TXTS; 3988 } 3989 3990 /* acknowledge the interrupts */ 3991 wr32(IGC_TSICR, ack); 3992 } 3993 3994 /** 3995 * igc_msix_other - msix other interrupt handler 3996 * @irq: interrupt number 3997 * @data: pointer to a q_vector 3998 */ 3999 static irqreturn_t igc_msix_other(int irq, void *data) 4000 { 4001 struct igc_adapter *adapter = data; 4002 struct igc_hw *hw = &adapter->hw; 4003 u32 icr = rd32(IGC_ICR); 4004 4005 /* reading ICR causes bit 31 of EICR to be cleared */ 4006 if (icr & IGC_ICR_DRSTA) 4007 schedule_work(&adapter->reset_task); 4008 4009 if (icr & IGC_ICR_DOUTSYNC) { 4010 /* HW is reporting DMA is out of sync */ 4011 adapter->stats.doosync++; 4012 } 4013 4014 if (icr & IGC_ICR_LSC) { 4015 hw->mac.get_link_status = 1; 4016 /* guard against interrupt when we're going down */ 4017 if (!test_bit(__IGC_DOWN, &adapter->state)) 4018 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4019 } 4020 4021 if (icr & IGC_ICR_TS) 4022 igc_tsync_interrupt(adapter); 4023 4024 wr32(IGC_EIMS, adapter->eims_other); 4025 4026 return IRQ_HANDLED; 4027 } 4028 4029 static void igc_write_itr(struct igc_q_vector *q_vector) 4030 { 4031 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 4032 4033 if (!q_vector->set_itr) 4034 return; 4035 4036 if (!itr_val) 4037 itr_val = IGC_ITR_VAL_MASK; 4038 4039 itr_val |= IGC_EITR_CNT_IGNR; 4040 4041 writel(itr_val, q_vector->itr_register); 4042 q_vector->set_itr = 0; 4043 } 4044 4045 static irqreturn_t igc_msix_ring(int irq, void *data) 4046 { 4047 struct igc_q_vector *q_vector = data; 4048 4049 /* Write the ITR value calculated from the previous interrupt. */ 4050 igc_write_itr(q_vector); 4051 4052 napi_schedule(&q_vector->napi); 4053 4054 return IRQ_HANDLED; 4055 } 4056 4057 /** 4058 * igc_request_msix - Initialize MSI-X interrupts 4059 * @adapter: Pointer to adapter structure 4060 * 4061 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 4062 * kernel. 4063 */ 4064 static int igc_request_msix(struct igc_adapter *adapter) 4065 { 4066 int i = 0, err = 0, vector = 0, free_vector = 0; 4067 struct net_device *netdev = adapter->netdev; 4068 4069 err = request_irq(adapter->msix_entries[vector].vector, 4070 &igc_msix_other, 0, netdev->name, adapter); 4071 if (err) 4072 goto err_out; 4073 4074 for (i = 0; i < adapter->num_q_vectors; i++) { 4075 struct igc_q_vector *q_vector = adapter->q_vector[i]; 4076 4077 vector++; 4078 4079 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 4080 4081 if (q_vector->rx.ring && q_vector->tx.ring) 4082 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 4083 q_vector->rx.ring->queue_index); 4084 else if (q_vector->tx.ring) 4085 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 4086 q_vector->tx.ring->queue_index); 4087 else if (q_vector->rx.ring) 4088 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 4089 q_vector->rx.ring->queue_index); 4090 else 4091 sprintf(q_vector->name, "%s-unused", netdev->name); 4092 4093 err = request_irq(adapter->msix_entries[vector].vector, 4094 igc_msix_ring, 0, q_vector->name, 4095 q_vector); 4096 if (err) 4097 goto err_free; 4098 } 4099 4100 igc_configure_msix(adapter); 4101 return 0; 4102 4103 err_free: 4104 /* free already assigned IRQs */ 4105 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 4106 4107 vector--; 4108 for (i = 0; i < vector; i++) { 4109 free_irq(adapter->msix_entries[free_vector++].vector, 4110 adapter->q_vector[i]); 4111 } 4112 err_out: 4113 return err; 4114 } 4115 4116 /** 4117 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 4118 * @adapter: Pointer to adapter structure 4119 * 4120 * This function resets the device so that it has 0 rx queues, tx queues, and 4121 * MSI-X interrupts allocated. 4122 */ 4123 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 4124 { 4125 igc_free_q_vectors(adapter); 4126 igc_reset_interrupt_capability(adapter); 4127 } 4128 4129 /* Need to wait a few seconds after link up to get diagnostic information from 4130 * the phy 4131 */ 4132 static void igc_update_phy_info(struct timer_list *t) 4133 { 4134 struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); 4135 4136 igc_get_phy_info(&adapter->hw); 4137 } 4138 4139 /** 4140 * igc_has_link - check shared code for link and determine up/down 4141 * @adapter: pointer to driver private info 4142 */ 4143 bool igc_has_link(struct igc_adapter *adapter) 4144 { 4145 struct igc_hw *hw = &adapter->hw; 4146 bool link_active = false; 4147 4148 /* get_link_status is set on LSC (link status) interrupt or 4149 * rx sequence error interrupt. get_link_status will stay 4150 * false until the igc_check_for_link establishes link 4151 * for copper adapters ONLY 4152 */ 4153 switch (hw->phy.media_type) { 4154 case igc_media_type_copper: 4155 if (!hw->mac.get_link_status) 4156 return true; 4157 hw->mac.ops.check_for_link(hw); 4158 link_active = !hw->mac.get_link_status; 4159 break; 4160 default: 4161 case igc_media_type_unknown: 4162 break; 4163 } 4164 4165 if (hw->mac.type == igc_i225 && 4166 hw->phy.id == I225_I_PHY_ID) { 4167 if (!netif_carrier_ok(adapter->netdev)) { 4168 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4169 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 4170 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 4171 adapter->link_check_timeout = jiffies; 4172 } 4173 } 4174 4175 return link_active; 4176 } 4177 4178 /** 4179 * igc_watchdog - Timer Call-back 4180 * @t: timer for the watchdog 4181 */ 4182 static void igc_watchdog(struct timer_list *t) 4183 { 4184 struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); 4185 /* Do the rest outside of interrupt context */ 4186 schedule_work(&adapter->watchdog_task); 4187 } 4188 4189 static void igc_watchdog_task(struct work_struct *work) 4190 { 4191 struct igc_adapter *adapter = container_of(work, 4192 struct igc_adapter, 4193 watchdog_task); 4194 struct net_device *netdev = adapter->netdev; 4195 struct igc_hw *hw = &adapter->hw; 4196 struct igc_phy_info *phy = &hw->phy; 4197 u16 phy_data, retry_count = 20; 4198 u32 link; 4199 int i; 4200 4201 link = igc_has_link(adapter); 4202 4203 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 4204 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 4205 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4206 else 4207 link = false; 4208 } 4209 4210 if (link) { 4211 /* Cancel scheduled suspend requests. */ 4212 pm_runtime_resume(netdev->dev.parent); 4213 4214 if (!netif_carrier_ok(netdev)) { 4215 u32 ctrl; 4216 4217 hw->mac.ops.get_speed_and_duplex(hw, 4218 &adapter->link_speed, 4219 &adapter->link_duplex); 4220 4221 ctrl = rd32(IGC_CTRL); 4222 /* Link status message must follow this format */ 4223 netdev_info(netdev, 4224 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 4225 adapter->link_speed, 4226 adapter->link_duplex == FULL_DUPLEX ? 4227 "Full" : "Half", 4228 (ctrl & IGC_CTRL_TFCE) && 4229 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 4230 (ctrl & IGC_CTRL_RFCE) ? "RX" : 4231 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 4232 4233 /* disable EEE if enabled */ 4234 if ((adapter->flags & IGC_FLAG_EEE) && 4235 adapter->link_duplex == HALF_DUPLEX) { 4236 netdev_info(netdev, 4237 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n"); 4238 adapter->hw.dev_spec._base.eee_enable = false; 4239 adapter->flags &= ~IGC_FLAG_EEE; 4240 } 4241 4242 /* check if SmartSpeed worked */ 4243 igc_check_downshift(hw); 4244 if (phy->speed_downgraded) 4245 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 4246 4247 /* adjust timeout factor according to speed/duplex */ 4248 adapter->tx_timeout_factor = 1; 4249 switch (adapter->link_speed) { 4250 case SPEED_10: 4251 adapter->tx_timeout_factor = 14; 4252 break; 4253 case SPEED_100: 4254 /* maybe add some timeout factor ? */ 4255 break; 4256 } 4257 4258 if (adapter->link_speed != SPEED_1000) 4259 goto no_wait; 4260 4261 /* wait for Remote receiver status OK */ 4262 retry_read_status: 4263 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 4264 &phy_data)) { 4265 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 4266 retry_count) { 4267 msleep(100); 4268 retry_count--; 4269 goto retry_read_status; 4270 } else if (!retry_count) { 4271 netdev_err(netdev, "exceed max 2 second\n"); 4272 } 4273 } else { 4274 netdev_err(netdev, "read 1000Base-T Status Reg\n"); 4275 } 4276 no_wait: 4277 netif_carrier_on(netdev); 4278 4279 /* link state has changed, schedule phy info update */ 4280 if (!test_bit(__IGC_DOWN, &adapter->state)) 4281 mod_timer(&adapter->phy_info_timer, 4282 round_jiffies(jiffies + 2 * HZ)); 4283 } 4284 } else { 4285 if (netif_carrier_ok(netdev)) { 4286 adapter->link_speed = 0; 4287 adapter->link_duplex = 0; 4288 4289 /* Links status message must follow this format */ 4290 netdev_info(netdev, "NIC Link is Down\n"); 4291 netif_carrier_off(netdev); 4292 4293 /* link state has changed, schedule phy info update */ 4294 if (!test_bit(__IGC_DOWN, &adapter->state)) 4295 mod_timer(&adapter->phy_info_timer, 4296 round_jiffies(jiffies + 2 * HZ)); 4297 4298 /* link is down, time to check for alternate media */ 4299 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 4300 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4301 schedule_work(&adapter->reset_task); 4302 /* return immediately */ 4303 return; 4304 } 4305 } 4306 pm_schedule_suspend(netdev->dev.parent, 4307 MSEC_PER_SEC * 5); 4308 4309 /* also check for alternate media here */ 4310 } else if (!netif_carrier_ok(netdev) && 4311 (adapter->flags & IGC_FLAG_MAS_ENABLE)) { 4312 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4313 schedule_work(&adapter->reset_task); 4314 /* return immediately */ 4315 return; 4316 } 4317 } 4318 } 4319 4320 spin_lock(&adapter->stats64_lock); 4321 igc_update_stats(adapter); 4322 spin_unlock(&adapter->stats64_lock); 4323 4324 for (i = 0; i < adapter->num_tx_queues; i++) { 4325 struct igc_ring *tx_ring = adapter->tx_ring[i]; 4326 4327 if (!netif_carrier_ok(netdev)) { 4328 /* We've lost link, so the controller stops DMA, 4329 * but we've got queued Tx work that's never going 4330 * to get done, so reset controller to flush Tx. 4331 * (Do the reset outside of interrupt context). 4332 */ 4333 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 4334 adapter->tx_timeout_count++; 4335 schedule_work(&adapter->reset_task); 4336 /* return immediately since reset is imminent */ 4337 return; 4338 } 4339 } 4340 4341 /* Force detection of hung controller every watchdog period */ 4342 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 4343 } 4344 4345 /* Cause software interrupt to ensure Rx ring is cleaned */ 4346 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4347 u32 eics = 0; 4348 4349 for (i = 0; i < adapter->num_q_vectors; i++) 4350 eics |= adapter->q_vector[i]->eims_value; 4351 wr32(IGC_EICS, eics); 4352 } else { 4353 wr32(IGC_ICS, IGC_ICS_RXDMT0); 4354 } 4355 4356 igc_ptp_tx_hang(adapter); 4357 4358 /* Reset the timer */ 4359 if (!test_bit(__IGC_DOWN, &adapter->state)) { 4360 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 4361 mod_timer(&adapter->watchdog_timer, 4362 round_jiffies(jiffies + HZ)); 4363 else 4364 mod_timer(&adapter->watchdog_timer, 4365 round_jiffies(jiffies + 2 * HZ)); 4366 } 4367 } 4368 4369 /** 4370 * igc_intr_msi - Interrupt Handler 4371 * @irq: interrupt number 4372 * @data: pointer to a network interface device structure 4373 */ 4374 static irqreturn_t igc_intr_msi(int irq, void *data) 4375 { 4376 struct igc_adapter *adapter = data; 4377 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4378 struct igc_hw *hw = &adapter->hw; 4379 /* read ICR disables interrupts using IAM */ 4380 u32 icr = rd32(IGC_ICR); 4381 4382 igc_write_itr(q_vector); 4383 4384 if (icr & IGC_ICR_DRSTA) 4385 schedule_work(&adapter->reset_task); 4386 4387 if (icr & IGC_ICR_DOUTSYNC) { 4388 /* HW is reporting DMA is out of sync */ 4389 adapter->stats.doosync++; 4390 } 4391 4392 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4393 hw->mac.get_link_status = 1; 4394 if (!test_bit(__IGC_DOWN, &adapter->state)) 4395 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4396 } 4397 4398 napi_schedule(&q_vector->napi); 4399 4400 return IRQ_HANDLED; 4401 } 4402 4403 /** 4404 * igc_intr - Legacy Interrupt Handler 4405 * @irq: interrupt number 4406 * @data: pointer to a network interface device structure 4407 */ 4408 static irqreturn_t igc_intr(int irq, void *data) 4409 { 4410 struct igc_adapter *adapter = data; 4411 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4412 struct igc_hw *hw = &adapter->hw; 4413 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 4414 * need for the IMC write 4415 */ 4416 u32 icr = rd32(IGC_ICR); 4417 4418 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 4419 * not set, then the adapter didn't send an interrupt 4420 */ 4421 if (!(icr & IGC_ICR_INT_ASSERTED)) 4422 return IRQ_NONE; 4423 4424 igc_write_itr(q_vector); 4425 4426 if (icr & IGC_ICR_DRSTA) 4427 schedule_work(&adapter->reset_task); 4428 4429 if (icr & IGC_ICR_DOUTSYNC) { 4430 /* HW is reporting DMA is out of sync */ 4431 adapter->stats.doosync++; 4432 } 4433 4434 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4435 hw->mac.get_link_status = 1; 4436 /* guard against interrupt when we're going down */ 4437 if (!test_bit(__IGC_DOWN, &adapter->state)) 4438 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4439 } 4440 4441 napi_schedule(&q_vector->napi); 4442 4443 return IRQ_HANDLED; 4444 } 4445 4446 static void igc_free_irq(struct igc_adapter *adapter) 4447 { 4448 if (adapter->msix_entries) { 4449 int vector = 0, i; 4450 4451 free_irq(adapter->msix_entries[vector++].vector, adapter); 4452 4453 for (i = 0; i < adapter->num_q_vectors; i++) 4454 free_irq(adapter->msix_entries[vector++].vector, 4455 adapter->q_vector[i]); 4456 } else { 4457 free_irq(adapter->pdev->irq, adapter); 4458 } 4459 } 4460 4461 /** 4462 * igc_request_irq - initialize interrupts 4463 * @adapter: Pointer to adapter structure 4464 * 4465 * Attempts to configure interrupts using the best available 4466 * capabilities of the hardware and kernel. 4467 */ 4468 static int igc_request_irq(struct igc_adapter *adapter) 4469 { 4470 struct net_device *netdev = adapter->netdev; 4471 struct pci_dev *pdev = adapter->pdev; 4472 int err = 0; 4473 4474 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4475 err = igc_request_msix(adapter); 4476 if (!err) 4477 goto request_done; 4478 /* fall back to MSI */ 4479 igc_free_all_tx_resources(adapter); 4480 igc_free_all_rx_resources(adapter); 4481 4482 igc_clear_interrupt_scheme(adapter); 4483 err = igc_init_interrupt_scheme(adapter, false); 4484 if (err) 4485 goto request_done; 4486 igc_setup_all_tx_resources(adapter); 4487 igc_setup_all_rx_resources(adapter); 4488 igc_configure(adapter); 4489 } 4490 4491 igc_assign_vector(adapter->q_vector[0], 0); 4492 4493 if (adapter->flags & IGC_FLAG_HAS_MSI) { 4494 err = request_irq(pdev->irq, &igc_intr_msi, 0, 4495 netdev->name, adapter); 4496 if (!err) 4497 goto request_done; 4498 4499 /* fall back to legacy interrupts */ 4500 igc_reset_interrupt_capability(adapter); 4501 adapter->flags &= ~IGC_FLAG_HAS_MSI; 4502 } 4503 4504 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 4505 netdev->name, adapter); 4506 4507 if (err) 4508 netdev_err(netdev, "Error %d getting interrupt\n", err); 4509 4510 request_done: 4511 return err; 4512 } 4513 4514 /** 4515 * __igc_open - Called when a network interface is made active 4516 * @netdev: network interface device structure 4517 * @resuming: boolean indicating if the device is resuming 4518 * 4519 * Returns 0 on success, negative value on failure 4520 * 4521 * The open entry point is called when a network interface is made 4522 * active by the system (IFF_UP). At this point all resources needed 4523 * for transmit and receive operations are allocated, the interrupt 4524 * handler is registered with the OS, the watchdog timer is started, 4525 * and the stack is notified that the interface is ready. 4526 */ 4527 static int __igc_open(struct net_device *netdev, bool resuming) 4528 { 4529 struct igc_adapter *adapter = netdev_priv(netdev); 4530 struct pci_dev *pdev = adapter->pdev; 4531 struct igc_hw *hw = &adapter->hw; 4532 int err = 0; 4533 int i = 0; 4534 4535 /* disallow open during test */ 4536 4537 if (test_bit(__IGC_TESTING, &adapter->state)) { 4538 WARN_ON(resuming); 4539 return -EBUSY; 4540 } 4541 4542 if (!resuming) 4543 pm_runtime_get_sync(&pdev->dev); 4544 4545 netif_carrier_off(netdev); 4546 4547 /* allocate transmit descriptors */ 4548 err = igc_setup_all_tx_resources(adapter); 4549 if (err) 4550 goto err_setup_tx; 4551 4552 /* allocate receive descriptors */ 4553 err = igc_setup_all_rx_resources(adapter); 4554 if (err) 4555 goto err_setup_rx; 4556 4557 igc_power_up_link(adapter); 4558 4559 igc_configure(adapter); 4560 4561 err = igc_request_irq(adapter); 4562 if (err) 4563 goto err_req_irq; 4564 4565 /* Notify the stack of the actual queue counts. */ 4566 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 4567 if (err) 4568 goto err_set_queues; 4569 4570 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 4571 if (err) 4572 goto err_set_queues; 4573 4574 clear_bit(__IGC_DOWN, &adapter->state); 4575 4576 for (i = 0; i < adapter->num_q_vectors; i++) 4577 napi_enable(&adapter->q_vector[i]->napi); 4578 4579 /* Clear any pending interrupts. */ 4580 rd32(IGC_ICR); 4581 igc_irq_enable(adapter); 4582 4583 if (!resuming) 4584 pm_runtime_put(&pdev->dev); 4585 4586 netif_tx_start_all_queues(netdev); 4587 4588 /* start the watchdog. */ 4589 hw->mac.get_link_status = 1; 4590 schedule_work(&adapter->watchdog_task); 4591 4592 return IGC_SUCCESS; 4593 4594 err_set_queues: 4595 igc_free_irq(adapter); 4596 err_req_irq: 4597 igc_release_hw_control(adapter); 4598 igc_power_down_phy_copper_base(&adapter->hw); 4599 igc_free_all_rx_resources(adapter); 4600 err_setup_rx: 4601 igc_free_all_tx_resources(adapter); 4602 err_setup_tx: 4603 igc_reset(adapter); 4604 if (!resuming) 4605 pm_runtime_put(&pdev->dev); 4606 4607 return err; 4608 } 4609 4610 int igc_open(struct net_device *netdev) 4611 { 4612 return __igc_open(netdev, false); 4613 } 4614 4615 /** 4616 * __igc_close - Disables a network interface 4617 * @netdev: network interface device structure 4618 * @suspending: boolean indicating the device is suspending 4619 * 4620 * Returns 0, this is not allowed to fail 4621 * 4622 * The close entry point is called when an interface is de-activated 4623 * by the OS. The hardware is still under the driver's control, but 4624 * needs to be disabled. A global MAC reset is issued to stop the 4625 * hardware, and all transmit and receive resources are freed. 4626 */ 4627 static int __igc_close(struct net_device *netdev, bool suspending) 4628 { 4629 struct igc_adapter *adapter = netdev_priv(netdev); 4630 struct pci_dev *pdev = adapter->pdev; 4631 4632 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 4633 4634 if (!suspending) 4635 pm_runtime_get_sync(&pdev->dev); 4636 4637 igc_down(adapter); 4638 4639 igc_release_hw_control(adapter); 4640 4641 igc_free_irq(adapter); 4642 4643 igc_free_all_tx_resources(adapter); 4644 igc_free_all_rx_resources(adapter); 4645 4646 if (!suspending) 4647 pm_runtime_put_sync(&pdev->dev); 4648 4649 return 0; 4650 } 4651 4652 int igc_close(struct net_device *netdev) 4653 { 4654 if (netif_device_present(netdev) || netdev->dismantle) 4655 return __igc_close(netdev, false); 4656 return 0; 4657 } 4658 4659 /** 4660 * igc_ioctl - Access the hwtstamp interface 4661 * @netdev: network interface device structure 4662 * @ifreq: interface request data 4663 * @cmd: ioctl command 4664 **/ 4665 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 4666 { 4667 switch (cmd) { 4668 case SIOCGHWTSTAMP: 4669 return igc_ptp_get_ts_config(netdev, ifr); 4670 case SIOCSHWTSTAMP: 4671 return igc_ptp_set_ts_config(netdev, ifr); 4672 default: 4673 return -EOPNOTSUPP; 4674 } 4675 } 4676 4677 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, 4678 bool enable) 4679 { 4680 struct igc_ring *ring; 4681 int i; 4682 4683 if (queue < 0 || queue >= adapter->num_tx_queues) 4684 return -EINVAL; 4685 4686 ring = adapter->tx_ring[queue]; 4687 ring->launchtime_enable = enable; 4688 4689 if (adapter->base_time) 4690 return 0; 4691 4692 adapter->cycle_time = NSEC_PER_SEC; 4693 4694 for (i = 0; i < adapter->num_tx_queues; i++) { 4695 ring = adapter->tx_ring[i]; 4696 ring->start_time = 0; 4697 ring->end_time = NSEC_PER_SEC; 4698 } 4699 4700 return 0; 4701 } 4702 4703 static bool validate_schedule(const struct tc_taprio_qopt_offload *qopt) 4704 { 4705 int queue_uses[IGC_MAX_TX_QUEUES] = { }; 4706 size_t n; 4707 4708 if (qopt->cycle_time_extension) 4709 return false; 4710 4711 for (n = 0; n < qopt->num_entries; n++) { 4712 const struct tc_taprio_sched_entry *e; 4713 int i; 4714 4715 e = &qopt->entries[n]; 4716 4717 /* i225 only supports "global" frame preemption 4718 * settings. 4719 */ 4720 if (e->command != TC_TAPRIO_CMD_SET_GATES) 4721 return false; 4722 4723 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4724 if (e->gate_mask & BIT(i)) 4725 queue_uses[i]++; 4726 4727 if (queue_uses[i] > 1) 4728 return false; 4729 } 4730 } 4731 4732 return true; 4733 } 4734 4735 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, 4736 struct tc_etf_qopt_offload *qopt) 4737 { 4738 struct igc_hw *hw = &adapter->hw; 4739 int err; 4740 4741 if (hw->mac.type != igc_i225) 4742 return -EOPNOTSUPP; 4743 4744 err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable); 4745 if (err) 4746 return err; 4747 4748 return igc_tsn_offload_apply(adapter); 4749 } 4750 4751 static int igc_save_qbv_schedule(struct igc_adapter *adapter, 4752 struct tc_taprio_qopt_offload *qopt) 4753 { 4754 u32 start_time = 0, end_time = 0; 4755 size_t n; 4756 4757 if (!qopt->enable) { 4758 adapter->base_time = 0; 4759 return 0; 4760 } 4761 4762 if (adapter->base_time) 4763 return -EALREADY; 4764 4765 if (!validate_schedule(qopt)) 4766 return -EINVAL; 4767 4768 adapter->cycle_time = qopt->cycle_time; 4769 adapter->base_time = qopt->base_time; 4770 4771 /* FIXME: be a little smarter about cases when the gate for a 4772 * queue stays open for more than one entry. 4773 */ 4774 for (n = 0; n < qopt->num_entries; n++) { 4775 struct tc_taprio_sched_entry *e = &qopt->entries[n]; 4776 int i; 4777 4778 end_time += e->interval; 4779 4780 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4781 struct igc_ring *ring = adapter->tx_ring[i]; 4782 4783 if (!(e->gate_mask & BIT(i))) 4784 continue; 4785 4786 ring->start_time = start_time; 4787 ring->end_time = end_time; 4788 } 4789 4790 start_time += e->interval; 4791 } 4792 4793 return 0; 4794 } 4795 4796 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, 4797 struct tc_taprio_qopt_offload *qopt) 4798 { 4799 struct igc_hw *hw = &adapter->hw; 4800 int err; 4801 4802 if (hw->mac.type != igc_i225) 4803 return -EOPNOTSUPP; 4804 4805 err = igc_save_qbv_schedule(adapter, qopt); 4806 if (err) 4807 return err; 4808 4809 return igc_tsn_offload_apply(adapter); 4810 } 4811 4812 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, 4813 void *type_data) 4814 { 4815 struct igc_adapter *adapter = netdev_priv(dev); 4816 4817 switch (type) { 4818 case TC_SETUP_QDISC_TAPRIO: 4819 return igc_tsn_enable_qbv_scheduling(adapter, type_data); 4820 4821 case TC_SETUP_QDISC_ETF: 4822 return igc_tsn_enable_launchtime(adapter, type_data); 4823 4824 default: 4825 return -EOPNOTSUPP; 4826 } 4827 } 4828 4829 static const struct net_device_ops igc_netdev_ops = { 4830 .ndo_open = igc_open, 4831 .ndo_stop = igc_close, 4832 .ndo_start_xmit = igc_xmit_frame, 4833 .ndo_set_rx_mode = igc_set_rx_mode, 4834 .ndo_set_mac_address = igc_set_mac, 4835 .ndo_change_mtu = igc_change_mtu, 4836 .ndo_get_stats = igc_get_stats, 4837 .ndo_fix_features = igc_fix_features, 4838 .ndo_set_features = igc_set_features, 4839 .ndo_features_check = igc_features_check, 4840 .ndo_do_ioctl = igc_ioctl, 4841 .ndo_setup_tc = igc_setup_tc, 4842 }; 4843 4844 /* PCIe configuration access */ 4845 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4846 { 4847 struct igc_adapter *adapter = hw->back; 4848 4849 pci_read_config_word(adapter->pdev, reg, value); 4850 } 4851 4852 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4853 { 4854 struct igc_adapter *adapter = hw->back; 4855 4856 pci_write_config_word(adapter->pdev, reg, *value); 4857 } 4858 4859 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4860 { 4861 struct igc_adapter *adapter = hw->back; 4862 4863 if (!pci_is_pcie(adapter->pdev)) 4864 return -IGC_ERR_CONFIG; 4865 4866 pcie_capability_read_word(adapter->pdev, reg, value); 4867 4868 return IGC_SUCCESS; 4869 } 4870 4871 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4872 { 4873 struct igc_adapter *adapter = hw->back; 4874 4875 if (!pci_is_pcie(adapter->pdev)) 4876 return -IGC_ERR_CONFIG; 4877 4878 pcie_capability_write_word(adapter->pdev, reg, *value); 4879 4880 return IGC_SUCCESS; 4881 } 4882 4883 u32 igc_rd32(struct igc_hw *hw, u32 reg) 4884 { 4885 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 4886 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 4887 u32 value = 0; 4888 4889 value = readl(&hw_addr[reg]); 4890 4891 /* reads should not return all F's */ 4892 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 4893 struct net_device *netdev = igc->netdev; 4894 4895 hw->hw_addr = NULL; 4896 netif_device_detach(netdev); 4897 netdev_err(netdev, "PCIe link lost, device now detached\n"); 4898 WARN(pci_device_is_present(igc->pdev), 4899 "igc: Failed to read reg 0x%x!\n", reg); 4900 } 4901 4902 return value; 4903 } 4904 4905 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) 4906 { 4907 struct igc_mac_info *mac = &adapter->hw.mac; 4908 4909 mac->autoneg = 0; 4910 4911 /* Make sure dplx is at most 1 bit and lsb of speed is not set 4912 * for the switch() below to work 4913 */ 4914 if ((spd & 1) || (dplx & ~1)) 4915 goto err_inval; 4916 4917 switch (spd + dplx) { 4918 case SPEED_10 + DUPLEX_HALF: 4919 mac->forced_speed_duplex = ADVERTISE_10_HALF; 4920 break; 4921 case SPEED_10 + DUPLEX_FULL: 4922 mac->forced_speed_duplex = ADVERTISE_10_FULL; 4923 break; 4924 case SPEED_100 + DUPLEX_HALF: 4925 mac->forced_speed_duplex = ADVERTISE_100_HALF; 4926 break; 4927 case SPEED_100 + DUPLEX_FULL: 4928 mac->forced_speed_duplex = ADVERTISE_100_FULL; 4929 break; 4930 case SPEED_1000 + DUPLEX_FULL: 4931 mac->autoneg = 1; 4932 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 4933 break; 4934 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 4935 goto err_inval; 4936 case SPEED_2500 + DUPLEX_FULL: 4937 mac->autoneg = 1; 4938 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 4939 break; 4940 case SPEED_2500 + DUPLEX_HALF: /* not supported */ 4941 default: 4942 goto err_inval; 4943 } 4944 4945 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 4946 adapter->hw.phy.mdix = AUTO_ALL_MODES; 4947 4948 return 0; 4949 4950 err_inval: 4951 netdev_err(adapter->netdev, "Unsupported Speed/Duplex configuration\n"); 4952 return -EINVAL; 4953 } 4954 4955 /** 4956 * igc_probe - Device Initialization Routine 4957 * @pdev: PCI device information struct 4958 * @ent: entry in igc_pci_tbl 4959 * 4960 * Returns 0 on success, negative on failure 4961 * 4962 * igc_probe initializes an adapter identified by a pci_dev structure. 4963 * The OS initialization, configuring the adapter private structure, 4964 * and a hardware reset occur. 4965 */ 4966 static int igc_probe(struct pci_dev *pdev, 4967 const struct pci_device_id *ent) 4968 { 4969 struct igc_adapter *adapter; 4970 struct net_device *netdev; 4971 struct igc_hw *hw; 4972 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 4973 int err, pci_using_dac; 4974 4975 err = pci_enable_device_mem(pdev); 4976 if (err) 4977 return err; 4978 4979 pci_using_dac = 0; 4980 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4981 if (!err) { 4982 pci_using_dac = 1; 4983 } else { 4984 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 4985 if (err) { 4986 dev_err(&pdev->dev, 4987 "No usable DMA configuration, aborting\n"); 4988 goto err_dma; 4989 } 4990 } 4991 4992 err = pci_request_mem_regions(pdev, igc_driver_name); 4993 if (err) 4994 goto err_pci_reg; 4995 4996 pci_enable_pcie_error_reporting(pdev); 4997 4998 pci_set_master(pdev); 4999 5000 err = -ENOMEM; 5001 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 5002 IGC_MAX_TX_QUEUES); 5003 5004 if (!netdev) 5005 goto err_alloc_etherdev; 5006 5007 SET_NETDEV_DEV(netdev, &pdev->dev); 5008 5009 pci_set_drvdata(pdev, netdev); 5010 adapter = netdev_priv(netdev); 5011 adapter->netdev = netdev; 5012 adapter->pdev = pdev; 5013 hw = &adapter->hw; 5014 hw->back = adapter; 5015 adapter->port_num = hw->bus.func; 5016 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 5017 5018 err = pci_save_state(pdev); 5019 if (err) 5020 goto err_ioremap; 5021 5022 err = -EIO; 5023 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 5024 pci_resource_len(pdev, 0)); 5025 if (!adapter->io_addr) 5026 goto err_ioremap; 5027 5028 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 5029 hw->hw_addr = adapter->io_addr; 5030 5031 netdev->netdev_ops = &igc_netdev_ops; 5032 igc_ethtool_set_ops(netdev); 5033 netdev->watchdog_timeo = 5 * HZ; 5034 5035 netdev->mem_start = pci_resource_start(pdev, 0); 5036 netdev->mem_end = pci_resource_end(pdev, 0); 5037 5038 /* PCI config space info */ 5039 hw->vendor_id = pdev->vendor; 5040 hw->device_id = pdev->device; 5041 hw->revision_id = pdev->revision; 5042 hw->subsystem_vendor_id = pdev->subsystem_vendor; 5043 hw->subsystem_device_id = pdev->subsystem_device; 5044 5045 /* Copy the default MAC and PHY function pointers */ 5046 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 5047 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 5048 5049 /* Initialize skew-specific constants */ 5050 err = ei->get_invariants(hw); 5051 if (err) 5052 goto err_sw_init; 5053 5054 /* Add supported features to the features list*/ 5055 netdev->features |= NETIF_F_SG; 5056 netdev->features |= NETIF_F_TSO; 5057 netdev->features |= NETIF_F_TSO6; 5058 netdev->features |= NETIF_F_TSO_ECN; 5059 netdev->features |= NETIF_F_RXCSUM; 5060 netdev->features |= NETIF_F_HW_CSUM; 5061 netdev->features |= NETIF_F_SCTP_CRC; 5062 netdev->features |= NETIF_F_HW_TC; 5063 5064 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 5065 NETIF_F_GSO_GRE_CSUM | \ 5066 NETIF_F_GSO_IPXIP4 | \ 5067 NETIF_F_GSO_IPXIP6 | \ 5068 NETIF_F_GSO_UDP_TUNNEL | \ 5069 NETIF_F_GSO_UDP_TUNNEL_CSUM) 5070 5071 netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; 5072 netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; 5073 5074 /* setup the private structure */ 5075 err = igc_sw_init(adapter); 5076 if (err) 5077 goto err_sw_init; 5078 5079 /* copy netdev features into list of user selectable features */ 5080 netdev->hw_features |= NETIF_F_NTUPLE; 5081 netdev->hw_features |= netdev->features; 5082 5083 if (pci_using_dac) 5084 netdev->features |= NETIF_F_HIGHDMA; 5085 5086 /* MTU range: 68 - 9216 */ 5087 netdev->min_mtu = ETH_MIN_MTU; 5088 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 5089 5090 /* before reading the NVM, reset the controller to put the device in a 5091 * known good starting state 5092 */ 5093 hw->mac.ops.reset_hw(hw); 5094 5095 if (igc_get_flash_presence_i225(hw)) { 5096 if (hw->nvm.ops.validate(hw) < 0) { 5097 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 5098 err = -EIO; 5099 goto err_eeprom; 5100 } 5101 } 5102 5103 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 5104 /* copy the MAC address out of the NVM */ 5105 if (hw->mac.ops.read_mac_addr(hw)) 5106 dev_err(&pdev->dev, "NVM Read Error\n"); 5107 } 5108 5109 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 5110 5111 if (!is_valid_ether_addr(netdev->dev_addr)) { 5112 dev_err(&pdev->dev, "Invalid MAC Address\n"); 5113 err = -EIO; 5114 goto err_eeprom; 5115 } 5116 5117 /* configure RXPBSIZE and TXPBSIZE */ 5118 wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT); 5119 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); 5120 5121 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 5122 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 5123 5124 INIT_WORK(&adapter->reset_task, igc_reset_task); 5125 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 5126 5127 /* Initialize link properties that are user-changeable */ 5128 adapter->fc_autoneg = true; 5129 hw->mac.autoneg = true; 5130 hw->phy.autoneg_advertised = 0xaf; 5131 5132 hw->fc.requested_mode = igc_fc_default; 5133 hw->fc.current_mode = igc_fc_default; 5134 5135 /* By default, support wake on port A */ 5136 adapter->flags |= IGC_FLAG_WOL_SUPPORTED; 5137 5138 /* initialize the wol settings based on the eeprom settings */ 5139 if (adapter->flags & IGC_FLAG_WOL_SUPPORTED) 5140 adapter->wol |= IGC_WUFC_MAG; 5141 5142 device_set_wakeup_enable(&adapter->pdev->dev, 5143 adapter->flags & IGC_FLAG_WOL_SUPPORTED); 5144 5145 /* reset the hardware with the new settings */ 5146 igc_reset(adapter); 5147 5148 /* let the f/w know that the h/w is now under the control of the 5149 * driver. 5150 */ 5151 igc_get_hw_control(adapter); 5152 5153 strncpy(netdev->name, "eth%d", IFNAMSIZ); 5154 err = register_netdev(netdev); 5155 if (err) 5156 goto err_register; 5157 5158 /* carrier off reporting is important to ethtool even BEFORE open */ 5159 netif_carrier_off(netdev); 5160 5161 /* do hw tstamp init after resetting */ 5162 igc_ptp_init(adapter); 5163 5164 /* Check if Media Autosense is enabled */ 5165 adapter->ei = *ei; 5166 5167 /* print pcie link status and MAC address */ 5168 pcie_print_link_status(pdev); 5169 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 5170 5171 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); 5172 /* Disable EEE for internal PHY devices */ 5173 hw->dev_spec._base.eee_enable = false; 5174 adapter->flags &= ~IGC_FLAG_EEE; 5175 igc_set_eee_i225(hw, false, false, false); 5176 5177 pm_runtime_put_noidle(&pdev->dev); 5178 5179 return 0; 5180 5181 err_register: 5182 igc_release_hw_control(adapter); 5183 err_eeprom: 5184 if (!igc_check_reset_block(hw)) 5185 igc_reset_phy(hw); 5186 err_sw_init: 5187 igc_clear_interrupt_scheme(adapter); 5188 iounmap(adapter->io_addr); 5189 err_ioremap: 5190 free_netdev(netdev); 5191 err_alloc_etherdev: 5192 pci_release_mem_regions(pdev); 5193 err_pci_reg: 5194 err_dma: 5195 pci_disable_device(pdev); 5196 return err; 5197 } 5198 5199 /** 5200 * igc_remove - Device Removal Routine 5201 * @pdev: PCI device information struct 5202 * 5203 * igc_remove is called by the PCI subsystem to alert the driver 5204 * that it should release a PCI device. This could be caused by a 5205 * Hot-Plug event, or because the driver is going to be removed from 5206 * memory. 5207 */ 5208 static void igc_remove(struct pci_dev *pdev) 5209 { 5210 struct net_device *netdev = pci_get_drvdata(pdev); 5211 struct igc_adapter *adapter = netdev_priv(netdev); 5212 5213 pm_runtime_get_noresume(&pdev->dev); 5214 5215 igc_flush_nfc_rules(adapter); 5216 5217 igc_ptp_stop(adapter); 5218 5219 set_bit(__IGC_DOWN, &adapter->state); 5220 5221 del_timer_sync(&adapter->watchdog_timer); 5222 del_timer_sync(&adapter->phy_info_timer); 5223 5224 cancel_work_sync(&adapter->reset_task); 5225 cancel_work_sync(&adapter->watchdog_task); 5226 5227 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5228 * would have already happened in close and is redundant. 5229 */ 5230 igc_release_hw_control(adapter); 5231 unregister_netdev(netdev); 5232 5233 igc_clear_interrupt_scheme(adapter); 5234 pci_iounmap(pdev, adapter->io_addr); 5235 pci_release_mem_regions(pdev); 5236 5237 free_netdev(netdev); 5238 5239 pci_disable_pcie_error_reporting(pdev); 5240 5241 pci_disable_device(pdev); 5242 } 5243 5244 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, 5245 bool runtime) 5246 { 5247 struct net_device *netdev = pci_get_drvdata(pdev); 5248 struct igc_adapter *adapter = netdev_priv(netdev); 5249 u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol; 5250 struct igc_hw *hw = &adapter->hw; 5251 u32 ctrl, rctl, status; 5252 bool wake; 5253 5254 rtnl_lock(); 5255 netif_device_detach(netdev); 5256 5257 if (netif_running(netdev)) 5258 __igc_close(netdev, true); 5259 5260 igc_ptp_suspend(adapter); 5261 5262 igc_clear_interrupt_scheme(adapter); 5263 rtnl_unlock(); 5264 5265 status = rd32(IGC_STATUS); 5266 if (status & IGC_STATUS_LU) 5267 wufc &= ~IGC_WUFC_LNKC; 5268 5269 if (wufc) { 5270 igc_setup_rctl(adapter); 5271 igc_set_rx_mode(netdev); 5272 5273 /* turn on all-multi mode if wake on multicast is enabled */ 5274 if (wufc & IGC_WUFC_MC) { 5275 rctl = rd32(IGC_RCTL); 5276 rctl |= IGC_RCTL_MPE; 5277 wr32(IGC_RCTL, rctl); 5278 } 5279 5280 ctrl = rd32(IGC_CTRL); 5281 ctrl |= IGC_CTRL_ADVD3WUC; 5282 wr32(IGC_CTRL, ctrl); 5283 5284 /* Allow time for pending master requests to run */ 5285 igc_disable_pcie_master(hw); 5286 5287 wr32(IGC_WUC, IGC_WUC_PME_EN); 5288 wr32(IGC_WUFC, wufc); 5289 } else { 5290 wr32(IGC_WUC, 0); 5291 wr32(IGC_WUFC, 0); 5292 } 5293 5294 wake = wufc || adapter->en_mng_pt; 5295 if (!wake) 5296 igc_power_down_phy_copper_base(&adapter->hw); 5297 else 5298 igc_power_up_link(adapter); 5299 5300 if (enable_wake) 5301 *enable_wake = wake; 5302 5303 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5304 * would have already happened in close and is redundant. 5305 */ 5306 igc_release_hw_control(adapter); 5307 5308 pci_disable_device(pdev); 5309 5310 return 0; 5311 } 5312 5313 #ifdef CONFIG_PM 5314 static int __maybe_unused igc_runtime_suspend(struct device *dev) 5315 { 5316 return __igc_shutdown(to_pci_dev(dev), NULL, 1); 5317 } 5318 5319 static void igc_deliver_wake_packet(struct net_device *netdev) 5320 { 5321 struct igc_adapter *adapter = netdev_priv(netdev); 5322 struct igc_hw *hw = &adapter->hw; 5323 struct sk_buff *skb; 5324 u32 wupl; 5325 5326 wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK; 5327 5328 /* WUPM stores only the first 128 bytes of the wake packet. 5329 * Read the packet only if we have the whole thing. 5330 */ 5331 if (wupl == 0 || wupl > IGC_WUPM_BYTES) 5332 return; 5333 5334 skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES); 5335 if (!skb) 5336 return; 5337 5338 skb_put(skb, wupl); 5339 5340 /* Ensure reads are 32-bit aligned */ 5341 wupl = roundup(wupl, 4); 5342 5343 memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl); 5344 5345 skb->protocol = eth_type_trans(skb, netdev); 5346 netif_rx(skb); 5347 } 5348 5349 static int __maybe_unused igc_resume(struct device *dev) 5350 { 5351 struct pci_dev *pdev = to_pci_dev(dev); 5352 struct net_device *netdev = pci_get_drvdata(pdev); 5353 struct igc_adapter *adapter = netdev_priv(netdev); 5354 struct igc_hw *hw = &adapter->hw; 5355 u32 err, val; 5356 5357 pci_set_power_state(pdev, PCI_D0); 5358 pci_restore_state(pdev); 5359 pci_save_state(pdev); 5360 5361 if (!pci_device_is_present(pdev)) 5362 return -ENODEV; 5363 err = pci_enable_device_mem(pdev); 5364 if (err) { 5365 netdev_err(netdev, "Cannot enable PCI device from suspend\n"); 5366 return err; 5367 } 5368 pci_set_master(pdev); 5369 5370 pci_enable_wake(pdev, PCI_D3hot, 0); 5371 pci_enable_wake(pdev, PCI_D3cold, 0); 5372 5373 if (igc_init_interrupt_scheme(adapter, true)) { 5374 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5375 return -ENOMEM; 5376 } 5377 5378 igc_reset(adapter); 5379 5380 /* let the f/w know that the h/w is now under the control of the 5381 * driver. 5382 */ 5383 igc_get_hw_control(adapter); 5384 5385 val = rd32(IGC_WUS); 5386 if (val & WAKE_PKT_WUS) 5387 igc_deliver_wake_packet(netdev); 5388 5389 wr32(IGC_WUS, ~0); 5390 5391 rtnl_lock(); 5392 if (!err && netif_running(netdev)) 5393 err = __igc_open(netdev, true); 5394 5395 if (!err) 5396 netif_device_attach(netdev); 5397 rtnl_unlock(); 5398 5399 return err; 5400 } 5401 5402 static int __maybe_unused igc_runtime_resume(struct device *dev) 5403 { 5404 return igc_resume(dev); 5405 } 5406 5407 static int __maybe_unused igc_suspend(struct device *dev) 5408 { 5409 return __igc_shutdown(to_pci_dev(dev), NULL, 0); 5410 } 5411 5412 static int __maybe_unused igc_runtime_idle(struct device *dev) 5413 { 5414 struct net_device *netdev = dev_get_drvdata(dev); 5415 struct igc_adapter *adapter = netdev_priv(netdev); 5416 5417 if (!igc_has_link(adapter)) 5418 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 5419 5420 return -EBUSY; 5421 } 5422 #endif /* CONFIG_PM */ 5423 5424 static void igc_shutdown(struct pci_dev *pdev) 5425 { 5426 bool wake; 5427 5428 __igc_shutdown(pdev, &wake, 0); 5429 5430 if (system_state == SYSTEM_POWER_OFF) { 5431 pci_wake_from_d3(pdev, wake); 5432 pci_set_power_state(pdev, PCI_D3hot); 5433 } 5434 } 5435 5436 /** 5437 * igc_io_error_detected - called when PCI error is detected 5438 * @pdev: Pointer to PCI device 5439 * @state: The current PCI connection state 5440 * 5441 * This function is called after a PCI bus error affecting 5442 * this device has been detected. 5443 **/ 5444 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, 5445 pci_channel_state_t state) 5446 { 5447 struct net_device *netdev = pci_get_drvdata(pdev); 5448 struct igc_adapter *adapter = netdev_priv(netdev); 5449 5450 netif_device_detach(netdev); 5451 5452 if (state == pci_channel_io_perm_failure) 5453 return PCI_ERS_RESULT_DISCONNECT; 5454 5455 if (netif_running(netdev)) 5456 igc_down(adapter); 5457 pci_disable_device(pdev); 5458 5459 /* Request a slot reset. */ 5460 return PCI_ERS_RESULT_NEED_RESET; 5461 } 5462 5463 /** 5464 * igc_io_slot_reset - called after the PCI bus has been reset. 5465 * @pdev: Pointer to PCI device 5466 * 5467 * Restart the card from scratch, as if from a cold-boot. Implementation 5468 * resembles the first-half of the igc_resume routine. 5469 **/ 5470 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) 5471 { 5472 struct net_device *netdev = pci_get_drvdata(pdev); 5473 struct igc_adapter *adapter = netdev_priv(netdev); 5474 struct igc_hw *hw = &adapter->hw; 5475 pci_ers_result_t result; 5476 5477 if (pci_enable_device_mem(pdev)) { 5478 netdev_err(netdev, "Could not re-enable PCI device after reset\n"); 5479 result = PCI_ERS_RESULT_DISCONNECT; 5480 } else { 5481 pci_set_master(pdev); 5482 pci_restore_state(pdev); 5483 pci_save_state(pdev); 5484 5485 pci_enable_wake(pdev, PCI_D3hot, 0); 5486 pci_enable_wake(pdev, PCI_D3cold, 0); 5487 5488 /* In case of PCI error, adapter loses its HW address 5489 * so we should re-assign it here. 5490 */ 5491 hw->hw_addr = adapter->io_addr; 5492 5493 igc_reset(adapter); 5494 wr32(IGC_WUS, ~0); 5495 result = PCI_ERS_RESULT_RECOVERED; 5496 } 5497 5498 return result; 5499 } 5500 5501 /** 5502 * igc_io_resume - called when traffic can start to flow again. 5503 * @pdev: Pointer to PCI device 5504 * 5505 * This callback is called when the error recovery driver tells us that 5506 * its OK to resume normal operation. Implementation resembles the 5507 * second-half of the igc_resume routine. 5508 */ 5509 static void igc_io_resume(struct pci_dev *pdev) 5510 { 5511 struct net_device *netdev = pci_get_drvdata(pdev); 5512 struct igc_adapter *adapter = netdev_priv(netdev); 5513 5514 rtnl_lock(); 5515 if (netif_running(netdev)) { 5516 if (igc_open(netdev)) { 5517 netdev_err(netdev, "igc_open failed after reset\n"); 5518 return; 5519 } 5520 } 5521 5522 netif_device_attach(netdev); 5523 5524 /* let the f/w know that the h/w is now under the control of the 5525 * driver. 5526 */ 5527 igc_get_hw_control(adapter); 5528 rtnl_unlock(); 5529 } 5530 5531 static const struct pci_error_handlers igc_err_handler = { 5532 .error_detected = igc_io_error_detected, 5533 .slot_reset = igc_io_slot_reset, 5534 .resume = igc_io_resume, 5535 }; 5536 5537 #ifdef CONFIG_PM 5538 static const struct dev_pm_ops igc_pm_ops = { 5539 SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume) 5540 SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume, 5541 igc_runtime_idle) 5542 }; 5543 #endif 5544 5545 static struct pci_driver igc_driver = { 5546 .name = igc_driver_name, 5547 .id_table = igc_pci_tbl, 5548 .probe = igc_probe, 5549 .remove = igc_remove, 5550 #ifdef CONFIG_PM 5551 .driver.pm = &igc_pm_ops, 5552 #endif 5553 .shutdown = igc_shutdown, 5554 .err_handler = &igc_err_handler, 5555 }; 5556 5557 /** 5558 * igc_reinit_queues - return error 5559 * @adapter: pointer to adapter structure 5560 */ 5561 int igc_reinit_queues(struct igc_adapter *adapter) 5562 { 5563 struct net_device *netdev = adapter->netdev; 5564 int err = 0; 5565 5566 if (netif_running(netdev)) 5567 igc_close(netdev); 5568 5569 igc_reset_interrupt_capability(adapter); 5570 5571 if (igc_init_interrupt_scheme(adapter, true)) { 5572 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5573 return -ENOMEM; 5574 } 5575 5576 if (netif_running(netdev)) 5577 err = igc_open(netdev); 5578 5579 return err; 5580 } 5581 5582 /** 5583 * igc_get_hw_dev - return device 5584 * @hw: pointer to hardware structure 5585 * 5586 * used by hardware layer to print debugging information 5587 */ 5588 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 5589 { 5590 struct igc_adapter *adapter = hw->back; 5591 5592 return adapter->netdev; 5593 } 5594 5595 /** 5596 * igc_init_module - Driver Registration Routine 5597 * 5598 * igc_init_module is the first routine called when the driver is 5599 * loaded. All it does is register with the PCI subsystem. 5600 */ 5601 static int __init igc_init_module(void) 5602 { 5603 int ret; 5604 5605 pr_info("%s\n", igc_driver_string); 5606 pr_info("%s\n", igc_copyright); 5607 5608 ret = pci_register_driver(&igc_driver); 5609 return ret; 5610 } 5611 5612 module_init(igc_init_module); 5613 5614 /** 5615 * igc_exit_module - Driver Exit Cleanup Routine 5616 * 5617 * igc_exit_module is called just before the driver is removed 5618 * from memory. 5619 */ 5620 static void __exit igc_exit_module(void) 5621 { 5622 pci_unregister_driver(&igc_driver); 5623 } 5624 5625 module_exit(igc_exit_module); 5626 /* igc_main.c */ 5627