1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/aer.h> 8 #include <linux/tcp.h> 9 #include <linux/udp.h> 10 #include <linux/ip.h> 11 #include <linux/pm_runtime.h> 12 #include <net/pkt_sched.h> 13 14 #include <net/ipv6.h> 15 16 #include "igc.h" 17 #include "igc_hw.h" 18 #include "igc_tsn.h" 19 20 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 21 22 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 23 24 static int debug = -1; 25 26 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 27 MODULE_DESCRIPTION(DRV_SUMMARY); 28 MODULE_LICENSE("GPL v2"); 29 module_param(debug, int, 0); 30 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 31 32 char igc_driver_name[] = "igc"; 33 static const char igc_driver_string[] = DRV_SUMMARY; 34 static const char igc_copyright[] = 35 "Copyright(c) 2018 Intel Corporation."; 36 37 static const struct igc_info *igc_info_tbl[] = { 38 [board_base] = &igc_base_info, 39 }; 40 41 static const struct pci_device_id igc_pci_tbl[] = { 42 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base }, 43 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base }, 44 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, 45 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, 46 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, 47 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base }, 48 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base }, 49 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base }, 50 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, 51 /* required last entry */ 52 {0, } 53 }; 54 55 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 56 57 enum latency_range { 58 lowest_latency = 0, 59 low_latency = 1, 60 bulk_latency = 2, 61 latency_invalid = 255 62 }; 63 64 void igc_reset(struct igc_adapter *adapter) 65 { 66 struct net_device *dev = adapter->netdev; 67 struct igc_hw *hw = &adapter->hw; 68 struct igc_fc_info *fc = &hw->fc; 69 u32 pba, hwm; 70 71 /* Repartition PBA for greater than 9k MTU if required */ 72 pba = IGC_PBA_34K; 73 74 /* flow control settings 75 * The high water mark must be low enough to fit one full frame 76 * after transmitting the pause frame. As such we must have enough 77 * space to allow for us to complete our current transmit and then 78 * receive the frame that is in progress from the link partner. 79 * Set it to: 80 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 81 */ 82 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 83 84 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 85 fc->low_water = fc->high_water - 16; 86 fc->pause_time = 0xFFFF; 87 fc->send_xon = 1; 88 fc->current_mode = fc->requested_mode; 89 90 hw->mac.ops.reset_hw(hw); 91 92 if (hw->mac.ops.init_hw(hw)) 93 netdev_err(dev, "Error on hardware initialization\n"); 94 95 /* Re-establish EEE setting */ 96 igc_set_eee_i225(hw, true, true, true); 97 98 if (!netif_running(adapter->netdev)) 99 igc_power_down_phy_copper_base(&adapter->hw); 100 101 /* Re-enable PTP, where applicable. */ 102 igc_ptp_reset(adapter); 103 104 /* Re-enable TSN offloading, where applicable. */ 105 igc_tsn_offload_apply(adapter); 106 107 igc_get_phy_info(hw); 108 } 109 110 /** 111 * igc_power_up_link - Power up the phy link 112 * @adapter: address of board private structure 113 */ 114 static void igc_power_up_link(struct igc_adapter *adapter) 115 { 116 igc_reset_phy(&adapter->hw); 117 118 igc_power_up_phy_copper(&adapter->hw); 119 120 igc_setup_link(&adapter->hw); 121 } 122 123 /** 124 * igc_release_hw_control - release control of the h/w to f/w 125 * @adapter: address of board private structure 126 * 127 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 128 * For ASF and Pass Through versions of f/w this means that the 129 * driver is no longer loaded. 130 */ 131 static void igc_release_hw_control(struct igc_adapter *adapter) 132 { 133 struct igc_hw *hw = &adapter->hw; 134 u32 ctrl_ext; 135 136 /* Let firmware take over control of h/w */ 137 ctrl_ext = rd32(IGC_CTRL_EXT); 138 wr32(IGC_CTRL_EXT, 139 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 140 } 141 142 /** 143 * igc_get_hw_control - get control of the h/w from f/w 144 * @adapter: address of board private structure 145 * 146 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 147 * For ASF and Pass Through versions of f/w this means that 148 * the driver is loaded. 149 */ 150 static void igc_get_hw_control(struct igc_adapter *adapter) 151 { 152 struct igc_hw *hw = &adapter->hw; 153 u32 ctrl_ext; 154 155 /* Let firmware know the driver has taken over */ 156 ctrl_ext = rd32(IGC_CTRL_EXT); 157 wr32(IGC_CTRL_EXT, 158 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 159 } 160 161 /** 162 * igc_clean_tx_ring - Free Tx Buffers 163 * @tx_ring: ring to be cleaned 164 */ 165 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 166 { 167 u16 i = tx_ring->next_to_clean; 168 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 169 170 while (i != tx_ring->next_to_use) { 171 union igc_adv_tx_desc *eop_desc, *tx_desc; 172 173 /* Free all the Tx ring sk_buffs */ 174 dev_kfree_skb_any(tx_buffer->skb); 175 176 /* unmap skb header data */ 177 dma_unmap_single(tx_ring->dev, 178 dma_unmap_addr(tx_buffer, dma), 179 dma_unmap_len(tx_buffer, len), 180 DMA_TO_DEVICE); 181 182 /* check for eop_desc to determine the end of the packet */ 183 eop_desc = tx_buffer->next_to_watch; 184 tx_desc = IGC_TX_DESC(tx_ring, i); 185 186 /* unmap remaining buffers */ 187 while (tx_desc != eop_desc) { 188 tx_buffer++; 189 tx_desc++; 190 i++; 191 if (unlikely(i == tx_ring->count)) { 192 i = 0; 193 tx_buffer = tx_ring->tx_buffer_info; 194 tx_desc = IGC_TX_DESC(tx_ring, 0); 195 } 196 197 /* unmap any remaining paged data */ 198 if (dma_unmap_len(tx_buffer, len)) 199 dma_unmap_page(tx_ring->dev, 200 dma_unmap_addr(tx_buffer, dma), 201 dma_unmap_len(tx_buffer, len), 202 DMA_TO_DEVICE); 203 } 204 205 /* move us one more past the eop_desc for start of next pkt */ 206 tx_buffer++; 207 i++; 208 if (unlikely(i == tx_ring->count)) { 209 i = 0; 210 tx_buffer = tx_ring->tx_buffer_info; 211 } 212 } 213 214 /* reset BQL for queue */ 215 netdev_tx_reset_queue(txring_txq(tx_ring)); 216 217 /* reset next_to_use and next_to_clean */ 218 tx_ring->next_to_use = 0; 219 tx_ring->next_to_clean = 0; 220 } 221 222 /** 223 * igc_free_tx_resources - Free Tx Resources per Queue 224 * @tx_ring: Tx descriptor ring for a specific queue 225 * 226 * Free all transmit software resources 227 */ 228 void igc_free_tx_resources(struct igc_ring *tx_ring) 229 { 230 igc_clean_tx_ring(tx_ring); 231 232 vfree(tx_ring->tx_buffer_info); 233 tx_ring->tx_buffer_info = NULL; 234 235 /* if not set, then don't free */ 236 if (!tx_ring->desc) 237 return; 238 239 dma_free_coherent(tx_ring->dev, tx_ring->size, 240 tx_ring->desc, tx_ring->dma); 241 242 tx_ring->desc = NULL; 243 } 244 245 /** 246 * igc_free_all_tx_resources - Free Tx Resources for All Queues 247 * @adapter: board private structure 248 * 249 * Free all transmit software resources 250 */ 251 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 252 { 253 int i; 254 255 for (i = 0; i < adapter->num_tx_queues; i++) 256 igc_free_tx_resources(adapter->tx_ring[i]); 257 } 258 259 /** 260 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 261 * @adapter: board private structure 262 */ 263 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 264 { 265 int i; 266 267 for (i = 0; i < adapter->num_tx_queues; i++) 268 if (adapter->tx_ring[i]) 269 igc_clean_tx_ring(adapter->tx_ring[i]); 270 } 271 272 /** 273 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 274 * @tx_ring: tx descriptor ring (for a specific queue) to setup 275 * 276 * Return 0 on success, negative on failure 277 */ 278 int igc_setup_tx_resources(struct igc_ring *tx_ring) 279 { 280 struct net_device *ndev = tx_ring->netdev; 281 struct device *dev = tx_ring->dev; 282 int size = 0; 283 284 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 285 tx_ring->tx_buffer_info = vzalloc(size); 286 if (!tx_ring->tx_buffer_info) 287 goto err; 288 289 /* round up to nearest 4K */ 290 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 291 tx_ring->size = ALIGN(tx_ring->size, 4096); 292 293 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 294 &tx_ring->dma, GFP_KERNEL); 295 296 if (!tx_ring->desc) 297 goto err; 298 299 tx_ring->next_to_use = 0; 300 tx_ring->next_to_clean = 0; 301 302 return 0; 303 304 err: 305 vfree(tx_ring->tx_buffer_info); 306 netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n"); 307 return -ENOMEM; 308 } 309 310 /** 311 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 312 * @adapter: board private structure 313 * 314 * Return 0 on success, negative on failure 315 */ 316 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 317 { 318 struct net_device *dev = adapter->netdev; 319 int i, err = 0; 320 321 for (i = 0; i < adapter->num_tx_queues; i++) { 322 err = igc_setup_tx_resources(adapter->tx_ring[i]); 323 if (err) { 324 netdev_err(dev, "Error on Tx queue %u setup\n", i); 325 for (i--; i >= 0; i--) 326 igc_free_tx_resources(adapter->tx_ring[i]); 327 break; 328 } 329 } 330 331 return err; 332 } 333 334 /** 335 * igc_clean_rx_ring - Free Rx Buffers per Queue 336 * @rx_ring: ring to free buffers from 337 */ 338 static void igc_clean_rx_ring(struct igc_ring *rx_ring) 339 { 340 u16 i = rx_ring->next_to_clean; 341 342 dev_kfree_skb(rx_ring->skb); 343 rx_ring->skb = NULL; 344 345 /* Free all the Rx ring sk_buffs */ 346 while (i != rx_ring->next_to_alloc) { 347 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 348 349 /* Invalidate cache lines that may have been written to by 350 * device so that we avoid corrupting memory. 351 */ 352 dma_sync_single_range_for_cpu(rx_ring->dev, 353 buffer_info->dma, 354 buffer_info->page_offset, 355 igc_rx_bufsz(rx_ring), 356 DMA_FROM_DEVICE); 357 358 /* free resources associated with mapping */ 359 dma_unmap_page_attrs(rx_ring->dev, 360 buffer_info->dma, 361 igc_rx_pg_size(rx_ring), 362 DMA_FROM_DEVICE, 363 IGC_RX_DMA_ATTR); 364 __page_frag_cache_drain(buffer_info->page, 365 buffer_info->pagecnt_bias); 366 367 i++; 368 if (i == rx_ring->count) 369 i = 0; 370 } 371 372 rx_ring->next_to_alloc = 0; 373 rx_ring->next_to_clean = 0; 374 rx_ring->next_to_use = 0; 375 } 376 377 /** 378 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 379 * @adapter: board private structure 380 */ 381 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 382 { 383 int i; 384 385 for (i = 0; i < adapter->num_rx_queues; i++) 386 if (adapter->rx_ring[i]) 387 igc_clean_rx_ring(adapter->rx_ring[i]); 388 } 389 390 /** 391 * igc_free_rx_resources - Free Rx Resources 392 * @rx_ring: ring to clean the resources from 393 * 394 * Free all receive software resources 395 */ 396 void igc_free_rx_resources(struct igc_ring *rx_ring) 397 { 398 igc_clean_rx_ring(rx_ring); 399 400 vfree(rx_ring->rx_buffer_info); 401 rx_ring->rx_buffer_info = NULL; 402 403 /* if not set, then don't free */ 404 if (!rx_ring->desc) 405 return; 406 407 dma_free_coherent(rx_ring->dev, rx_ring->size, 408 rx_ring->desc, rx_ring->dma); 409 410 rx_ring->desc = NULL; 411 } 412 413 /** 414 * igc_free_all_rx_resources - Free Rx Resources for All Queues 415 * @adapter: board private structure 416 * 417 * Free all receive software resources 418 */ 419 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 420 { 421 int i; 422 423 for (i = 0; i < adapter->num_rx_queues; i++) 424 igc_free_rx_resources(adapter->rx_ring[i]); 425 } 426 427 /** 428 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 429 * @rx_ring: rx descriptor ring (for a specific queue) to setup 430 * 431 * Returns 0 on success, negative on failure 432 */ 433 int igc_setup_rx_resources(struct igc_ring *rx_ring) 434 { 435 struct net_device *ndev = rx_ring->netdev; 436 struct device *dev = rx_ring->dev; 437 int size, desc_len; 438 439 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 440 rx_ring->rx_buffer_info = vzalloc(size); 441 if (!rx_ring->rx_buffer_info) 442 goto err; 443 444 desc_len = sizeof(union igc_adv_rx_desc); 445 446 /* Round up to nearest 4K */ 447 rx_ring->size = rx_ring->count * desc_len; 448 rx_ring->size = ALIGN(rx_ring->size, 4096); 449 450 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 451 &rx_ring->dma, GFP_KERNEL); 452 453 if (!rx_ring->desc) 454 goto err; 455 456 rx_ring->next_to_alloc = 0; 457 rx_ring->next_to_clean = 0; 458 rx_ring->next_to_use = 0; 459 460 return 0; 461 462 err: 463 vfree(rx_ring->rx_buffer_info); 464 rx_ring->rx_buffer_info = NULL; 465 netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n"); 466 return -ENOMEM; 467 } 468 469 /** 470 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 471 * (Descriptors) for all queues 472 * @adapter: board private structure 473 * 474 * Return 0 on success, negative on failure 475 */ 476 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 477 { 478 struct net_device *dev = adapter->netdev; 479 int i, err = 0; 480 481 for (i = 0; i < adapter->num_rx_queues; i++) { 482 err = igc_setup_rx_resources(adapter->rx_ring[i]); 483 if (err) { 484 netdev_err(dev, "Error on Rx queue %u setup\n", i); 485 for (i--; i >= 0; i--) 486 igc_free_rx_resources(adapter->rx_ring[i]); 487 break; 488 } 489 } 490 491 return err; 492 } 493 494 /** 495 * igc_configure_rx_ring - Configure a receive ring after Reset 496 * @adapter: board private structure 497 * @ring: receive ring to be configured 498 * 499 * Configure the Rx unit of the MAC after a reset. 500 */ 501 static void igc_configure_rx_ring(struct igc_adapter *adapter, 502 struct igc_ring *ring) 503 { 504 struct igc_hw *hw = &adapter->hw; 505 union igc_adv_rx_desc *rx_desc; 506 int reg_idx = ring->reg_idx; 507 u32 srrctl = 0, rxdctl = 0; 508 u64 rdba = ring->dma; 509 510 /* disable the queue */ 511 wr32(IGC_RXDCTL(reg_idx), 0); 512 513 /* Set DMA base address registers */ 514 wr32(IGC_RDBAL(reg_idx), 515 rdba & 0x00000000ffffffffULL); 516 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 517 wr32(IGC_RDLEN(reg_idx), 518 ring->count * sizeof(union igc_adv_rx_desc)); 519 520 /* initialize head and tail */ 521 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 522 wr32(IGC_RDH(reg_idx), 0); 523 writel(0, ring->tail); 524 525 /* reset next-to- use/clean to place SW in sync with hardware */ 526 ring->next_to_clean = 0; 527 ring->next_to_use = 0; 528 529 /* set descriptor configuration */ 530 srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; 531 if (ring_uses_large_buffer(ring)) 532 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 533 else 534 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 535 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 536 537 wr32(IGC_SRRCTL(reg_idx), srrctl); 538 539 rxdctl |= IGC_RX_PTHRESH; 540 rxdctl |= IGC_RX_HTHRESH << 8; 541 rxdctl |= IGC_RX_WTHRESH << 16; 542 543 /* initialize rx_buffer_info */ 544 memset(ring->rx_buffer_info, 0, 545 sizeof(struct igc_rx_buffer) * ring->count); 546 547 /* initialize Rx descriptor 0 */ 548 rx_desc = IGC_RX_DESC(ring, 0); 549 rx_desc->wb.upper.length = 0; 550 551 /* enable receive descriptor fetching */ 552 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 553 554 wr32(IGC_RXDCTL(reg_idx), rxdctl); 555 } 556 557 /** 558 * igc_configure_rx - Configure receive Unit after Reset 559 * @adapter: board private structure 560 * 561 * Configure the Rx unit of the MAC after a reset. 562 */ 563 static void igc_configure_rx(struct igc_adapter *adapter) 564 { 565 int i; 566 567 /* Setup the HW Rx Head and Tail Descriptor Pointers and 568 * the Base and Length of the Rx Descriptor Ring 569 */ 570 for (i = 0; i < adapter->num_rx_queues; i++) 571 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 572 } 573 574 /** 575 * igc_configure_tx_ring - Configure transmit ring after Reset 576 * @adapter: board private structure 577 * @ring: tx ring to configure 578 * 579 * Configure a transmit ring after a reset. 580 */ 581 static void igc_configure_tx_ring(struct igc_adapter *adapter, 582 struct igc_ring *ring) 583 { 584 struct igc_hw *hw = &adapter->hw; 585 int reg_idx = ring->reg_idx; 586 u64 tdba = ring->dma; 587 u32 txdctl = 0; 588 589 /* disable the queue */ 590 wr32(IGC_TXDCTL(reg_idx), 0); 591 wrfl(); 592 mdelay(10); 593 594 wr32(IGC_TDLEN(reg_idx), 595 ring->count * sizeof(union igc_adv_tx_desc)); 596 wr32(IGC_TDBAL(reg_idx), 597 tdba & 0x00000000ffffffffULL); 598 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 599 600 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 601 wr32(IGC_TDH(reg_idx), 0); 602 writel(0, ring->tail); 603 604 txdctl |= IGC_TX_PTHRESH; 605 txdctl |= IGC_TX_HTHRESH << 8; 606 txdctl |= IGC_TX_WTHRESH << 16; 607 608 txdctl |= IGC_TXDCTL_QUEUE_ENABLE; 609 wr32(IGC_TXDCTL(reg_idx), txdctl); 610 } 611 612 /** 613 * igc_configure_tx - Configure transmit Unit after Reset 614 * @adapter: board private structure 615 * 616 * Configure the Tx unit of the MAC after a reset. 617 */ 618 static void igc_configure_tx(struct igc_adapter *adapter) 619 { 620 int i; 621 622 for (i = 0; i < adapter->num_tx_queues; i++) 623 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 624 } 625 626 /** 627 * igc_setup_mrqc - configure the multiple receive queue control registers 628 * @adapter: Board private structure 629 */ 630 static void igc_setup_mrqc(struct igc_adapter *adapter) 631 { 632 struct igc_hw *hw = &adapter->hw; 633 u32 j, num_rx_queues; 634 u32 mrqc, rxcsum; 635 u32 rss_key[10]; 636 637 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 638 for (j = 0; j < 10; j++) 639 wr32(IGC_RSSRK(j), rss_key[j]); 640 641 num_rx_queues = adapter->rss_queues; 642 643 if (adapter->rss_indir_tbl_init != num_rx_queues) { 644 for (j = 0; j < IGC_RETA_SIZE; j++) 645 adapter->rss_indir_tbl[j] = 646 (j * num_rx_queues) / IGC_RETA_SIZE; 647 adapter->rss_indir_tbl_init = num_rx_queues; 648 } 649 igc_write_rss_indir_tbl(adapter); 650 651 /* Disable raw packet checksumming so that RSS hash is placed in 652 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 653 * offloads as they are enabled by default 654 */ 655 rxcsum = rd32(IGC_RXCSUM); 656 rxcsum |= IGC_RXCSUM_PCSD; 657 658 /* Enable Receive Checksum Offload for SCTP */ 659 rxcsum |= IGC_RXCSUM_CRCOFL; 660 661 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 662 wr32(IGC_RXCSUM, rxcsum); 663 664 /* Generate RSS hash based on packet types, TCP/UDP 665 * port numbers and/or IPv4/v6 src and dst addresses 666 */ 667 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 668 IGC_MRQC_RSS_FIELD_IPV4_TCP | 669 IGC_MRQC_RSS_FIELD_IPV6 | 670 IGC_MRQC_RSS_FIELD_IPV6_TCP | 671 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 672 673 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 674 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 675 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 676 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 677 678 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 679 680 wr32(IGC_MRQC, mrqc); 681 } 682 683 /** 684 * igc_setup_rctl - configure the receive control registers 685 * @adapter: Board private structure 686 */ 687 static void igc_setup_rctl(struct igc_adapter *adapter) 688 { 689 struct igc_hw *hw = &adapter->hw; 690 u32 rctl; 691 692 rctl = rd32(IGC_RCTL); 693 694 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 695 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 696 697 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 698 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 699 700 /* enable stripping of CRC. Newer features require 701 * that the HW strips the CRC. 702 */ 703 rctl |= IGC_RCTL_SECRC; 704 705 /* disable store bad packets and clear size bits. */ 706 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 707 708 /* enable LPE to allow for reception of jumbo frames */ 709 rctl |= IGC_RCTL_LPE; 710 711 /* disable queue 0 to prevent tail write w/o re-config */ 712 wr32(IGC_RXDCTL(0), 0); 713 714 /* This is useful for sniffing bad packets. */ 715 if (adapter->netdev->features & NETIF_F_RXALL) { 716 /* UPE and MPE will be handled by normal PROMISC logic 717 * in set_rx_mode 718 */ 719 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 720 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 721 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 722 723 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 724 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 725 } 726 727 wr32(IGC_RCTL, rctl); 728 } 729 730 /** 731 * igc_setup_tctl - configure the transmit control registers 732 * @adapter: Board private structure 733 */ 734 static void igc_setup_tctl(struct igc_adapter *adapter) 735 { 736 struct igc_hw *hw = &adapter->hw; 737 u32 tctl; 738 739 /* disable queue 0 which icould be enabled by default */ 740 wr32(IGC_TXDCTL(0), 0); 741 742 /* Program the Transmit Control Register */ 743 tctl = rd32(IGC_TCTL); 744 tctl &= ~IGC_TCTL_CT; 745 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 746 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 747 748 /* Enable transmits */ 749 tctl |= IGC_TCTL_EN; 750 751 wr32(IGC_TCTL, tctl); 752 } 753 754 /** 755 * igc_set_mac_filter_hw() - Set MAC address filter in hardware 756 * @adapter: Pointer to adapter where the filter should be set 757 * @index: Filter index 758 * @type: MAC address filter type (source or destination) 759 * @addr: MAC address 760 * @queue: If non-negative, queue assignment feature is enabled and frames 761 * matching the filter are enqueued onto 'queue'. Otherwise, queue 762 * assignment is disabled. 763 */ 764 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, 765 enum igc_mac_filter_type type, 766 const u8 *addr, int queue) 767 { 768 struct net_device *dev = adapter->netdev; 769 struct igc_hw *hw = &adapter->hw; 770 u32 ral, rah; 771 772 if (WARN_ON(index >= hw->mac.rar_entry_count)) 773 return; 774 775 ral = le32_to_cpup((__le32 *)(addr)); 776 rah = le16_to_cpup((__le16 *)(addr + 4)); 777 778 if (type == IGC_MAC_FILTER_TYPE_SRC) { 779 rah &= ~IGC_RAH_ASEL_MASK; 780 rah |= IGC_RAH_ASEL_SRC_ADDR; 781 } 782 783 if (queue >= 0) { 784 rah &= ~IGC_RAH_QSEL_MASK; 785 rah |= (queue << IGC_RAH_QSEL_SHIFT); 786 rah |= IGC_RAH_QSEL_ENABLE; 787 } 788 789 rah |= IGC_RAH_AV; 790 791 wr32(IGC_RAL(index), ral); 792 wr32(IGC_RAH(index), rah); 793 794 netdev_dbg(dev, "MAC address filter set in HW: index %d", index); 795 } 796 797 /** 798 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware 799 * @adapter: Pointer to adapter where the filter should be cleared 800 * @index: Filter index 801 */ 802 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) 803 { 804 struct net_device *dev = adapter->netdev; 805 struct igc_hw *hw = &adapter->hw; 806 807 if (WARN_ON(index >= hw->mac.rar_entry_count)) 808 return; 809 810 wr32(IGC_RAL(index), 0); 811 wr32(IGC_RAH(index), 0); 812 813 netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index); 814 } 815 816 /* Set default MAC address for the PF in the first RAR entry */ 817 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 818 { 819 struct net_device *dev = adapter->netdev; 820 u8 *addr = adapter->hw.mac.addr; 821 822 netdev_dbg(dev, "Set default MAC address filter: address %pM", addr); 823 824 igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1); 825 } 826 827 /** 828 * igc_set_mac - Change the Ethernet Address of the NIC 829 * @netdev: network interface device structure 830 * @p: pointer to an address structure 831 * 832 * Returns 0 on success, negative on failure 833 */ 834 static int igc_set_mac(struct net_device *netdev, void *p) 835 { 836 struct igc_adapter *adapter = netdev_priv(netdev); 837 struct igc_hw *hw = &adapter->hw; 838 struct sockaddr *addr = p; 839 840 if (!is_valid_ether_addr(addr->sa_data)) 841 return -EADDRNOTAVAIL; 842 843 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 844 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 845 846 /* set the correct pool for the new PF MAC address in entry 0 */ 847 igc_set_default_mac_filter(adapter); 848 849 return 0; 850 } 851 852 /** 853 * igc_write_mc_addr_list - write multicast addresses to MTA 854 * @netdev: network interface device structure 855 * 856 * Writes multicast address list to the MTA hash table. 857 * Returns: -ENOMEM on failure 858 * 0 on no addresses written 859 * X on writing X addresses to MTA 860 **/ 861 static int igc_write_mc_addr_list(struct net_device *netdev) 862 { 863 struct igc_adapter *adapter = netdev_priv(netdev); 864 struct igc_hw *hw = &adapter->hw; 865 struct netdev_hw_addr *ha; 866 u8 *mta_list; 867 int i; 868 869 if (netdev_mc_empty(netdev)) { 870 /* nothing to program, so clear mc list */ 871 igc_update_mc_addr_list(hw, NULL, 0); 872 return 0; 873 } 874 875 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 876 if (!mta_list) 877 return -ENOMEM; 878 879 /* The shared function expects a packed array of only addresses. */ 880 i = 0; 881 netdev_for_each_mc_addr(ha, netdev) 882 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 883 884 igc_update_mc_addr_list(hw, mta_list, i); 885 kfree(mta_list); 886 887 return netdev_mc_count(netdev); 888 } 889 890 static __le32 igc_tx_launchtime(struct igc_adapter *adapter, ktime_t txtime) 891 { 892 ktime_t cycle_time = adapter->cycle_time; 893 ktime_t base_time = adapter->base_time; 894 u32 launchtime; 895 896 /* FIXME: when using ETF together with taprio, we may have a 897 * case where 'delta' is larger than the cycle_time, this may 898 * cause problems if we don't read the current value of 899 * IGC_BASET, as the value writen into the launchtime 900 * descriptor field may be misinterpreted. 901 */ 902 div_s64_rem(ktime_sub_ns(txtime, base_time), cycle_time, &launchtime); 903 904 return cpu_to_le32(launchtime); 905 } 906 907 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, 908 struct igc_tx_buffer *first, 909 u32 vlan_macip_lens, u32 type_tucmd, 910 u32 mss_l4len_idx) 911 { 912 struct igc_adv_tx_context_desc *context_desc; 913 u16 i = tx_ring->next_to_use; 914 915 context_desc = IGC_TX_CTXTDESC(tx_ring, i); 916 917 i++; 918 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 919 920 /* set bits to identify this as an advanced context descriptor */ 921 type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 922 923 /* For i225, context index must be unique per ring. */ 924 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 925 mss_l4len_idx |= tx_ring->reg_idx << 4; 926 927 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 928 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 929 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 930 931 /* We assume there is always a valid Tx time available. Invalid times 932 * should have been handled by the upper layers. 933 */ 934 if (tx_ring->launchtime_enable) { 935 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 936 ktime_t txtime = first->skb->tstamp; 937 938 first->skb->tstamp = ktime_set(0, 0); 939 context_desc->launch_time = igc_tx_launchtime(adapter, 940 txtime); 941 } else { 942 context_desc->launch_time = 0; 943 } 944 } 945 946 static inline bool igc_ipv6_csum_is_sctp(struct sk_buff *skb) 947 { 948 unsigned int offset = 0; 949 950 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 951 952 return offset == skb_checksum_start_offset(skb); 953 } 954 955 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first) 956 { 957 struct sk_buff *skb = first->skb; 958 u32 vlan_macip_lens = 0; 959 u32 type_tucmd = 0; 960 961 if (skb->ip_summed != CHECKSUM_PARTIAL) { 962 csum_failed: 963 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && 964 !tx_ring->launchtime_enable) 965 return; 966 goto no_csum; 967 } 968 969 switch (skb->csum_offset) { 970 case offsetof(struct tcphdr, check): 971 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 972 fallthrough; 973 case offsetof(struct udphdr, check): 974 break; 975 case offsetof(struct sctphdr, checksum): 976 /* validate that this is actually an SCTP request */ 977 if ((first->protocol == htons(ETH_P_IP) && 978 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 979 (first->protocol == htons(ETH_P_IPV6) && 980 igc_ipv6_csum_is_sctp(skb))) { 981 type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; 982 break; 983 } 984 fallthrough; 985 default: 986 skb_checksum_help(skb); 987 goto csum_failed; 988 } 989 990 /* update TX checksum flag */ 991 first->tx_flags |= IGC_TX_FLAGS_CSUM; 992 vlan_macip_lens = skb_checksum_start_offset(skb) - 993 skb_network_offset(skb); 994 no_csum: 995 vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; 996 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 997 998 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0); 999 } 1000 1001 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1002 { 1003 struct net_device *netdev = tx_ring->netdev; 1004 1005 netif_stop_subqueue(netdev, tx_ring->queue_index); 1006 1007 /* memory barriier comment */ 1008 smp_mb(); 1009 1010 /* We need to check again in a case another CPU has just 1011 * made room available. 1012 */ 1013 if (igc_desc_unused(tx_ring) < size) 1014 return -EBUSY; 1015 1016 /* A reprieve! */ 1017 netif_wake_subqueue(netdev, tx_ring->queue_index); 1018 1019 u64_stats_update_begin(&tx_ring->tx_syncp2); 1020 tx_ring->tx_stats.restart_queue2++; 1021 u64_stats_update_end(&tx_ring->tx_syncp2); 1022 1023 return 0; 1024 } 1025 1026 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1027 { 1028 if (igc_desc_unused(tx_ring) >= size) 1029 return 0; 1030 return __igc_maybe_stop_tx(tx_ring, size); 1031 } 1032 1033 #define IGC_SET_FLAG(_input, _flag, _result) \ 1034 (((_flag) <= (_result)) ? \ 1035 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ 1036 ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) 1037 1038 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 1039 { 1040 /* set type for advanced descriptor with frame checksum insertion */ 1041 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 1042 IGC_ADVTXD_DCMD_DEXT | 1043 IGC_ADVTXD_DCMD_IFCS; 1044 1045 /* set segmentation bits for TSO */ 1046 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, 1047 (IGC_ADVTXD_DCMD_TSE)); 1048 1049 /* set timestamp bit if present */ 1050 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, 1051 (IGC_ADVTXD_MAC_TSTAMP)); 1052 1053 return cmd_type; 1054 } 1055 1056 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 1057 union igc_adv_tx_desc *tx_desc, 1058 u32 tx_flags, unsigned int paylen) 1059 { 1060 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 1061 1062 /* insert L4 checksum */ 1063 olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) * 1064 ((IGC_TXD_POPTS_TXSM << 8) / 1065 IGC_TX_FLAGS_CSUM); 1066 1067 /* insert IPv4 checksum */ 1068 olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) * 1069 (((IGC_TXD_POPTS_IXSM << 8)) / 1070 IGC_TX_FLAGS_IPV4); 1071 1072 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 1073 } 1074 1075 static int igc_tx_map(struct igc_ring *tx_ring, 1076 struct igc_tx_buffer *first, 1077 const u8 hdr_len) 1078 { 1079 struct sk_buff *skb = first->skb; 1080 struct igc_tx_buffer *tx_buffer; 1081 union igc_adv_tx_desc *tx_desc; 1082 u32 tx_flags = first->tx_flags; 1083 skb_frag_t *frag; 1084 u16 i = tx_ring->next_to_use; 1085 unsigned int data_len, size; 1086 dma_addr_t dma; 1087 u32 cmd_type = igc_tx_cmd_type(skb, tx_flags); 1088 1089 tx_desc = IGC_TX_DESC(tx_ring, i); 1090 1091 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 1092 1093 size = skb_headlen(skb); 1094 data_len = skb->data_len; 1095 1096 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1097 1098 tx_buffer = first; 1099 1100 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1101 if (dma_mapping_error(tx_ring->dev, dma)) 1102 goto dma_error; 1103 1104 /* record length, and DMA address */ 1105 dma_unmap_len_set(tx_buffer, len, size); 1106 dma_unmap_addr_set(tx_buffer, dma, dma); 1107 1108 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1109 1110 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 1111 tx_desc->read.cmd_type_len = 1112 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 1113 1114 i++; 1115 tx_desc++; 1116 if (i == tx_ring->count) { 1117 tx_desc = IGC_TX_DESC(tx_ring, 0); 1118 i = 0; 1119 } 1120 tx_desc->read.olinfo_status = 0; 1121 1122 dma += IGC_MAX_DATA_PER_TXD; 1123 size -= IGC_MAX_DATA_PER_TXD; 1124 1125 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1126 } 1127 1128 if (likely(!data_len)) 1129 break; 1130 1131 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 1132 1133 i++; 1134 tx_desc++; 1135 if (i == tx_ring->count) { 1136 tx_desc = IGC_TX_DESC(tx_ring, 0); 1137 i = 0; 1138 } 1139 tx_desc->read.olinfo_status = 0; 1140 1141 size = skb_frag_size(frag); 1142 data_len -= size; 1143 1144 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 1145 size, DMA_TO_DEVICE); 1146 1147 tx_buffer = &tx_ring->tx_buffer_info[i]; 1148 } 1149 1150 /* write last descriptor with RS and EOP bits */ 1151 cmd_type |= size | IGC_TXD_DCMD; 1152 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1153 1154 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1155 1156 /* set the timestamp */ 1157 first->time_stamp = jiffies; 1158 1159 skb_tx_timestamp(skb); 1160 1161 /* Force memory writes to complete before letting h/w know there 1162 * are new descriptors to fetch. (Only applicable for weak-ordered 1163 * memory model archs, such as IA-64). 1164 * 1165 * We also need this memory barrier to make certain all of the 1166 * status bits have been updated before next_to_watch is written. 1167 */ 1168 wmb(); 1169 1170 /* set next_to_watch value indicating a packet is present */ 1171 first->next_to_watch = tx_desc; 1172 1173 i++; 1174 if (i == tx_ring->count) 1175 i = 0; 1176 1177 tx_ring->next_to_use = i; 1178 1179 /* Make sure there is space in the ring for the next send. */ 1180 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 1181 1182 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1183 writel(i, tx_ring->tail); 1184 } 1185 1186 return 0; 1187 dma_error: 1188 netdev_err(tx_ring->netdev, "TX DMA map failed\n"); 1189 tx_buffer = &tx_ring->tx_buffer_info[i]; 1190 1191 /* clear dma mappings for failed tx_buffer_info map */ 1192 while (tx_buffer != first) { 1193 if (dma_unmap_len(tx_buffer, len)) 1194 dma_unmap_page(tx_ring->dev, 1195 dma_unmap_addr(tx_buffer, dma), 1196 dma_unmap_len(tx_buffer, len), 1197 DMA_TO_DEVICE); 1198 dma_unmap_len_set(tx_buffer, len, 0); 1199 1200 if (i-- == 0) 1201 i += tx_ring->count; 1202 tx_buffer = &tx_ring->tx_buffer_info[i]; 1203 } 1204 1205 if (dma_unmap_len(tx_buffer, len)) 1206 dma_unmap_single(tx_ring->dev, 1207 dma_unmap_addr(tx_buffer, dma), 1208 dma_unmap_len(tx_buffer, len), 1209 DMA_TO_DEVICE); 1210 dma_unmap_len_set(tx_buffer, len, 0); 1211 1212 dev_kfree_skb_any(tx_buffer->skb); 1213 tx_buffer->skb = NULL; 1214 1215 tx_ring->next_to_use = i; 1216 1217 return -1; 1218 } 1219 1220 static int igc_tso(struct igc_ring *tx_ring, 1221 struct igc_tx_buffer *first, 1222 u8 *hdr_len) 1223 { 1224 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 1225 struct sk_buff *skb = first->skb; 1226 union { 1227 struct iphdr *v4; 1228 struct ipv6hdr *v6; 1229 unsigned char *hdr; 1230 } ip; 1231 union { 1232 struct tcphdr *tcp; 1233 struct udphdr *udp; 1234 unsigned char *hdr; 1235 } l4; 1236 u32 paylen, l4_offset; 1237 int err; 1238 1239 if (skb->ip_summed != CHECKSUM_PARTIAL) 1240 return 0; 1241 1242 if (!skb_is_gso(skb)) 1243 return 0; 1244 1245 err = skb_cow_head(skb, 0); 1246 if (err < 0) 1247 return err; 1248 1249 ip.hdr = skb_network_header(skb); 1250 l4.hdr = skb_checksum_start(skb); 1251 1252 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 1253 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 1254 1255 /* initialize outer IP header fields */ 1256 if (ip.v4->version == 4) { 1257 unsigned char *csum_start = skb_checksum_start(skb); 1258 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 1259 1260 /* IP header will have to cancel out any data that 1261 * is not a part of the outer IP header 1262 */ 1263 ip.v4->check = csum_fold(csum_partial(trans_start, 1264 csum_start - trans_start, 1265 0)); 1266 type_tucmd |= IGC_ADVTXD_TUCMD_IPV4; 1267 1268 ip.v4->tot_len = 0; 1269 first->tx_flags |= IGC_TX_FLAGS_TSO | 1270 IGC_TX_FLAGS_CSUM | 1271 IGC_TX_FLAGS_IPV4; 1272 } else { 1273 ip.v6->payload_len = 0; 1274 first->tx_flags |= IGC_TX_FLAGS_TSO | 1275 IGC_TX_FLAGS_CSUM; 1276 } 1277 1278 /* determine offset of inner transport header */ 1279 l4_offset = l4.hdr - skb->data; 1280 1281 /* remove payload length from inner checksum */ 1282 paylen = skb->len - l4_offset; 1283 if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) { 1284 /* compute length of segmentation header */ 1285 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 1286 csum_replace_by_diff(&l4.tcp->check, 1287 (__force __wsum)htonl(paylen)); 1288 } else { 1289 /* compute length of segmentation header */ 1290 *hdr_len = sizeof(*l4.udp) + l4_offset; 1291 csum_replace_by_diff(&l4.udp->check, 1292 (__force __wsum)htonl(paylen)); 1293 } 1294 1295 /* update gso size and bytecount with header size */ 1296 first->gso_segs = skb_shinfo(skb)->gso_segs; 1297 first->bytecount += (first->gso_segs - 1) * *hdr_len; 1298 1299 /* MSS L4LEN IDX */ 1300 mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT; 1301 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT; 1302 1303 /* VLAN MACLEN IPLEN */ 1304 vlan_macip_lens = l4.hdr - ip.hdr; 1305 vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT; 1306 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1307 1308 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, 1309 type_tucmd, mss_l4len_idx); 1310 1311 return 1; 1312 } 1313 1314 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1315 struct igc_ring *tx_ring) 1316 { 1317 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1318 __be16 protocol = vlan_get_protocol(skb); 1319 struct igc_tx_buffer *first; 1320 u32 tx_flags = 0; 1321 unsigned short f; 1322 u8 hdr_len = 0; 1323 int tso = 0; 1324 1325 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1326 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1327 * + 2 desc gap to keep tail from touching head, 1328 * + 1 desc for context descriptor, 1329 * otherwise try next time 1330 */ 1331 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1332 count += TXD_USE_COUNT(skb_frag_size( 1333 &skb_shinfo(skb)->frags[f])); 1334 1335 if (igc_maybe_stop_tx(tx_ring, count + 3)) { 1336 /* this is a hard error */ 1337 return NETDEV_TX_BUSY; 1338 } 1339 1340 /* record the location of the first descriptor for this packet */ 1341 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1342 first->skb = skb; 1343 first->bytecount = skb->len; 1344 first->gso_segs = 1; 1345 1346 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 1347 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 1348 1349 /* FIXME: add support for retrieving timestamps from 1350 * the other timer registers before skipping the 1351 * timestamping request. 1352 */ 1353 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && 1354 !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS, 1355 &adapter->state)) { 1356 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1357 tx_flags |= IGC_TX_FLAGS_TSTAMP; 1358 1359 adapter->ptp_tx_skb = skb_get(skb); 1360 adapter->ptp_tx_start = jiffies; 1361 } else { 1362 adapter->tx_hwtstamp_skipped++; 1363 } 1364 } 1365 1366 /* record initial flags and protocol */ 1367 first->tx_flags = tx_flags; 1368 first->protocol = protocol; 1369 1370 tso = igc_tso(tx_ring, first, &hdr_len); 1371 if (tso < 0) 1372 goto out_drop; 1373 else if (!tso) 1374 igc_tx_csum(tx_ring, first); 1375 1376 igc_tx_map(tx_ring, first, hdr_len); 1377 1378 return NETDEV_TX_OK; 1379 1380 out_drop: 1381 dev_kfree_skb_any(first->skb); 1382 first->skb = NULL; 1383 1384 return NETDEV_TX_OK; 1385 } 1386 1387 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1388 struct sk_buff *skb) 1389 { 1390 unsigned int r_idx = skb->queue_mapping; 1391 1392 if (r_idx >= adapter->num_tx_queues) 1393 r_idx = r_idx % adapter->num_tx_queues; 1394 1395 return adapter->tx_ring[r_idx]; 1396 } 1397 1398 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1399 struct net_device *netdev) 1400 { 1401 struct igc_adapter *adapter = netdev_priv(netdev); 1402 1403 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1404 * in order to meet this minimum size requirement. 1405 */ 1406 if (skb->len < 17) { 1407 if (skb_padto(skb, 17)) 1408 return NETDEV_TX_OK; 1409 skb->len = 17; 1410 } 1411 1412 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1413 } 1414 1415 static void igc_rx_checksum(struct igc_ring *ring, 1416 union igc_adv_rx_desc *rx_desc, 1417 struct sk_buff *skb) 1418 { 1419 skb_checksum_none_assert(skb); 1420 1421 /* Ignore Checksum bit is set */ 1422 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) 1423 return; 1424 1425 /* Rx checksum disabled via ethtool */ 1426 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1427 return; 1428 1429 /* TCP/UDP checksum error bit is set */ 1430 if (igc_test_staterr(rx_desc, 1431 IGC_RXDEXT_STATERR_TCPE | 1432 IGC_RXDEXT_STATERR_IPE)) { 1433 /* work around errata with sctp packets where the TCPE aka 1434 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 1435 * packets (aka let the stack check the crc32c) 1436 */ 1437 if (!(skb->len == 60 && 1438 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 1439 u64_stats_update_begin(&ring->rx_syncp); 1440 ring->rx_stats.csum_err++; 1441 u64_stats_update_end(&ring->rx_syncp); 1442 } 1443 /* let the stack verify checksum errors */ 1444 return; 1445 } 1446 /* It must be a TCP or UDP packet with a valid checksum */ 1447 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | 1448 IGC_RXD_STAT_UDPCS)) 1449 skb->ip_summed = CHECKSUM_UNNECESSARY; 1450 1451 netdev_dbg(ring->netdev, "cksum success: bits %08X\n", 1452 le32_to_cpu(rx_desc->wb.upper.status_error)); 1453 } 1454 1455 static inline void igc_rx_hash(struct igc_ring *ring, 1456 union igc_adv_rx_desc *rx_desc, 1457 struct sk_buff *skb) 1458 { 1459 if (ring->netdev->features & NETIF_F_RXHASH) 1460 skb_set_hash(skb, 1461 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 1462 PKT_HASH_TYPE_L3); 1463 } 1464 1465 /** 1466 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1467 * @rx_ring: rx descriptor ring packet is being transacted on 1468 * @rx_desc: pointer to the EOP Rx descriptor 1469 * @skb: pointer to current skb being populated 1470 * 1471 * This function checks the ring, descriptor, and packet information in order 1472 * to populate the hash, checksum, VLAN, protocol, and other fields within the 1473 * skb. 1474 */ 1475 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1476 union igc_adv_rx_desc *rx_desc, 1477 struct sk_buff *skb) 1478 { 1479 igc_rx_hash(rx_ring, rx_desc, skb); 1480 1481 igc_rx_checksum(rx_ring, rx_desc, skb); 1482 1483 skb_record_rx_queue(skb, rx_ring->queue_index); 1484 1485 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1486 } 1487 1488 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1489 const unsigned int size) 1490 { 1491 struct igc_rx_buffer *rx_buffer; 1492 1493 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1494 prefetchw(rx_buffer->page); 1495 1496 /* we are reusing so sync this buffer for CPU use */ 1497 dma_sync_single_range_for_cpu(rx_ring->dev, 1498 rx_buffer->dma, 1499 rx_buffer->page_offset, 1500 size, 1501 DMA_FROM_DEVICE); 1502 1503 rx_buffer->pagecnt_bias--; 1504 1505 return rx_buffer; 1506 } 1507 1508 /** 1509 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1510 * @rx_ring: rx descriptor ring to transact packets on 1511 * @rx_buffer: buffer containing page to add 1512 * @skb: sk_buff to place the data into 1513 * @size: size of buffer to be added 1514 * 1515 * This function will add the data contained in rx_buffer->page to the skb. 1516 */ 1517 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1518 struct igc_rx_buffer *rx_buffer, 1519 struct sk_buff *skb, 1520 unsigned int size) 1521 { 1522 #if (PAGE_SIZE < 8192) 1523 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1524 1525 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1526 rx_buffer->page_offset, size, truesize); 1527 rx_buffer->page_offset ^= truesize; 1528 #else 1529 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1530 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1531 SKB_DATA_ALIGN(size); 1532 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1533 rx_buffer->page_offset, size, truesize); 1534 rx_buffer->page_offset += truesize; 1535 #endif 1536 } 1537 1538 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1539 struct igc_rx_buffer *rx_buffer, 1540 union igc_adv_rx_desc *rx_desc, 1541 unsigned int size) 1542 { 1543 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1544 #if (PAGE_SIZE < 8192) 1545 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1546 #else 1547 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1548 SKB_DATA_ALIGN(IGC_SKB_PAD + size); 1549 #endif 1550 struct sk_buff *skb; 1551 1552 /* prefetch first cache line of first page */ 1553 net_prefetch(va); 1554 1555 /* build an skb around the page buffer */ 1556 skb = build_skb(va - IGC_SKB_PAD, truesize); 1557 if (unlikely(!skb)) 1558 return NULL; 1559 1560 /* update pointers within the skb to store the data */ 1561 skb_reserve(skb, IGC_SKB_PAD); 1562 __skb_put(skb, size); 1563 1564 /* update buffer offset */ 1565 #if (PAGE_SIZE < 8192) 1566 rx_buffer->page_offset ^= truesize; 1567 #else 1568 rx_buffer->page_offset += truesize; 1569 #endif 1570 1571 return skb; 1572 } 1573 1574 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1575 struct igc_rx_buffer *rx_buffer, 1576 union igc_adv_rx_desc *rx_desc, 1577 unsigned int size) 1578 { 1579 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1580 #if (PAGE_SIZE < 8192) 1581 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1582 #else 1583 unsigned int truesize = SKB_DATA_ALIGN(size); 1584 #endif 1585 unsigned int headlen; 1586 struct sk_buff *skb; 1587 1588 /* prefetch first cache line of first page */ 1589 net_prefetch(va); 1590 1591 /* allocate a skb to store the frags */ 1592 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); 1593 if (unlikely(!skb)) 1594 return NULL; 1595 1596 if (unlikely(igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP))) { 1597 igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb); 1598 va += IGC_TS_HDR_LEN; 1599 size -= IGC_TS_HDR_LEN; 1600 } 1601 1602 /* Determine available headroom for copy */ 1603 headlen = size; 1604 if (headlen > IGC_RX_HDR_LEN) 1605 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 1606 1607 /* align pull length to size of long to optimize memcpy performance */ 1608 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 1609 1610 /* update all of the pointers */ 1611 size -= headlen; 1612 if (size) { 1613 skb_add_rx_frag(skb, 0, rx_buffer->page, 1614 (va + headlen) - page_address(rx_buffer->page), 1615 size, truesize); 1616 #if (PAGE_SIZE < 8192) 1617 rx_buffer->page_offset ^= truesize; 1618 #else 1619 rx_buffer->page_offset += truesize; 1620 #endif 1621 } else { 1622 rx_buffer->pagecnt_bias++; 1623 } 1624 1625 return skb; 1626 } 1627 1628 /** 1629 * igc_reuse_rx_page - page flip buffer and store it back on the ring 1630 * @rx_ring: rx descriptor ring to store buffers on 1631 * @old_buff: donor buffer to have page reused 1632 * 1633 * Synchronizes page for reuse by the adapter 1634 */ 1635 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 1636 struct igc_rx_buffer *old_buff) 1637 { 1638 u16 nta = rx_ring->next_to_alloc; 1639 struct igc_rx_buffer *new_buff; 1640 1641 new_buff = &rx_ring->rx_buffer_info[nta]; 1642 1643 /* update, and store next to alloc */ 1644 nta++; 1645 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1646 1647 /* Transfer page from old buffer to new buffer. 1648 * Move each member individually to avoid possible store 1649 * forwarding stalls. 1650 */ 1651 new_buff->dma = old_buff->dma; 1652 new_buff->page = old_buff->page; 1653 new_buff->page_offset = old_buff->page_offset; 1654 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1655 } 1656 1657 static inline bool igc_page_is_reserved(struct page *page) 1658 { 1659 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 1660 } 1661 1662 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) 1663 { 1664 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1665 struct page *page = rx_buffer->page; 1666 1667 /* avoid re-using remote pages */ 1668 if (unlikely(igc_page_is_reserved(page))) 1669 return false; 1670 1671 #if (PAGE_SIZE < 8192) 1672 /* if we are only owner of page we can reuse it */ 1673 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 1674 return false; 1675 #else 1676 #define IGC_LAST_OFFSET \ 1677 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 1678 1679 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 1680 return false; 1681 #endif 1682 1683 /* If we have drained the page fragment pool we need to update 1684 * the pagecnt_bias and page count so that we fully restock the 1685 * number of references the driver holds. 1686 */ 1687 if (unlikely(!pagecnt_bias)) { 1688 page_ref_add(page, USHRT_MAX); 1689 rx_buffer->pagecnt_bias = USHRT_MAX; 1690 } 1691 1692 return true; 1693 } 1694 1695 /** 1696 * igc_is_non_eop - process handling of non-EOP buffers 1697 * @rx_ring: Rx ring being processed 1698 * @rx_desc: Rx descriptor for current buffer 1699 * 1700 * This function updates next to clean. If the buffer is an EOP buffer 1701 * this function exits returning false, otherwise it will place the 1702 * sk_buff in the next buffer to be chained and return true indicating 1703 * that this is in fact a non-EOP buffer. 1704 */ 1705 static bool igc_is_non_eop(struct igc_ring *rx_ring, 1706 union igc_adv_rx_desc *rx_desc) 1707 { 1708 u32 ntc = rx_ring->next_to_clean + 1; 1709 1710 /* fetch, update, and store next to clean */ 1711 ntc = (ntc < rx_ring->count) ? ntc : 0; 1712 rx_ring->next_to_clean = ntc; 1713 1714 prefetch(IGC_RX_DESC(rx_ring, ntc)); 1715 1716 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 1717 return false; 1718 1719 return true; 1720 } 1721 1722 /** 1723 * igc_cleanup_headers - Correct corrupted or empty headers 1724 * @rx_ring: rx descriptor ring packet is being transacted on 1725 * @rx_desc: pointer to the EOP Rx descriptor 1726 * @skb: pointer to current skb being fixed 1727 * 1728 * Address the case where we are pulling data in on pages only 1729 * and as such no data is present in the skb header. 1730 * 1731 * In addition if skb is not at least 60 bytes we need to pad it so that 1732 * it is large enough to qualify as a valid Ethernet frame. 1733 * 1734 * Returns true if an error was encountered and skb was freed. 1735 */ 1736 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 1737 union igc_adv_rx_desc *rx_desc, 1738 struct sk_buff *skb) 1739 { 1740 if (unlikely((igc_test_staterr(rx_desc, 1741 IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) { 1742 struct net_device *netdev = rx_ring->netdev; 1743 1744 if (!(netdev->features & NETIF_F_RXALL)) { 1745 dev_kfree_skb_any(skb); 1746 return true; 1747 } 1748 } 1749 1750 /* if eth_skb_pad returns an error the skb was freed */ 1751 if (eth_skb_pad(skb)) 1752 return true; 1753 1754 return false; 1755 } 1756 1757 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 1758 struct igc_rx_buffer *rx_buffer) 1759 { 1760 if (igc_can_reuse_rx_page(rx_buffer)) { 1761 /* hand second half of page back to the ring */ 1762 igc_reuse_rx_page(rx_ring, rx_buffer); 1763 } else { 1764 /* We are not reusing the buffer so unmap it and free 1765 * any references we are holding to it 1766 */ 1767 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1768 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 1769 IGC_RX_DMA_ATTR); 1770 __page_frag_cache_drain(rx_buffer->page, 1771 rx_buffer->pagecnt_bias); 1772 } 1773 1774 /* clear contents of rx_buffer */ 1775 rx_buffer->page = NULL; 1776 } 1777 1778 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 1779 { 1780 return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; 1781 } 1782 1783 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 1784 struct igc_rx_buffer *bi) 1785 { 1786 struct page *page = bi->page; 1787 dma_addr_t dma; 1788 1789 /* since we are recycling buffers we should seldom need to alloc */ 1790 if (likely(page)) 1791 return true; 1792 1793 /* alloc new page for storage */ 1794 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 1795 if (unlikely(!page)) { 1796 rx_ring->rx_stats.alloc_failed++; 1797 return false; 1798 } 1799 1800 /* map page for use */ 1801 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1802 igc_rx_pg_size(rx_ring), 1803 DMA_FROM_DEVICE, 1804 IGC_RX_DMA_ATTR); 1805 1806 /* if mapping failed free memory back to system since 1807 * there isn't much point in holding memory we can't use 1808 */ 1809 if (dma_mapping_error(rx_ring->dev, dma)) { 1810 __free_page(page); 1811 1812 rx_ring->rx_stats.alloc_failed++; 1813 return false; 1814 } 1815 1816 bi->dma = dma; 1817 bi->page = page; 1818 bi->page_offset = igc_rx_offset(rx_ring); 1819 bi->pagecnt_bias = 1; 1820 1821 return true; 1822 } 1823 1824 /** 1825 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 1826 * @rx_ring: rx descriptor ring 1827 * @cleaned_count: number of buffers to clean 1828 */ 1829 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 1830 { 1831 union igc_adv_rx_desc *rx_desc; 1832 u16 i = rx_ring->next_to_use; 1833 struct igc_rx_buffer *bi; 1834 u16 bufsz; 1835 1836 /* nothing to do */ 1837 if (!cleaned_count) 1838 return; 1839 1840 rx_desc = IGC_RX_DESC(rx_ring, i); 1841 bi = &rx_ring->rx_buffer_info[i]; 1842 i -= rx_ring->count; 1843 1844 bufsz = igc_rx_bufsz(rx_ring); 1845 1846 do { 1847 if (!igc_alloc_mapped_page(rx_ring, bi)) 1848 break; 1849 1850 /* sync the buffer for use by the device */ 1851 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1852 bi->page_offset, bufsz, 1853 DMA_FROM_DEVICE); 1854 1855 /* Refresh the desc even if buffer_addrs didn't change 1856 * because each write-back erases this info. 1857 */ 1858 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1859 1860 rx_desc++; 1861 bi++; 1862 i++; 1863 if (unlikely(!i)) { 1864 rx_desc = IGC_RX_DESC(rx_ring, 0); 1865 bi = rx_ring->rx_buffer_info; 1866 i -= rx_ring->count; 1867 } 1868 1869 /* clear the length for the next_to_use descriptor */ 1870 rx_desc->wb.upper.length = 0; 1871 1872 cleaned_count--; 1873 } while (cleaned_count); 1874 1875 i += rx_ring->count; 1876 1877 if (rx_ring->next_to_use != i) { 1878 /* record the next descriptor to use */ 1879 rx_ring->next_to_use = i; 1880 1881 /* update next to alloc since we have filled the ring */ 1882 rx_ring->next_to_alloc = i; 1883 1884 /* Force memory writes to complete before letting h/w 1885 * know there are new descriptors to fetch. (Only 1886 * applicable for weak-ordered memory model archs, 1887 * such as IA-64). 1888 */ 1889 wmb(); 1890 writel(i, rx_ring->tail); 1891 } 1892 } 1893 1894 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 1895 { 1896 unsigned int total_bytes = 0, total_packets = 0; 1897 struct igc_ring *rx_ring = q_vector->rx.ring; 1898 struct sk_buff *skb = rx_ring->skb; 1899 u16 cleaned_count = igc_desc_unused(rx_ring); 1900 1901 while (likely(total_packets < budget)) { 1902 union igc_adv_rx_desc *rx_desc; 1903 struct igc_rx_buffer *rx_buffer; 1904 unsigned int size; 1905 1906 /* return some buffers to hardware, one at a time is too slow */ 1907 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 1908 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1909 cleaned_count = 0; 1910 } 1911 1912 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 1913 size = le16_to_cpu(rx_desc->wb.upper.length); 1914 if (!size) 1915 break; 1916 1917 /* This memory barrier is needed to keep us from reading 1918 * any other fields out of the rx_desc until we know the 1919 * descriptor has been written back 1920 */ 1921 dma_rmb(); 1922 1923 rx_buffer = igc_get_rx_buffer(rx_ring, size); 1924 1925 /* retrieve a buffer from the ring */ 1926 if (skb) 1927 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 1928 else if (ring_uses_build_skb(rx_ring)) 1929 skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); 1930 else 1931 skb = igc_construct_skb(rx_ring, rx_buffer, 1932 rx_desc, size); 1933 1934 /* exit if we failed to retrieve a buffer */ 1935 if (!skb) { 1936 rx_ring->rx_stats.alloc_failed++; 1937 rx_buffer->pagecnt_bias++; 1938 break; 1939 } 1940 1941 igc_put_rx_buffer(rx_ring, rx_buffer); 1942 cleaned_count++; 1943 1944 /* fetch next buffer in frame if non-eop */ 1945 if (igc_is_non_eop(rx_ring, rx_desc)) 1946 continue; 1947 1948 /* verify the packet layout is correct */ 1949 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) { 1950 skb = NULL; 1951 continue; 1952 } 1953 1954 /* probably a little skewed due to removing CRC */ 1955 total_bytes += skb->len; 1956 1957 /* populate checksum, VLAN, and protocol */ 1958 igc_process_skb_fields(rx_ring, rx_desc, skb); 1959 1960 napi_gro_receive(&q_vector->napi, skb); 1961 1962 /* reset skb pointer */ 1963 skb = NULL; 1964 1965 /* update budget accounting */ 1966 total_packets++; 1967 } 1968 1969 /* place incomplete frames back on ring for completion */ 1970 rx_ring->skb = skb; 1971 1972 u64_stats_update_begin(&rx_ring->rx_syncp); 1973 rx_ring->rx_stats.packets += total_packets; 1974 rx_ring->rx_stats.bytes += total_bytes; 1975 u64_stats_update_end(&rx_ring->rx_syncp); 1976 q_vector->rx.total_packets += total_packets; 1977 q_vector->rx.total_bytes += total_bytes; 1978 1979 if (cleaned_count) 1980 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1981 1982 return total_packets; 1983 } 1984 1985 /** 1986 * igc_clean_tx_irq - Reclaim resources after transmit completes 1987 * @q_vector: pointer to q_vector containing needed info 1988 * @napi_budget: Used to determine if we are in netpoll 1989 * 1990 * returns true if ring is completely cleaned 1991 */ 1992 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 1993 { 1994 struct igc_adapter *adapter = q_vector->adapter; 1995 unsigned int total_bytes = 0, total_packets = 0; 1996 unsigned int budget = q_vector->tx.work_limit; 1997 struct igc_ring *tx_ring = q_vector->tx.ring; 1998 unsigned int i = tx_ring->next_to_clean; 1999 struct igc_tx_buffer *tx_buffer; 2000 union igc_adv_tx_desc *tx_desc; 2001 2002 if (test_bit(__IGC_DOWN, &adapter->state)) 2003 return true; 2004 2005 tx_buffer = &tx_ring->tx_buffer_info[i]; 2006 tx_desc = IGC_TX_DESC(tx_ring, i); 2007 i -= tx_ring->count; 2008 2009 do { 2010 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 2011 2012 /* if next_to_watch is not set then there is no work pending */ 2013 if (!eop_desc) 2014 break; 2015 2016 /* prevent any other reads prior to eop_desc */ 2017 smp_rmb(); 2018 2019 /* if DD is not set pending work has not been completed */ 2020 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 2021 break; 2022 2023 /* clear next_to_watch to prevent false hangs */ 2024 tx_buffer->next_to_watch = NULL; 2025 2026 /* update the statistics for this packet */ 2027 total_bytes += tx_buffer->bytecount; 2028 total_packets += tx_buffer->gso_segs; 2029 2030 /* free the skb */ 2031 napi_consume_skb(tx_buffer->skb, napi_budget); 2032 2033 /* unmap skb header data */ 2034 dma_unmap_single(tx_ring->dev, 2035 dma_unmap_addr(tx_buffer, dma), 2036 dma_unmap_len(tx_buffer, len), 2037 DMA_TO_DEVICE); 2038 2039 /* clear tx_buffer data */ 2040 dma_unmap_len_set(tx_buffer, len, 0); 2041 2042 /* clear last DMA location and unmap remaining buffers */ 2043 while (tx_desc != eop_desc) { 2044 tx_buffer++; 2045 tx_desc++; 2046 i++; 2047 if (unlikely(!i)) { 2048 i -= tx_ring->count; 2049 tx_buffer = tx_ring->tx_buffer_info; 2050 tx_desc = IGC_TX_DESC(tx_ring, 0); 2051 } 2052 2053 /* unmap any remaining paged data */ 2054 if (dma_unmap_len(tx_buffer, len)) { 2055 dma_unmap_page(tx_ring->dev, 2056 dma_unmap_addr(tx_buffer, dma), 2057 dma_unmap_len(tx_buffer, len), 2058 DMA_TO_DEVICE); 2059 dma_unmap_len_set(tx_buffer, len, 0); 2060 } 2061 } 2062 2063 /* move us one more past the eop_desc for start of next pkt */ 2064 tx_buffer++; 2065 tx_desc++; 2066 i++; 2067 if (unlikely(!i)) { 2068 i -= tx_ring->count; 2069 tx_buffer = tx_ring->tx_buffer_info; 2070 tx_desc = IGC_TX_DESC(tx_ring, 0); 2071 } 2072 2073 /* issue prefetch for next Tx descriptor */ 2074 prefetch(tx_desc); 2075 2076 /* update budget accounting */ 2077 budget--; 2078 } while (likely(budget)); 2079 2080 netdev_tx_completed_queue(txring_txq(tx_ring), 2081 total_packets, total_bytes); 2082 2083 i += tx_ring->count; 2084 tx_ring->next_to_clean = i; 2085 u64_stats_update_begin(&tx_ring->tx_syncp); 2086 tx_ring->tx_stats.bytes += total_bytes; 2087 tx_ring->tx_stats.packets += total_packets; 2088 u64_stats_update_end(&tx_ring->tx_syncp); 2089 q_vector->tx.total_bytes += total_bytes; 2090 q_vector->tx.total_packets += total_packets; 2091 2092 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 2093 struct igc_hw *hw = &adapter->hw; 2094 2095 /* Detect a transmit hang in hardware, this serializes the 2096 * check with the clearing of time_stamp and movement of i 2097 */ 2098 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 2099 if (tx_buffer->next_to_watch && 2100 time_after(jiffies, tx_buffer->time_stamp + 2101 (adapter->tx_timeout_factor * HZ)) && 2102 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { 2103 /* detected Tx unit hang */ 2104 netdev_err(tx_ring->netdev, 2105 "Detected Tx Unit Hang\n" 2106 " Tx Queue <%d>\n" 2107 " TDH <%x>\n" 2108 " TDT <%x>\n" 2109 " next_to_use <%x>\n" 2110 " next_to_clean <%x>\n" 2111 "buffer_info[next_to_clean]\n" 2112 " time_stamp <%lx>\n" 2113 " next_to_watch <%p>\n" 2114 " jiffies <%lx>\n" 2115 " desc.status <%x>\n", 2116 tx_ring->queue_index, 2117 rd32(IGC_TDH(tx_ring->reg_idx)), 2118 readl(tx_ring->tail), 2119 tx_ring->next_to_use, 2120 tx_ring->next_to_clean, 2121 tx_buffer->time_stamp, 2122 tx_buffer->next_to_watch, 2123 jiffies, 2124 tx_buffer->next_to_watch->wb.status); 2125 netif_stop_subqueue(tx_ring->netdev, 2126 tx_ring->queue_index); 2127 2128 /* we are about to reset, no point in enabling stuff */ 2129 return true; 2130 } 2131 } 2132 2133 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 2134 if (unlikely(total_packets && 2135 netif_carrier_ok(tx_ring->netdev) && 2136 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 2137 /* Make sure that anybody stopping the queue after this 2138 * sees the new next_to_clean. 2139 */ 2140 smp_mb(); 2141 if (__netif_subqueue_stopped(tx_ring->netdev, 2142 tx_ring->queue_index) && 2143 !(test_bit(__IGC_DOWN, &adapter->state))) { 2144 netif_wake_subqueue(tx_ring->netdev, 2145 tx_ring->queue_index); 2146 2147 u64_stats_update_begin(&tx_ring->tx_syncp); 2148 tx_ring->tx_stats.restart_queue++; 2149 u64_stats_update_end(&tx_ring->tx_syncp); 2150 } 2151 } 2152 2153 return !!budget; 2154 } 2155 2156 static int igc_find_mac_filter(struct igc_adapter *adapter, 2157 enum igc_mac_filter_type type, const u8 *addr) 2158 { 2159 struct igc_hw *hw = &adapter->hw; 2160 int max_entries = hw->mac.rar_entry_count; 2161 u32 ral, rah; 2162 int i; 2163 2164 for (i = 0; i < max_entries; i++) { 2165 ral = rd32(IGC_RAL(i)); 2166 rah = rd32(IGC_RAH(i)); 2167 2168 if (!(rah & IGC_RAH_AV)) 2169 continue; 2170 if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type) 2171 continue; 2172 if ((rah & IGC_RAH_RAH_MASK) != 2173 le16_to_cpup((__le16 *)(addr + 4))) 2174 continue; 2175 if (ral != le32_to_cpup((__le32 *)(addr))) 2176 continue; 2177 2178 return i; 2179 } 2180 2181 return -1; 2182 } 2183 2184 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) 2185 { 2186 struct igc_hw *hw = &adapter->hw; 2187 int max_entries = hw->mac.rar_entry_count; 2188 u32 rah; 2189 int i; 2190 2191 for (i = 0; i < max_entries; i++) { 2192 rah = rd32(IGC_RAH(i)); 2193 2194 if (!(rah & IGC_RAH_AV)) 2195 return i; 2196 } 2197 2198 return -1; 2199 } 2200 2201 /** 2202 * igc_add_mac_filter() - Add MAC address filter 2203 * @adapter: Pointer to adapter where the filter should be added 2204 * @type: MAC address filter type (source or destination) 2205 * @addr: MAC address 2206 * @queue: If non-negative, queue assignment feature is enabled and frames 2207 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2208 * assignment is disabled. 2209 * 2210 * Return: 0 in case of success, negative errno code otherwise. 2211 */ 2212 static int igc_add_mac_filter(struct igc_adapter *adapter, 2213 enum igc_mac_filter_type type, const u8 *addr, 2214 int queue) 2215 { 2216 struct net_device *dev = adapter->netdev; 2217 int index; 2218 2219 index = igc_find_mac_filter(adapter, type, addr); 2220 if (index >= 0) 2221 goto update_filter; 2222 2223 index = igc_get_avail_mac_filter_slot(adapter); 2224 if (index < 0) 2225 return -ENOSPC; 2226 2227 netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n", 2228 index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2229 addr, queue); 2230 2231 update_filter: 2232 igc_set_mac_filter_hw(adapter, index, type, addr, queue); 2233 return 0; 2234 } 2235 2236 /** 2237 * igc_del_mac_filter() - Delete MAC address filter 2238 * @adapter: Pointer to adapter where the filter should be deleted from 2239 * @type: MAC address filter type (source or destination) 2240 * @addr: MAC address 2241 */ 2242 static void igc_del_mac_filter(struct igc_adapter *adapter, 2243 enum igc_mac_filter_type type, const u8 *addr) 2244 { 2245 struct net_device *dev = adapter->netdev; 2246 int index; 2247 2248 index = igc_find_mac_filter(adapter, type, addr); 2249 if (index < 0) 2250 return; 2251 2252 if (index == 0) { 2253 /* If this is the default filter, we don't actually delete it. 2254 * We just reset to its default value i.e. disable queue 2255 * assignment. 2256 */ 2257 netdev_dbg(dev, "Disable default MAC filter queue assignment"); 2258 2259 igc_set_mac_filter_hw(adapter, 0, type, addr, -1); 2260 } else { 2261 netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n", 2262 index, 2263 type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2264 addr); 2265 2266 igc_clear_mac_filter_hw(adapter, index); 2267 } 2268 } 2269 2270 /** 2271 * igc_add_vlan_prio_filter() - Add VLAN priority filter 2272 * @adapter: Pointer to adapter where the filter should be added 2273 * @prio: VLAN priority value 2274 * @queue: Queue number which matching frames are assigned to 2275 * 2276 * Return: 0 in case of success, negative errno code otherwise. 2277 */ 2278 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, 2279 int queue) 2280 { 2281 struct net_device *dev = adapter->netdev; 2282 struct igc_hw *hw = &adapter->hw; 2283 u32 vlanpqf; 2284 2285 vlanpqf = rd32(IGC_VLANPQF); 2286 2287 if (vlanpqf & IGC_VLANPQF_VALID(prio)) { 2288 netdev_dbg(dev, "VLAN priority filter already in use\n"); 2289 return -EEXIST; 2290 } 2291 2292 vlanpqf |= IGC_VLANPQF_QSEL(prio, queue); 2293 vlanpqf |= IGC_VLANPQF_VALID(prio); 2294 2295 wr32(IGC_VLANPQF, vlanpqf); 2296 2297 netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n", 2298 prio, queue); 2299 return 0; 2300 } 2301 2302 /** 2303 * igc_del_vlan_prio_filter() - Delete VLAN priority filter 2304 * @adapter: Pointer to adapter where the filter should be deleted from 2305 * @prio: VLAN priority value 2306 */ 2307 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) 2308 { 2309 struct igc_hw *hw = &adapter->hw; 2310 u32 vlanpqf; 2311 2312 vlanpqf = rd32(IGC_VLANPQF); 2313 2314 vlanpqf &= ~IGC_VLANPQF_VALID(prio); 2315 vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK); 2316 2317 wr32(IGC_VLANPQF, vlanpqf); 2318 2319 netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n", 2320 prio); 2321 } 2322 2323 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) 2324 { 2325 struct igc_hw *hw = &adapter->hw; 2326 int i; 2327 2328 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2329 u32 etqf = rd32(IGC_ETQF(i)); 2330 2331 if (!(etqf & IGC_ETQF_FILTER_ENABLE)) 2332 return i; 2333 } 2334 2335 return -1; 2336 } 2337 2338 /** 2339 * igc_add_etype_filter() - Add ethertype filter 2340 * @adapter: Pointer to adapter where the filter should be added 2341 * @etype: Ethertype value 2342 * @queue: If non-negative, queue assignment feature is enabled and frames 2343 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2344 * assignment is disabled. 2345 * 2346 * Return: 0 in case of success, negative errno code otherwise. 2347 */ 2348 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, 2349 int queue) 2350 { 2351 struct igc_hw *hw = &adapter->hw; 2352 int index; 2353 u32 etqf; 2354 2355 index = igc_get_avail_etype_filter_slot(adapter); 2356 if (index < 0) 2357 return -ENOSPC; 2358 2359 etqf = rd32(IGC_ETQF(index)); 2360 2361 etqf &= ~IGC_ETQF_ETYPE_MASK; 2362 etqf |= etype; 2363 2364 if (queue >= 0) { 2365 etqf &= ~IGC_ETQF_QUEUE_MASK; 2366 etqf |= (queue << IGC_ETQF_QUEUE_SHIFT); 2367 etqf |= IGC_ETQF_QUEUE_ENABLE; 2368 } 2369 2370 etqf |= IGC_ETQF_FILTER_ENABLE; 2371 2372 wr32(IGC_ETQF(index), etqf); 2373 2374 netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n", 2375 etype, queue); 2376 return 0; 2377 } 2378 2379 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) 2380 { 2381 struct igc_hw *hw = &adapter->hw; 2382 int i; 2383 2384 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2385 u32 etqf = rd32(IGC_ETQF(i)); 2386 2387 if ((etqf & IGC_ETQF_ETYPE_MASK) == etype) 2388 return i; 2389 } 2390 2391 return -1; 2392 } 2393 2394 /** 2395 * igc_del_etype_filter() - Delete ethertype filter 2396 * @adapter: Pointer to adapter where the filter should be deleted from 2397 * @etype: Ethertype value 2398 */ 2399 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) 2400 { 2401 struct igc_hw *hw = &adapter->hw; 2402 int index; 2403 2404 index = igc_find_etype_filter(adapter, etype); 2405 if (index < 0) 2406 return; 2407 2408 wr32(IGC_ETQF(index), 0); 2409 2410 netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n", 2411 etype); 2412 } 2413 2414 static int igc_enable_nfc_rule(struct igc_adapter *adapter, 2415 const struct igc_nfc_rule *rule) 2416 { 2417 int err; 2418 2419 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { 2420 err = igc_add_etype_filter(adapter, rule->filter.etype, 2421 rule->action); 2422 if (err) 2423 return err; 2424 } 2425 2426 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { 2427 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2428 rule->filter.src_addr, rule->action); 2429 if (err) 2430 return err; 2431 } 2432 2433 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { 2434 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2435 rule->filter.dst_addr, rule->action); 2436 if (err) 2437 return err; 2438 } 2439 2440 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2441 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2442 VLAN_PRIO_SHIFT; 2443 2444 err = igc_add_vlan_prio_filter(adapter, prio, rule->action); 2445 if (err) 2446 return err; 2447 } 2448 2449 return 0; 2450 } 2451 2452 static void igc_disable_nfc_rule(struct igc_adapter *adapter, 2453 const struct igc_nfc_rule *rule) 2454 { 2455 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) 2456 igc_del_etype_filter(adapter, rule->filter.etype); 2457 2458 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2459 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2460 VLAN_PRIO_SHIFT; 2461 2462 igc_del_vlan_prio_filter(adapter, prio); 2463 } 2464 2465 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) 2466 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2467 rule->filter.src_addr); 2468 2469 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) 2470 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2471 rule->filter.dst_addr); 2472 } 2473 2474 /** 2475 * igc_get_nfc_rule() - Get NFC rule 2476 * @adapter: Pointer to adapter 2477 * @location: Rule location 2478 * 2479 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2480 * 2481 * Return: Pointer to NFC rule at @location. If not found, NULL. 2482 */ 2483 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, 2484 u32 location) 2485 { 2486 struct igc_nfc_rule *rule; 2487 2488 list_for_each_entry(rule, &adapter->nfc_rule_list, list) { 2489 if (rule->location == location) 2490 return rule; 2491 if (rule->location > location) 2492 break; 2493 } 2494 2495 return NULL; 2496 } 2497 2498 /** 2499 * igc_del_nfc_rule() - Delete NFC rule 2500 * @adapter: Pointer to adapter 2501 * @rule: Pointer to rule to be deleted 2502 * 2503 * Disable NFC rule in hardware and delete it from adapter. 2504 * 2505 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2506 */ 2507 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2508 { 2509 igc_disable_nfc_rule(adapter, rule); 2510 2511 list_del(&rule->list); 2512 adapter->nfc_rule_count--; 2513 2514 kfree(rule); 2515 } 2516 2517 static void igc_flush_nfc_rules(struct igc_adapter *adapter) 2518 { 2519 struct igc_nfc_rule *rule, *tmp; 2520 2521 mutex_lock(&adapter->nfc_rule_lock); 2522 2523 list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list) 2524 igc_del_nfc_rule(adapter, rule); 2525 2526 mutex_unlock(&adapter->nfc_rule_lock); 2527 } 2528 2529 /** 2530 * igc_add_nfc_rule() - Add NFC rule 2531 * @adapter: Pointer to adapter 2532 * @rule: Pointer to rule to be added 2533 * 2534 * Enable NFC rule in hardware and add it to adapter. 2535 * 2536 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2537 * 2538 * Return: 0 on success, negative errno on failure. 2539 */ 2540 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2541 { 2542 struct igc_nfc_rule *pred, *cur; 2543 int err; 2544 2545 err = igc_enable_nfc_rule(adapter, rule); 2546 if (err) 2547 return err; 2548 2549 pred = NULL; 2550 list_for_each_entry(cur, &adapter->nfc_rule_list, list) { 2551 if (cur->location >= rule->location) 2552 break; 2553 pred = cur; 2554 } 2555 2556 list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list); 2557 adapter->nfc_rule_count++; 2558 return 0; 2559 } 2560 2561 static void igc_restore_nfc_rules(struct igc_adapter *adapter) 2562 { 2563 struct igc_nfc_rule *rule; 2564 2565 mutex_lock(&adapter->nfc_rule_lock); 2566 2567 list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list) 2568 igc_enable_nfc_rule(adapter, rule); 2569 2570 mutex_unlock(&adapter->nfc_rule_lock); 2571 } 2572 2573 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) 2574 { 2575 struct igc_adapter *adapter = netdev_priv(netdev); 2576 2577 return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1); 2578 } 2579 2580 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) 2581 { 2582 struct igc_adapter *adapter = netdev_priv(netdev); 2583 2584 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr); 2585 return 0; 2586 } 2587 2588 /** 2589 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 2590 * @netdev: network interface device structure 2591 * 2592 * The set_rx_mode entry point is called whenever the unicast or multicast 2593 * address lists or the network interface flags are updated. This routine is 2594 * responsible for configuring the hardware for proper unicast, multicast, 2595 * promiscuous mode, and all-multi behavior. 2596 */ 2597 static void igc_set_rx_mode(struct net_device *netdev) 2598 { 2599 struct igc_adapter *adapter = netdev_priv(netdev); 2600 struct igc_hw *hw = &adapter->hw; 2601 u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 2602 int count; 2603 2604 /* Check for Promiscuous and All Multicast modes */ 2605 if (netdev->flags & IFF_PROMISC) { 2606 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; 2607 } else { 2608 if (netdev->flags & IFF_ALLMULTI) { 2609 rctl |= IGC_RCTL_MPE; 2610 } else { 2611 /* Write addresses to the MTA, if the attempt fails 2612 * then we should just turn on promiscuous mode so 2613 * that we can at least receive multicast traffic 2614 */ 2615 count = igc_write_mc_addr_list(netdev); 2616 if (count < 0) 2617 rctl |= IGC_RCTL_MPE; 2618 } 2619 } 2620 2621 /* Write addresses to available RAR registers, if there is not 2622 * sufficient space to store all the addresses then enable 2623 * unicast promiscuous mode 2624 */ 2625 if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) 2626 rctl |= IGC_RCTL_UPE; 2627 2628 /* update state of unicast and multicast */ 2629 rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 2630 wr32(IGC_RCTL, rctl); 2631 2632 #if (PAGE_SIZE < 8192) 2633 if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) 2634 rlpml = IGC_MAX_FRAME_BUILD_SKB; 2635 #endif 2636 wr32(IGC_RLPML, rlpml); 2637 } 2638 2639 /** 2640 * igc_configure - configure the hardware for RX and TX 2641 * @adapter: private board structure 2642 */ 2643 static void igc_configure(struct igc_adapter *adapter) 2644 { 2645 struct net_device *netdev = adapter->netdev; 2646 int i = 0; 2647 2648 igc_get_hw_control(adapter); 2649 igc_set_rx_mode(netdev); 2650 2651 igc_setup_tctl(adapter); 2652 igc_setup_mrqc(adapter); 2653 igc_setup_rctl(adapter); 2654 2655 igc_set_default_mac_filter(adapter); 2656 igc_restore_nfc_rules(adapter); 2657 2658 igc_configure_tx(adapter); 2659 igc_configure_rx(adapter); 2660 2661 igc_rx_fifo_flush_base(&adapter->hw); 2662 2663 /* call igc_desc_unused which always leaves 2664 * at least 1 descriptor unused to make sure 2665 * next_to_use != next_to_clean 2666 */ 2667 for (i = 0; i < adapter->num_rx_queues; i++) { 2668 struct igc_ring *ring = adapter->rx_ring[i]; 2669 2670 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 2671 } 2672 } 2673 2674 /** 2675 * igc_write_ivar - configure ivar for given MSI-X vector 2676 * @hw: pointer to the HW structure 2677 * @msix_vector: vector number we are allocating to a given ring 2678 * @index: row index of IVAR register to write within IVAR table 2679 * @offset: column offset of in IVAR, should be multiple of 8 2680 * 2681 * The IVAR table consists of 2 columns, 2682 * each containing an cause allocation for an Rx and Tx ring, and a 2683 * variable number of rows depending on the number of queues supported. 2684 */ 2685 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 2686 int index, int offset) 2687 { 2688 u32 ivar = array_rd32(IGC_IVAR0, index); 2689 2690 /* clear any bits that are currently set */ 2691 ivar &= ~((u32)0xFF << offset); 2692 2693 /* write vector and valid bit */ 2694 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 2695 2696 array_wr32(IGC_IVAR0, index, ivar); 2697 } 2698 2699 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 2700 { 2701 struct igc_adapter *adapter = q_vector->adapter; 2702 struct igc_hw *hw = &adapter->hw; 2703 int rx_queue = IGC_N0_QUEUE; 2704 int tx_queue = IGC_N0_QUEUE; 2705 2706 if (q_vector->rx.ring) 2707 rx_queue = q_vector->rx.ring->reg_idx; 2708 if (q_vector->tx.ring) 2709 tx_queue = q_vector->tx.ring->reg_idx; 2710 2711 switch (hw->mac.type) { 2712 case igc_i225: 2713 if (rx_queue > IGC_N0_QUEUE) 2714 igc_write_ivar(hw, msix_vector, 2715 rx_queue >> 1, 2716 (rx_queue & 0x1) << 4); 2717 if (tx_queue > IGC_N0_QUEUE) 2718 igc_write_ivar(hw, msix_vector, 2719 tx_queue >> 1, 2720 ((tx_queue & 0x1) << 4) + 8); 2721 q_vector->eims_value = BIT(msix_vector); 2722 break; 2723 default: 2724 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 2725 break; 2726 } 2727 2728 /* add q_vector eims value to global eims_enable_mask */ 2729 adapter->eims_enable_mask |= q_vector->eims_value; 2730 2731 /* configure q_vector to set itr on first interrupt */ 2732 q_vector->set_itr = 1; 2733 } 2734 2735 /** 2736 * igc_configure_msix - Configure MSI-X hardware 2737 * @adapter: Pointer to adapter structure 2738 * 2739 * igc_configure_msix sets up the hardware to properly 2740 * generate MSI-X interrupts. 2741 */ 2742 static void igc_configure_msix(struct igc_adapter *adapter) 2743 { 2744 struct igc_hw *hw = &adapter->hw; 2745 int i, vector = 0; 2746 u32 tmp; 2747 2748 adapter->eims_enable_mask = 0; 2749 2750 /* set vector for other causes, i.e. link changes */ 2751 switch (hw->mac.type) { 2752 case igc_i225: 2753 /* Turn on MSI-X capability first, or our settings 2754 * won't stick. And it will take days to debug. 2755 */ 2756 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 2757 IGC_GPIE_PBA | IGC_GPIE_EIAME | 2758 IGC_GPIE_NSICR); 2759 2760 /* enable msix_other interrupt */ 2761 adapter->eims_other = BIT(vector); 2762 tmp = (vector++ | IGC_IVAR_VALID) << 8; 2763 2764 wr32(IGC_IVAR_MISC, tmp); 2765 break; 2766 default: 2767 /* do nothing, since nothing else supports MSI-X */ 2768 break; 2769 } /* switch (hw->mac.type) */ 2770 2771 adapter->eims_enable_mask |= adapter->eims_other; 2772 2773 for (i = 0; i < adapter->num_q_vectors; i++) 2774 igc_assign_vector(adapter->q_vector[i], vector++); 2775 2776 wrfl(); 2777 } 2778 2779 /** 2780 * igc_irq_enable - Enable default interrupt generation settings 2781 * @adapter: board private structure 2782 */ 2783 static void igc_irq_enable(struct igc_adapter *adapter) 2784 { 2785 struct igc_hw *hw = &adapter->hw; 2786 2787 if (adapter->msix_entries) { 2788 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 2789 u32 regval = rd32(IGC_EIAC); 2790 2791 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 2792 regval = rd32(IGC_EIAM); 2793 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 2794 wr32(IGC_EIMS, adapter->eims_enable_mask); 2795 wr32(IGC_IMS, ims); 2796 } else { 2797 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2798 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2799 } 2800 } 2801 2802 /** 2803 * igc_irq_disable - Mask off interrupt generation on the NIC 2804 * @adapter: board private structure 2805 */ 2806 static void igc_irq_disable(struct igc_adapter *adapter) 2807 { 2808 struct igc_hw *hw = &adapter->hw; 2809 2810 if (adapter->msix_entries) { 2811 u32 regval = rd32(IGC_EIAM); 2812 2813 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 2814 wr32(IGC_EIMC, adapter->eims_enable_mask); 2815 regval = rd32(IGC_EIAC); 2816 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 2817 } 2818 2819 wr32(IGC_IAM, 0); 2820 wr32(IGC_IMC, ~0); 2821 wrfl(); 2822 2823 if (adapter->msix_entries) { 2824 int vector = 0, i; 2825 2826 synchronize_irq(adapter->msix_entries[vector++].vector); 2827 2828 for (i = 0; i < adapter->num_q_vectors; i++) 2829 synchronize_irq(adapter->msix_entries[vector++].vector); 2830 } else { 2831 synchronize_irq(adapter->pdev->irq); 2832 } 2833 } 2834 2835 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 2836 const u32 max_rss_queues) 2837 { 2838 /* Determine if we need to pair queues. */ 2839 /* If rss_queues > half of max_rss_queues, pair the queues in 2840 * order to conserve interrupts due to limited supply. 2841 */ 2842 if (adapter->rss_queues > (max_rss_queues / 2)) 2843 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 2844 else 2845 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 2846 } 2847 2848 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 2849 { 2850 return IGC_MAX_RX_QUEUES; 2851 } 2852 2853 static void igc_init_queue_configuration(struct igc_adapter *adapter) 2854 { 2855 u32 max_rss_queues; 2856 2857 max_rss_queues = igc_get_max_rss_queues(adapter); 2858 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 2859 2860 igc_set_flag_queue_pairs(adapter, max_rss_queues); 2861 } 2862 2863 /** 2864 * igc_reset_q_vector - Reset config for interrupt vector 2865 * @adapter: board private structure to initialize 2866 * @v_idx: Index of vector to be reset 2867 * 2868 * If NAPI is enabled it will delete any references to the 2869 * NAPI struct. This is preparation for igc_free_q_vector. 2870 */ 2871 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 2872 { 2873 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2874 2875 /* if we're coming from igc_set_interrupt_capability, the vectors are 2876 * not yet allocated 2877 */ 2878 if (!q_vector) 2879 return; 2880 2881 if (q_vector->tx.ring) 2882 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 2883 2884 if (q_vector->rx.ring) 2885 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 2886 2887 netif_napi_del(&q_vector->napi); 2888 } 2889 2890 /** 2891 * igc_free_q_vector - Free memory allocated for specific interrupt vector 2892 * @adapter: board private structure to initialize 2893 * @v_idx: Index of vector to be freed 2894 * 2895 * This function frees the memory allocated to the q_vector. 2896 */ 2897 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 2898 { 2899 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2900 2901 adapter->q_vector[v_idx] = NULL; 2902 2903 /* igc_get_stats64() might access the rings on this vector, 2904 * we must wait a grace period before freeing it. 2905 */ 2906 if (q_vector) 2907 kfree_rcu(q_vector, rcu); 2908 } 2909 2910 /** 2911 * igc_free_q_vectors - Free memory allocated for interrupt vectors 2912 * @adapter: board private structure to initialize 2913 * 2914 * This function frees the memory allocated to the q_vectors. In addition if 2915 * NAPI is enabled it will delete any references to the NAPI struct prior 2916 * to freeing the q_vector. 2917 */ 2918 static void igc_free_q_vectors(struct igc_adapter *adapter) 2919 { 2920 int v_idx = adapter->num_q_vectors; 2921 2922 adapter->num_tx_queues = 0; 2923 adapter->num_rx_queues = 0; 2924 adapter->num_q_vectors = 0; 2925 2926 while (v_idx--) { 2927 igc_reset_q_vector(adapter, v_idx); 2928 igc_free_q_vector(adapter, v_idx); 2929 } 2930 } 2931 2932 /** 2933 * igc_update_itr - update the dynamic ITR value based on statistics 2934 * @q_vector: pointer to q_vector 2935 * @ring_container: ring info to update the itr for 2936 * 2937 * Stores a new ITR value based on packets and byte 2938 * counts during the last interrupt. The advantage of per interrupt 2939 * computation is faster updates and more accurate ITR for the current 2940 * traffic pattern. Constants in this function were computed 2941 * based on theoretical maximum wire speed and thresholds were set based 2942 * on testing data as well as attempting to minimize response time 2943 * while increasing bulk throughput. 2944 * NOTE: These calculations are only valid when operating in a single- 2945 * queue environment. 2946 */ 2947 static void igc_update_itr(struct igc_q_vector *q_vector, 2948 struct igc_ring_container *ring_container) 2949 { 2950 unsigned int packets = ring_container->total_packets; 2951 unsigned int bytes = ring_container->total_bytes; 2952 u8 itrval = ring_container->itr; 2953 2954 /* no packets, exit with status unchanged */ 2955 if (packets == 0) 2956 return; 2957 2958 switch (itrval) { 2959 case lowest_latency: 2960 /* handle TSO and jumbo frames */ 2961 if (bytes / packets > 8000) 2962 itrval = bulk_latency; 2963 else if ((packets < 5) && (bytes > 512)) 2964 itrval = low_latency; 2965 break; 2966 case low_latency: /* 50 usec aka 20000 ints/s */ 2967 if (bytes > 10000) { 2968 /* this if handles the TSO accounting */ 2969 if (bytes / packets > 8000) 2970 itrval = bulk_latency; 2971 else if ((packets < 10) || ((bytes / packets) > 1200)) 2972 itrval = bulk_latency; 2973 else if ((packets > 35)) 2974 itrval = lowest_latency; 2975 } else if (bytes / packets > 2000) { 2976 itrval = bulk_latency; 2977 } else if (packets <= 2 && bytes < 512) { 2978 itrval = lowest_latency; 2979 } 2980 break; 2981 case bulk_latency: /* 250 usec aka 4000 ints/s */ 2982 if (bytes > 25000) { 2983 if (packets > 35) 2984 itrval = low_latency; 2985 } else if (bytes < 1500) { 2986 itrval = low_latency; 2987 } 2988 break; 2989 } 2990 2991 /* clear work counters since we have the values we need */ 2992 ring_container->total_bytes = 0; 2993 ring_container->total_packets = 0; 2994 2995 /* write updated itr to ring container */ 2996 ring_container->itr = itrval; 2997 } 2998 2999 static void igc_set_itr(struct igc_q_vector *q_vector) 3000 { 3001 struct igc_adapter *adapter = q_vector->adapter; 3002 u32 new_itr = q_vector->itr_val; 3003 u8 current_itr = 0; 3004 3005 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 3006 switch (adapter->link_speed) { 3007 case SPEED_10: 3008 case SPEED_100: 3009 current_itr = 0; 3010 new_itr = IGC_4K_ITR; 3011 goto set_itr_now; 3012 default: 3013 break; 3014 } 3015 3016 igc_update_itr(q_vector, &q_vector->tx); 3017 igc_update_itr(q_vector, &q_vector->rx); 3018 3019 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 3020 3021 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3022 if (current_itr == lowest_latency && 3023 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3024 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3025 current_itr = low_latency; 3026 3027 switch (current_itr) { 3028 /* counts and packets in update_itr are dependent on these numbers */ 3029 case lowest_latency: 3030 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 3031 break; 3032 case low_latency: 3033 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 3034 break; 3035 case bulk_latency: 3036 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 3037 break; 3038 default: 3039 break; 3040 } 3041 3042 set_itr_now: 3043 if (new_itr != q_vector->itr_val) { 3044 /* this attempts to bias the interrupt rate towards Bulk 3045 * by adding intermediate steps when interrupt rate is 3046 * increasing 3047 */ 3048 new_itr = new_itr > q_vector->itr_val ? 3049 max((new_itr * q_vector->itr_val) / 3050 (new_itr + (q_vector->itr_val >> 2)), 3051 new_itr) : new_itr; 3052 /* Don't write the value here; it resets the adapter's 3053 * internal timer, and causes us to delay far longer than 3054 * we should between interrupts. Instead, we write the ITR 3055 * value at the beginning of the next interrupt so the timing 3056 * ends up being correct. 3057 */ 3058 q_vector->itr_val = new_itr; 3059 q_vector->set_itr = 1; 3060 } 3061 } 3062 3063 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 3064 { 3065 int v_idx = adapter->num_q_vectors; 3066 3067 if (adapter->msix_entries) { 3068 pci_disable_msix(adapter->pdev); 3069 kfree(adapter->msix_entries); 3070 adapter->msix_entries = NULL; 3071 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 3072 pci_disable_msi(adapter->pdev); 3073 } 3074 3075 while (v_idx--) 3076 igc_reset_q_vector(adapter, v_idx); 3077 } 3078 3079 /** 3080 * igc_set_interrupt_capability - set MSI or MSI-X if supported 3081 * @adapter: Pointer to adapter structure 3082 * @msix: boolean value for MSI-X capability 3083 * 3084 * Attempt to configure interrupts using the best available 3085 * capabilities of the hardware and kernel. 3086 */ 3087 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 3088 bool msix) 3089 { 3090 int numvecs, i; 3091 int err; 3092 3093 if (!msix) 3094 goto msi_only; 3095 adapter->flags |= IGC_FLAG_HAS_MSIX; 3096 3097 /* Number of supported queues. */ 3098 adapter->num_rx_queues = adapter->rss_queues; 3099 3100 adapter->num_tx_queues = adapter->rss_queues; 3101 3102 /* start with one vector for every Rx queue */ 3103 numvecs = adapter->num_rx_queues; 3104 3105 /* if Tx handler is separate add 1 for every Tx queue */ 3106 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 3107 numvecs += adapter->num_tx_queues; 3108 3109 /* store the number of vectors reserved for queues */ 3110 adapter->num_q_vectors = numvecs; 3111 3112 /* add 1 vector for link status interrupts */ 3113 numvecs++; 3114 3115 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), 3116 GFP_KERNEL); 3117 3118 if (!adapter->msix_entries) 3119 return; 3120 3121 /* populate entry values */ 3122 for (i = 0; i < numvecs; i++) 3123 adapter->msix_entries[i].entry = i; 3124 3125 err = pci_enable_msix_range(adapter->pdev, 3126 adapter->msix_entries, 3127 numvecs, 3128 numvecs); 3129 if (err > 0) 3130 return; 3131 3132 kfree(adapter->msix_entries); 3133 adapter->msix_entries = NULL; 3134 3135 igc_reset_interrupt_capability(adapter); 3136 3137 msi_only: 3138 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 3139 3140 adapter->rss_queues = 1; 3141 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 3142 adapter->num_rx_queues = 1; 3143 adapter->num_tx_queues = 1; 3144 adapter->num_q_vectors = 1; 3145 if (!pci_enable_msi(adapter->pdev)) 3146 adapter->flags |= IGC_FLAG_HAS_MSI; 3147 } 3148 3149 /** 3150 * igc_update_ring_itr - update the dynamic ITR value based on packet size 3151 * @q_vector: pointer to q_vector 3152 * 3153 * Stores a new ITR value based on strictly on packet size. This 3154 * algorithm is less sophisticated than that used in igc_update_itr, 3155 * due to the difficulty of synchronizing statistics across multiple 3156 * receive rings. The divisors and thresholds used by this function 3157 * were determined based on theoretical maximum wire speed and testing 3158 * data, in order to minimize response time while increasing bulk 3159 * throughput. 3160 * NOTE: This function is called only when operating in a multiqueue 3161 * receive environment. 3162 */ 3163 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 3164 { 3165 struct igc_adapter *adapter = q_vector->adapter; 3166 int new_val = q_vector->itr_val; 3167 int avg_wire_size = 0; 3168 unsigned int packets; 3169 3170 /* For non-gigabit speeds, just fix the interrupt rate at 4000 3171 * ints/sec - ITR timer value of 120 ticks. 3172 */ 3173 switch (adapter->link_speed) { 3174 case SPEED_10: 3175 case SPEED_100: 3176 new_val = IGC_4K_ITR; 3177 goto set_itr_val; 3178 default: 3179 break; 3180 } 3181 3182 packets = q_vector->rx.total_packets; 3183 if (packets) 3184 avg_wire_size = q_vector->rx.total_bytes / packets; 3185 3186 packets = q_vector->tx.total_packets; 3187 if (packets) 3188 avg_wire_size = max_t(u32, avg_wire_size, 3189 q_vector->tx.total_bytes / packets); 3190 3191 /* if avg_wire_size isn't set no work was done */ 3192 if (!avg_wire_size) 3193 goto clear_counts; 3194 3195 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 3196 avg_wire_size += 24; 3197 3198 /* Don't starve jumbo frames */ 3199 avg_wire_size = min(avg_wire_size, 3000); 3200 3201 /* Give a little boost to mid-size frames */ 3202 if (avg_wire_size > 300 && avg_wire_size < 1200) 3203 new_val = avg_wire_size / 3; 3204 else 3205 new_val = avg_wire_size / 2; 3206 3207 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3208 if (new_val < IGC_20K_ITR && 3209 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3210 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3211 new_val = IGC_20K_ITR; 3212 3213 set_itr_val: 3214 if (new_val != q_vector->itr_val) { 3215 q_vector->itr_val = new_val; 3216 q_vector->set_itr = 1; 3217 } 3218 clear_counts: 3219 q_vector->rx.total_bytes = 0; 3220 q_vector->rx.total_packets = 0; 3221 q_vector->tx.total_bytes = 0; 3222 q_vector->tx.total_packets = 0; 3223 } 3224 3225 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 3226 { 3227 struct igc_adapter *adapter = q_vector->adapter; 3228 struct igc_hw *hw = &adapter->hw; 3229 3230 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 3231 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 3232 if (adapter->num_q_vectors == 1) 3233 igc_set_itr(q_vector); 3234 else 3235 igc_update_ring_itr(q_vector); 3236 } 3237 3238 if (!test_bit(__IGC_DOWN, &adapter->state)) { 3239 if (adapter->msix_entries) 3240 wr32(IGC_EIMS, q_vector->eims_value); 3241 else 3242 igc_irq_enable(adapter); 3243 } 3244 } 3245 3246 static void igc_add_ring(struct igc_ring *ring, 3247 struct igc_ring_container *head) 3248 { 3249 head->ring = ring; 3250 head->count++; 3251 } 3252 3253 /** 3254 * igc_cache_ring_register - Descriptor ring to register mapping 3255 * @adapter: board private structure to initialize 3256 * 3257 * Once we know the feature-set enabled for the device, we'll cache 3258 * the register offset the descriptor ring is assigned to. 3259 */ 3260 static void igc_cache_ring_register(struct igc_adapter *adapter) 3261 { 3262 int i = 0, j = 0; 3263 3264 switch (adapter->hw.mac.type) { 3265 case igc_i225: 3266 default: 3267 for (; i < adapter->num_rx_queues; i++) 3268 adapter->rx_ring[i]->reg_idx = i; 3269 for (; j < adapter->num_tx_queues; j++) 3270 adapter->tx_ring[j]->reg_idx = j; 3271 break; 3272 } 3273 } 3274 3275 /** 3276 * igc_poll - NAPI Rx polling callback 3277 * @napi: napi polling structure 3278 * @budget: count of how many packets we should handle 3279 */ 3280 static int igc_poll(struct napi_struct *napi, int budget) 3281 { 3282 struct igc_q_vector *q_vector = container_of(napi, 3283 struct igc_q_vector, 3284 napi); 3285 bool clean_complete = true; 3286 int work_done = 0; 3287 3288 if (q_vector->tx.ring) 3289 clean_complete = igc_clean_tx_irq(q_vector, budget); 3290 3291 if (q_vector->rx.ring) { 3292 int cleaned = igc_clean_rx_irq(q_vector, budget); 3293 3294 work_done += cleaned; 3295 if (cleaned >= budget) 3296 clean_complete = false; 3297 } 3298 3299 /* If all work not completed, return budget and keep polling */ 3300 if (!clean_complete) 3301 return budget; 3302 3303 /* Exit the polling mode, but don't re-enable interrupts if stack might 3304 * poll us due to busy-polling 3305 */ 3306 if (likely(napi_complete_done(napi, work_done))) 3307 igc_ring_irq_enable(q_vector); 3308 3309 return min(work_done, budget - 1); 3310 } 3311 3312 /** 3313 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 3314 * @adapter: board private structure to initialize 3315 * @v_count: q_vectors allocated on adapter, used for ring interleaving 3316 * @v_idx: index of vector in adapter struct 3317 * @txr_count: total number of Tx rings to allocate 3318 * @txr_idx: index of first Tx ring to allocate 3319 * @rxr_count: total number of Rx rings to allocate 3320 * @rxr_idx: index of first Rx ring to allocate 3321 * 3322 * We allocate one q_vector. If allocation fails we return -ENOMEM. 3323 */ 3324 static int igc_alloc_q_vector(struct igc_adapter *adapter, 3325 unsigned int v_count, unsigned int v_idx, 3326 unsigned int txr_count, unsigned int txr_idx, 3327 unsigned int rxr_count, unsigned int rxr_idx) 3328 { 3329 struct igc_q_vector *q_vector; 3330 struct igc_ring *ring; 3331 int ring_count; 3332 3333 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 3334 if (txr_count > 1 || rxr_count > 1) 3335 return -ENOMEM; 3336 3337 ring_count = txr_count + rxr_count; 3338 3339 /* allocate q_vector and rings */ 3340 q_vector = adapter->q_vector[v_idx]; 3341 if (!q_vector) 3342 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), 3343 GFP_KERNEL); 3344 else 3345 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 3346 if (!q_vector) 3347 return -ENOMEM; 3348 3349 /* initialize NAPI */ 3350 netif_napi_add(adapter->netdev, &q_vector->napi, 3351 igc_poll, 64); 3352 3353 /* tie q_vector and adapter together */ 3354 adapter->q_vector[v_idx] = q_vector; 3355 q_vector->adapter = adapter; 3356 3357 /* initialize work limits */ 3358 q_vector->tx.work_limit = adapter->tx_work_limit; 3359 3360 /* initialize ITR configuration */ 3361 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 3362 q_vector->itr_val = IGC_START_ITR; 3363 3364 /* initialize pointer to rings */ 3365 ring = q_vector->ring; 3366 3367 /* initialize ITR */ 3368 if (rxr_count) { 3369 /* rx or rx/tx vector */ 3370 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 3371 q_vector->itr_val = adapter->rx_itr_setting; 3372 } else { 3373 /* tx only vector */ 3374 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 3375 q_vector->itr_val = adapter->tx_itr_setting; 3376 } 3377 3378 if (txr_count) { 3379 /* assign generic ring traits */ 3380 ring->dev = &adapter->pdev->dev; 3381 ring->netdev = adapter->netdev; 3382 3383 /* configure backlink on ring */ 3384 ring->q_vector = q_vector; 3385 3386 /* update q_vector Tx values */ 3387 igc_add_ring(ring, &q_vector->tx); 3388 3389 /* apply Tx specific ring traits */ 3390 ring->count = adapter->tx_ring_count; 3391 ring->queue_index = txr_idx; 3392 3393 /* assign ring to adapter */ 3394 adapter->tx_ring[txr_idx] = ring; 3395 3396 /* push pointer to next ring */ 3397 ring++; 3398 } 3399 3400 if (rxr_count) { 3401 /* assign generic ring traits */ 3402 ring->dev = &adapter->pdev->dev; 3403 ring->netdev = adapter->netdev; 3404 3405 /* configure backlink on ring */ 3406 ring->q_vector = q_vector; 3407 3408 /* update q_vector Rx values */ 3409 igc_add_ring(ring, &q_vector->rx); 3410 3411 /* apply Rx specific ring traits */ 3412 ring->count = adapter->rx_ring_count; 3413 ring->queue_index = rxr_idx; 3414 3415 /* assign ring to adapter */ 3416 adapter->rx_ring[rxr_idx] = ring; 3417 } 3418 3419 return 0; 3420 } 3421 3422 /** 3423 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 3424 * @adapter: board private structure to initialize 3425 * 3426 * We allocate one q_vector per queue interrupt. If allocation fails we 3427 * return -ENOMEM. 3428 */ 3429 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 3430 { 3431 int rxr_remaining = adapter->num_rx_queues; 3432 int txr_remaining = adapter->num_tx_queues; 3433 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 3434 int q_vectors = adapter->num_q_vectors; 3435 int err; 3436 3437 if (q_vectors >= (rxr_remaining + txr_remaining)) { 3438 for (; rxr_remaining; v_idx++) { 3439 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3440 0, 0, 1, rxr_idx); 3441 3442 if (err) 3443 goto err_out; 3444 3445 /* update counts and index */ 3446 rxr_remaining--; 3447 rxr_idx++; 3448 } 3449 } 3450 3451 for (; v_idx < q_vectors; v_idx++) { 3452 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 3453 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 3454 3455 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3456 tqpv, txr_idx, rqpv, rxr_idx); 3457 3458 if (err) 3459 goto err_out; 3460 3461 /* update counts and index */ 3462 rxr_remaining -= rqpv; 3463 txr_remaining -= tqpv; 3464 rxr_idx++; 3465 txr_idx++; 3466 } 3467 3468 return 0; 3469 3470 err_out: 3471 adapter->num_tx_queues = 0; 3472 adapter->num_rx_queues = 0; 3473 adapter->num_q_vectors = 0; 3474 3475 while (v_idx--) 3476 igc_free_q_vector(adapter, v_idx); 3477 3478 return -ENOMEM; 3479 } 3480 3481 /** 3482 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 3483 * @adapter: Pointer to adapter structure 3484 * @msix: boolean for MSI-X capability 3485 * 3486 * This function initializes the interrupts and allocates all of the queues. 3487 */ 3488 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 3489 { 3490 struct net_device *dev = adapter->netdev; 3491 int err = 0; 3492 3493 igc_set_interrupt_capability(adapter, msix); 3494 3495 err = igc_alloc_q_vectors(adapter); 3496 if (err) { 3497 netdev_err(dev, "Unable to allocate memory for vectors\n"); 3498 goto err_alloc_q_vectors; 3499 } 3500 3501 igc_cache_ring_register(adapter); 3502 3503 return 0; 3504 3505 err_alloc_q_vectors: 3506 igc_reset_interrupt_capability(adapter); 3507 return err; 3508 } 3509 3510 /** 3511 * igc_sw_init - Initialize general software structures (struct igc_adapter) 3512 * @adapter: board private structure to initialize 3513 * 3514 * igc_sw_init initializes the Adapter private data structure. 3515 * Fields are initialized based on PCI device information and 3516 * OS network device settings (MTU size). 3517 */ 3518 static int igc_sw_init(struct igc_adapter *adapter) 3519 { 3520 struct net_device *netdev = adapter->netdev; 3521 struct pci_dev *pdev = adapter->pdev; 3522 struct igc_hw *hw = &adapter->hw; 3523 3524 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 3525 3526 /* set default ring sizes */ 3527 adapter->tx_ring_count = IGC_DEFAULT_TXD; 3528 adapter->rx_ring_count = IGC_DEFAULT_RXD; 3529 3530 /* set default ITR values */ 3531 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 3532 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 3533 3534 /* set default work limits */ 3535 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 3536 3537 /* adjust max frame to be at least the size of a standard frame */ 3538 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 3539 VLAN_HLEN; 3540 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 3541 3542 mutex_init(&adapter->nfc_rule_lock); 3543 INIT_LIST_HEAD(&adapter->nfc_rule_list); 3544 adapter->nfc_rule_count = 0; 3545 3546 spin_lock_init(&adapter->stats64_lock); 3547 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 3548 adapter->flags |= IGC_FLAG_HAS_MSIX; 3549 3550 igc_init_queue_configuration(adapter); 3551 3552 /* This call may decrease the number of queues */ 3553 if (igc_init_interrupt_scheme(adapter, true)) { 3554 netdev_err(netdev, "Unable to allocate memory for queues\n"); 3555 return -ENOMEM; 3556 } 3557 3558 /* Explicitly disable IRQ since the NIC can be in any state. */ 3559 igc_irq_disable(adapter); 3560 3561 set_bit(__IGC_DOWN, &adapter->state); 3562 3563 return 0; 3564 } 3565 3566 /** 3567 * igc_up - Open the interface and prepare it to handle traffic 3568 * @adapter: board private structure 3569 */ 3570 void igc_up(struct igc_adapter *adapter) 3571 { 3572 struct igc_hw *hw = &adapter->hw; 3573 int i = 0; 3574 3575 /* hardware has been reset, we need to reload some things */ 3576 igc_configure(adapter); 3577 3578 clear_bit(__IGC_DOWN, &adapter->state); 3579 3580 for (i = 0; i < adapter->num_q_vectors; i++) 3581 napi_enable(&adapter->q_vector[i]->napi); 3582 3583 if (adapter->msix_entries) 3584 igc_configure_msix(adapter); 3585 else 3586 igc_assign_vector(adapter->q_vector[0], 0); 3587 3588 /* Clear any pending interrupts. */ 3589 rd32(IGC_ICR); 3590 igc_irq_enable(adapter); 3591 3592 netif_tx_start_all_queues(adapter->netdev); 3593 3594 /* start the watchdog. */ 3595 hw->mac.get_link_status = 1; 3596 schedule_work(&adapter->watchdog_task); 3597 } 3598 3599 /** 3600 * igc_update_stats - Update the board statistics counters 3601 * @adapter: board private structure 3602 */ 3603 void igc_update_stats(struct igc_adapter *adapter) 3604 { 3605 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 3606 struct pci_dev *pdev = adapter->pdev; 3607 struct igc_hw *hw = &adapter->hw; 3608 u64 _bytes, _packets; 3609 u64 bytes, packets; 3610 unsigned int start; 3611 u32 mpc; 3612 int i; 3613 3614 /* Prevent stats update while adapter is being reset, or if the pci 3615 * connection is down. 3616 */ 3617 if (adapter->link_speed == 0) 3618 return; 3619 if (pci_channel_offline(pdev)) 3620 return; 3621 3622 packets = 0; 3623 bytes = 0; 3624 3625 rcu_read_lock(); 3626 for (i = 0; i < adapter->num_rx_queues; i++) { 3627 struct igc_ring *ring = adapter->rx_ring[i]; 3628 u32 rqdpc = rd32(IGC_RQDPC(i)); 3629 3630 if (hw->mac.type >= igc_i225) 3631 wr32(IGC_RQDPC(i), 0); 3632 3633 if (rqdpc) { 3634 ring->rx_stats.drops += rqdpc; 3635 net_stats->rx_fifo_errors += rqdpc; 3636 } 3637 3638 do { 3639 start = u64_stats_fetch_begin_irq(&ring->rx_syncp); 3640 _bytes = ring->rx_stats.bytes; 3641 _packets = ring->rx_stats.packets; 3642 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); 3643 bytes += _bytes; 3644 packets += _packets; 3645 } 3646 3647 net_stats->rx_bytes = bytes; 3648 net_stats->rx_packets = packets; 3649 3650 packets = 0; 3651 bytes = 0; 3652 for (i = 0; i < adapter->num_tx_queues; i++) { 3653 struct igc_ring *ring = adapter->tx_ring[i]; 3654 3655 do { 3656 start = u64_stats_fetch_begin_irq(&ring->tx_syncp); 3657 _bytes = ring->tx_stats.bytes; 3658 _packets = ring->tx_stats.packets; 3659 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); 3660 bytes += _bytes; 3661 packets += _packets; 3662 } 3663 net_stats->tx_bytes = bytes; 3664 net_stats->tx_packets = packets; 3665 rcu_read_unlock(); 3666 3667 /* read stats registers */ 3668 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 3669 adapter->stats.gprc += rd32(IGC_GPRC); 3670 adapter->stats.gorc += rd32(IGC_GORCL); 3671 rd32(IGC_GORCH); /* clear GORCL */ 3672 adapter->stats.bprc += rd32(IGC_BPRC); 3673 adapter->stats.mprc += rd32(IGC_MPRC); 3674 adapter->stats.roc += rd32(IGC_ROC); 3675 3676 adapter->stats.prc64 += rd32(IGC_PRC64); 3677 adapter->stats.prc127 += rd32(IGC_PRC127); 3678 adapter->stats.prc255 += rd32(IGC_PRC255); 3679 adapter->stats.prc511 += rd32(IGC_PRC511); 3680 adapter->stats.prc1023 += rd32(IGC_PRC1023); 3681 adapter->stats.prc1522 += rd32(IGC_PRC1522); 3682 3683 mpc = rd32(IGC_MPC); 3684 adapter->stats.mpc += mpc; 3685 net_stats->rx_fifo_errors += mpc; 3686 adapter->stats.scc += rd32(IGC_SCC); 3687 adapter->stats.ecol += rd32(IGC_ECOL); 3688 adapter->stats.mcc += rd32(IGC_MCC); 3689 adapter->stats.latecol += rd32(IGC_LATECOL); 3690 adapter->stats.dc += rd32(IGC_DC); 3691 adapter->stats.rlec += rd32(IGC_RLEC); 3692 adapter->stats.xonrxc += rd32(IGC_XONRXC); 3693 adapter->stats.xontxc += rd32(IGC_XONTXC); 3694 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 3695 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 3696 adapter->stats.fcruc += rd32(IGC_FCRUC); 3697 adapter->stats.gptc += rd32(IGC_GPTC); 3698 adapter->stats.gotc += rd32(IGC_GOTCL); 3699 rd32(IGC_GOTCH); /* clear GOTCL */ 3700 adapter->stats.rnbc += rd32(IGC_RNBC); 3701 adapter->stats.ruc += rd32(IGC_RUC); 3702 adapter->stats.rfc += rd32(IGC_RFC); 3703 adapter->stats.rjc += rd32(IGC_RJC); 3704 adapter->stats.tor += rd32(IGC_TORH); 3705 adapter->stats.tot += rd32(IGC_TOTH); 3706 adapter->stats.tpr += rd32(IGC_TPR); 3707 3708 adapter->stats.ptc64 += rd32(IGC_PTC64); 3709 adapter->stats.ptc127 += rd32(IGC_PTC127); 3710 adapter->stats.ptc255 += rd32(IGC_PTC255); 3711 adapter->stats.ptc511 += rd32(IGC_PTC511); 3712 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 3713 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 3714 3715 adapter->stats.mptc += rd32(IGC_MPTC); 3716 adapter->stats.bptc += rd32(IGC_BPTC); 3717 3718 adapter->stats.tpt += rd32(IGC_TPT); 3719 adapter->stats.colc += rd32(IGC_COLC); 3720 adapter->stats.colc += rd32(IGC_RERC); 3721 3722 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 3723 3724 adapter->stats.tsctc += rd32(IGC_TSCTC); 3725 3726 adapter->stats.iac += rd32(IGC_IAC); 3727 3728 /* Fill out the OS statistics structure */ 3729 net_stats->multicast = adapter->stats.mprc; 3730 net_stats->collisions = adapter->stats.colc; 3731 3732 /* Rx Errors */ 3733 3734 /* RLEC on some newer hardware can be incorrect so build 3735 * our own version based on RUC and ROC 3736 */ 3737 net_stats->rx_errors = adapter->stats.rxerrc + 3738 adapter->stats.crcerrs + adapter->stats.algnerrc + 3739 adapter->stats.ruc + adapter->stats.roc + 3740 adapter->stats.cexterr; 3741 net_stats->rx_length_errors = adapter->stats.ruc + 3742 adapter->stats.roc; 3743 net_stats->rx_crc_errors = adapter->stats.crcerrs; 3744 net_stats->rx_frame_errors = adapter->stats.algnerrc; 3745 net_stats->rx_missed_errors = adapter->stats.mpc; 3746 3747 /* Tx Errors */ 3748 net_stats->tx_errors = adapter->stats.ecol + 3749 adapter->stats.latecol; 3750 net_stats->tx_aborted_errors = adapter->stats.ecol; 3751 net_stats->tx_window_errors = adapter->stats.latecol; 3752 net_stats->tx_carrier_errors = adapter->stats.tncrs; 3753 3754 /* Tx Dropped needs to be maintained elsewhere */ 3755 3756 /* Management Stats */ 3757 adapter->stats.mgptc += rd32(IGC_MGTPTC); 3758 adapter->stats.mgprc += rd32(IGC_MGTPRC); 3759 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 3760 } 3761 3762 /** 3763 * igc_down - Close the interface 3764 * @adapter: board private structure 3765 */ 3766 void igc_down(struct igc_adapter *adapter) 3767 { 3768 struct net_device *netdev = adapter->netdev; 3769 struct igc_hw *hw = &adapter->hw; 3770 u32 tctl, rctl; 3771 int i = 0; 3772 3773 set_bit(__IGC_DOWN, &adapter->state); 3774 3775 /* disable receives in the hardware */ 3776 rctl = rd32(IGC_RCTL); 3777 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 3778 /* flush and sleep below */ 3779 3780 /* set trans_start so we don't get spurious watchdogs during reset */ 3781 netif_trans_update(netdev); 3782 3783 netif_carrier_off(netdev); 3784 netif_tx_stop_all_queues(netdev); 3785 3786 /* disable transmits in the hardware */ 3787 tctl = rd32(IGC_TCTL); 3788 tctl &= ~IGC_TCTL_EN; 3789 wr32(IGC_TCTL, tctl); 3790 /* flush both disables and wait for them to finish */ 3791 wrfl(); 3792 usleep_range(10000, 20000); 3793 3794 igc_irq_disable(adapter); 3795 3796 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3797 3798 for (i = 0; i < adapter->num_q_vectors; i++) { 3799 if (adapter->q_vector[i]) { 3800 napi_synchronize(&adapter->q_vector[i]->napi); 3801 napi_disable(&adapter->q_vector[i]->napi); 3802 } 3803 } 3804 3805 del_timer_sync(&adapter->watchdog_timer); 3806 del_timer_sync(&adapter->phy_info_timer); 3807 3808 /* record the stats before reset*/ 3809 spin_lock(&adapter->stats64_lock); 3810 igc_update_stats(adapter); 3811 spin_unlock(&adapter->stats64_lock); 3812 3813 adapter->link_speed = 0; 3814 adapter->link_duplex = 0; 3815 3816 if (!pci_channel_offline(adapter->pdev)) 3817 igc_reset(adapter); 3818 3819 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 3820 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 3821 3822 igc_clean_all_tx_rings(adapter); 3823 igc_clean_all_rx_rings(adapter); 3824 } 3825 3826 void igc_reinit_locked(struct igc_adapter *adapter) 3827 { 3828 WARN_ON(in_interrupt()); 3829 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3830 usleep_range(1000, 2000); 3831 igc_down(adapter); 3832 igc_up(adapter); 3833 clear_bit(__IGC_RESETTING, &adapter->state); 3834 } 3835 3836 static void igc_reset_task(struct work_struct *work) 3837 { 3838 struct igc_adapter *adapter; 3839 3840 adapter = container_of(work, struct igc_adapter, reset_task); 3841 3842 igc_rings_dump(adapter); 3843 igc_regs_dump(adapter); 3844 netdev_err(adapter->netdev, "Reset adapter\n"); 3845 igc_reinit_locked(adapter); 3846 } 3847 3848 /** 3849 * igc_change_mtu - Change the Maximum Transfer Unit 3850 * @netdev: network interface device structure 3851 * @new_mtu: new value for maximum frame size 3852 * 3853 * Returns 0 on success, negative on failure 3854 */ 3855 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 3856 { 3857 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 3858 struct igc_adapter *adapter = netdev_priv(netdev); 3859 3860 /* adjust max frame to be at least the size of a standard frame */ 3861 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 3862 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 3863 3864 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3865 usleep_range(1000, 2000); 3866 3867 /* igc_down has a dependency on max_frame_size */ 3868 adapter->max_frame_size = max_frame; 3869 3870 if (netif_running(netdev)) 3871 igc_down(adapter); 3872 3873 netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 3874 netdev->mtu = new_mtu; 3875 3876 if (netif_running(netdev)) 3877 igc_up(adapter); 3878 else 3879 igc_reset(adapter); 3880 3881 clear_bit(__IGC_RESETTING, &adapter->state); 3882 3883 return 0; 3884 } 3885 3886 /** 3887 * igc_get_stats - Get System Network Statistics 3888 * @netdev: network interface device structure 3889 * 3890 * Returns the address of the device statistics structure. 3891 * The statistics are updated here and also from the timer callback. 3892 */ 3893 static struct net_device_stats *igc_get_stats(struct net_device *netdev) 3894 { 3895 struct igc_adapter *adapter = netdev_priv(netdev); 3896 3897 if (!test_bit(__IGC_RESETTING, &adapter->state)) 3898 igc_update_stats(adapter); 3899 3900 /* only return the current stats */ 3901 return &netdev->stats; 3902 } 3903 3904 static netdev_features_t igc_fix_features(struct net_device *netdev, 3905 netdev_features_t features) 3906 { 3907 /* Since there is no support for separate Rx/Tx vlan accel 3908 * enable/disable make sure Tx flag is always in same state as Rx. 3909 */ 3910 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3911 features |= NETIF_F_HW_VLAN_CTAG_TX; 3912 else 3913 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 3914 3915 return features; 3916 } 3917 3918 static int igc_set_features(struct net_device *netdev, 3919 netdev_features_t features) 3920 { 3921 netdev_features_t changed = netdev->features ^ features; 3922 struct igc_adapter *adapter = netdev_priv(netdev); 3923 3924 /* Add VLAN support */ 3925 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 3926 return 0; 3927 3928 if (!(features & NETIF_F_NTUPLE)) 3929 igc_flush_nfc_rules(adapter); 3930 3931 netdev->features = features; 3932 3933 if (netif_running(netdev)) 3934 igc_reinit_locked(adapter); 3935 else 3936 igc_reset(adapter); 3937 3938 return 1; 3939 } 3940 3941 static netdev_features_t 3942 igc_features_check(struct sk_buff *skb, struct net_device *dev, 3943 netdev_features_t features) 3944 { 3945 unsigned int network_hdr_len, mac_hdr_len; 3946 3947 /* Make certain the headers can be described by a context descriptor */ 3948 mac_hdr_len = skb_network_header(skb) - skb->data; 3949 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 3950 return features & ~(NETIF_F_HW_CSUM | 3951 NETIF_F_SCTP_CRC | 3952 NETIF_F_HW_VLAN_CTAG_TX | 3953 NETIF_F_TSO | 3954 NETIF_F_TSO6); 3955 3956 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 3957 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 3958 return features & ~(NETIF_F_HW_CSUM | 3959 NETIF_F_SCTP_CRC | 3960 NETIF_F_TSO | 3961 NETIF_F_TSO6); 3962 3963 /* We can only support IPv4 TSO in tunnels if we can mangle the 3964 * inner IP ID field, so strip TSO if MANGLEID is not supported. 3965 */ 3966 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 3967 features &= ~NETIF_F_TSO; 3968 3969 return features; 3970 } 3971 3972 static void igc_tsync_interrupt(struct igc_adapter *adapter) 3973 { 3974 struct igc_hw *hw = &adapter->hw; 3975 u32 tsicr = rd32(IGC_TSICR); 3976 u32 ack = 0; 3977 3978 if (tsicr & IGC_TSICR_TXTS) { 3979 /* retrieve hardware timestamp */ 3980 schedule_work(&adapter->ptp_tx_work); 3981 ack |= IGC_TSICR_TXTS; 3982 } 3983 3984 /* acknowledge the interrupts */ 3985 wr32(IGC_TSICR, ack); 3986 } 3987 3988 /** 3989 * igc_msix_other - msix other interrupt handler 3990 * @irq: interrupt number 3991 * @data: pointer to a q_vector 3992 */ 3993 static irqreturn_t igc_msix_other(int irq, void *data) 3994 { 3995 struct igc_adapter *adapter = data; 3996 struct igc_hw *hw = &adapter->hw; 3997 u32 icr = rd32(IGC_ICR); 3998 3999 /* reading ICR causes bit 31 of EICR to be cleared */ 4000 if (icr & IGC_ICR_DRSTA) 4001 schedule_work(&adapter->reset_task); 4002 4003 if (icr & IGC_ICR_DOUTSYNC) { 4004 /* HW is reporting DMA is out of sync */ 4005 adapter->stats.doosync++; 4006 } 4007 4008 if (icr & IGC_ICR_LSC) { 4009 hw->mac.get_link_status = 1; 4010 /* guard against interrupt when we're going down */ 4011 if (!test_bit(__IGC_DOWN, &adapter->state)) 4012 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4013 } 4014 4015 if (icr & IGC_ICR_TS) 4016 igc_tsync_interrupt(adapter); 4017 4018 wr32(IGC_EIMS, adapter->eims_other); 4019 4020 return IRQ_HANDLED; 4021 } 4022 4023 static void igc_write_itr(struct igc_q_vector *q_vector) 4024 { 4025 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 4026 4027 if (!q_vector->set_itr) 4028 return; 4029 4030 if (!itr_val) 4031 itr_val = IGC_ITR_VAL_MASK; 4032 4033 itr_val |= IGC_EITR_CNT_IGNR; 4034 4035 writel(itr_val, q_vector->itr_register); 4036 q_vector->set_itr = 0; 4037 } 4038 4039 static irqreturn_t igc_msix_ring(int irq, void *data) 4040 { 4041 struct igc_q_vector *q_vector = data; 4042 4043 /* Write the ITR value calculated from the previous interrupt. */ 4044 igc_write_itr(q_vector); 4045 4046 napi_schedule(&q_vector->napi); 4047 4048 return IRQ_HANDLED; 4049 } 4050 4051 /** 4052 * igc_request_msix - Initialize MSI-X interrupts 4053 * @adapter: Pointer to adapter structure 4054 * 4055 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 4056 * kernel. 4057 */ 4058 static int igc_request_msix(struct igc_adapter *adapter) 4059 { 4060 int i = 0, err = 0, vector = 0, free_vector = 0; 4061 struct net_device *netdev = adapter->netdev; 4062 4063 err = request_irq(adapter->msix_entries[vector].vector, 4064 &igc_msix_other, 0, netdev->name, adapter); 4065 if (err) 4066 goto err_out; 4067 4068 for (i = 0; i < adapter->num_q_vectors; i++) { 4069 struct igc_q_vector *q_vector = adapter->q_vector[i]; 4070 4071 vector++; 4072 4073 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 4074 4075 if (q_vector->rx.ring && q_vector->tx.ring) 4076 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 4077 q_vector->rx.ring->queue_index); 4078 else if (q_vector->tx.ring) 4079 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 4080 q_vector->tx.ring->queue_index); 4081 else if (q_vector->rx.ring) 4082 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 4083 q_vector->rx.ring->queue_index); 4084 else 4085 sprintf(q_vector->name, "%s-unused", netdev->name); 4086 4087 err = request_irq(adapter->msix_entries[vector].vector, 4088 igc_msix_ring, 0, q_vector->name, 4089 q_vector); 4090 if (err) 4091 goto err_free; 4092 } 4093 4094 igc_configure_msix(adapter); 4095 return 0; 4096 4097 err_free: 4098 /* free already assigned IRQs */ 4099 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 4100 4101 vector--; 4102 for (i = 0; i < vector; i++) { 4103 free_irq(adapter->msix_entries[free_vector++].vector, 4104 adapter->q_vector[i]); 4105 } 4106 err_out: 4107 return err; 4108 } 4109 4110 /** 4111 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 4112 * @adapter: Pointer to adapter structure 4113 * 4114 * This function resets the device so that it has 0 rx queues, tx queues, and 4115 * MSI-X interrupts allocated. 4116 */ 4117 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 4118 { 4119 igc_free_q_vectors(adapter); 4120 igc_reset_interrupt_capability(adapter); 4121 } 4122 4123 /* Need to wait a few seconds after link up to get diagnostic information from 4124 * the phy 4125 */ 4126 static void igc_update_phy_info(struct timer_list *t) 4127 { 4128 struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); 4129 4130 igc_get_phy_info(&adapter->hw); 4131 } 4132 4133 /** 4134 * igc_has_link - check shared code for link and determine up/down 4135 * @adapter: pointer to driver private info 4136 */ 4137 bool igc_has_link(struct igc_adapter *adapter) 4138 { 4139 struct igc_hw *hw = &adapter->hw; 4140 bool link_active = false; 4141 4142 /* get_link_status is set on LSC (link status) interrupt or 4143 * rx sequence error interrupt. get_link_status will stay 4144 * false until the igc_check_for_link establishes link 4145 * for copper adapters ONLY 4146 */ 4147 switch (hw->phy.media_type) { 4148 case igc_media_type_copper: 4149 if (!hw->mac.get_link_status) 4150 return true; 4151 hw->mac.ops.check_for_link(hw); 4152 link_active = !hw->mac.get_link_status; 4153 break; 4154 default: 4155 case igc_media_type_unknown: 4156 break; 4157 } 4158 4159 if (hw->mac.type == igc_i225 && 4160 hw->phy.id == I225_I_PHY_ID) { 4161 if (!netif_carrier_ok(adapter->netdev)) { 4162 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4163 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 4164 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 4165 adapter->link_check_timeout = jiffies; 4166 } 4167 } 4168 4169 return link_active; 4170 } 4171 4172 /** 4173 * igc_watchdog - Timer Call-back 4174 * @t: timer for the watchdog 4175 */ 4176 static void igc_watchdog(struct timer_list *t) 4177 { 4178 struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); 4179 /* Do the rest outside of interrupt context */ 4180 schedule_work(&adapter->watchdog_task); 4181 } 4182 4183 static void igc_watchdog_task(struct work_struct *work) 4184 { 4185 struct igc_adapter *adapter = container_of(work, 4186 struct igc_adapter, 4187 watchdog_task); 4188 struct net_device *netdev = adapter->netdev; 4189 struct igc_hw *hw = &adapter->hw; 4190 struct igc_phy_info *phy = &hw->phy; 4191 u16 phy_data, retry_count = 20; 4192 u32 link; 4193 int i; 4194 4195 link = igc_has_link(adapter); 4196 4197 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 4198 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 4199 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4200 else 4201 link = false; 4202 } 4203 4204 if (link) { 4205 /* Cancel scheduled suspend requests. */ 4206 pm_runtime_resume(netdev->dev.parent); 4207 4208 if (!netif_carrier_ok(netdev)) { 4209 u32 ctrl; 4210 4211 hw->mac.ops.get_speed_and_duplex(hw, 4212 &adapter->link_speed, 4213 &adapter->link_duplex); 4214 4215 ctrl = rd32(IGC_CTRL); 4216 /* Link status message must follow this format */ 4217 netdev_info(netdev, 4218 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 4219 adapter->link_speed, 4220 adapter->link_duplex == FULL_DUPLEX ? 4221 "Full" : "Half", 4222 (ctrl & IGC_CTRL_TFCE) && 4223 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 4224 (ctrl & IGC_CTRL_RFCE) ? "RX" : 4225 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 4226 4227 /* disable EEE if enabled */ 4228 if ((adapter->flags & IGC_FLAG_EEE) && 4229 adapter->link_duplex == HALF_DUPLEX) { 4230 netdev_info(netdev, 4231 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n"); 4232 adapter->hw.dev_spec._base.eee_enable = false; 4233 adapter->flags &= ~IGC_FLAG_EEE; 4234 } 4235 4236 /* check if SmartSpeed worked */ 4237 igc_check_downshift(hw); 4238 if (phy->speed_downgraded) 4239 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 4240 4241 /* adjust timeout factor according to speed/duplex */ 4242 adapter->tx_timeout_factor = 1; 4243 switch (adapter->link_speed) { 4244 case SPEED_10: 4245 adapter->tx_timeout_factor = 14; 4246 break; 4247 case SPEED_100: 4248 /* maybe add some timeout factor ? */ 4249 break; 4250 } 4251 4252 if (adapter->link_speed != SPEED_1000) 4253 goto no_wait; 4254 4255 /* wait for Remote receiver status OK */ 4256 retry_read_status: 4257 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 4258 &phy_data)) { 4259 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 4260 retry_count) { 4261 msleep(100); 4262 retry_count--; 4263 goto retry_read_status; 4264 } else if (!retry_count) { 4265 netdev_err(netdev, "exceed max 2 second\n"); 4266 } 4267 } else { 4268 netdev_err(netdev, "read 1000Base-T Status Reg\n"); 4269 } 4270 no_wait: 4271 netif_carrier_on(netdev); 4272 4273 /* link state has changed, schedule phy info update */ 4274 if (!test_bit(__IGC_DOWN, &adapter->state)) 4275 mod_timer(&adapter->phy_info_timer, 4276 round_jiffies(jiffies + 2 * HZ)); 4277 } 4278 } else { 4279 if (netif_carrier_ok(netdev)) { 4280 adapter->link_speed = 0; 4281 adapter->link_duplex = 0; 4282 4283 /* Links status message must follow this format */ 4284 netdev_info(netdev, "NIC Link is Down\n"); 4285 netif_carrier_off(netdev); 4286 4287 /* link state has changed, schedule phy info update */ 4288 if (!test_bit(__IGC_DOWN, &adapter->state)) 4289 mod_timer(&adapter->phy_info_timer, 4290 round_jiffies(jiffies + 2 * HZ)); 4291 4292 /* link is down, time to check for alternate media */ 4293 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 4294 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4295 schedule_work(&adapter->reset_task); 4296 /* return immediately */ 4297 return; 4298 } 4299 } 4300 pm_schedule_suspend(netdev->dev.parent, 4301 MSEC_PER_SEC * 5); 4302 4303 /* also check for alternate media here */ 4304 } else if (!netif_carrier_ok(netdev) && 4305 (adapter->flags & IGC_FLAG_MAS_ENABLE)) { 4306 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4307 schedule_work(&adapter->reset_task); 4308 /* return immediately */ 4309 return; 4310 } 4311 } 4312 } 4313 4314 spin_lock(&adapter->stats64_lock); 4315 igc_update_stats(adapter); 4316 spin_unlock(&adapter->stats64_lock); 4317 4318 for (i = 0; i < adapter->num_tx_queues; i++) { 4319 struct igc_ring *tx_ring = adapter->tx_ring[i]; 4320 4321 if (!netif_carrier_ok(netdev)) { 4322 /* We've lost link, so the controller stops DMA, 4323 * but we've got queued Tx work that's never going 4324 * to get done, so reset controller to flush Tx. 4325 * (Do the reset outside of interrupt context). 4326 */ 4327 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 4328 adapter->tx_timeout_count++; 4329 schedule_work(&adapter->reset_task); 4330 /* return immediately since reset is imminent */ 4331 return; 4332 } 4333 } 4334 4335 /* Force detection of hung controller every watchdog period */ 4336 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 4337 } 4338 4339 /* Cause software interrupt to ensure Rx ring is cleaned */ 4340 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4341 u32 eics = 0; 4342 4343 for (i = 0; i < adapter->num_q_vectors; i++) 4344 eics |= adapter->q_vector[i]->eims_value; 4345 wr32(IGC_EICS, eics); 4346 } else { 4347 wr32(IGC_ICS, IGC_ICS_RXDMT0); 4348 } 4349 4350 igc_ptp_tx_hang(adapter); 4351 4352 /* Reset the timer */ 4353 if (!test_bit(__IGC_DOWN, &adapter->state)) { 4354 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 4355 mod_timer(&adapter->watchdog_timer, 4356 round_jiffies(jiffies + HZ)); 4357 else 4358 mod_timer(&adapter->watchdog_timer, 4359 round_jiffies(jiffies + 2 * HZ)); 4360 } 4361 } 4362 4363 /** 4364 * igc_intr_msi - Interrupt Handler 4365 * @irq: interrupt number 4366 * @data: pointer to a network interface device structure 4367 */ 4368 static irqreturn_t igc_intr_msi(int irq, void *data) 4369 { 4370 struct igc_adapter *adapter = data; 4371 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4372 struct igc_hw *hw = &adapter->hw; 4373 /* read ICR disables interrupts using IAM */ 4374 u32 icr = rd32(IGC_ICR); 4375 4376 igc_write_itr(q_vector); 4377 4378 if (icr & IGC_ICR_DRSTA) 4379 schedule_work(&adapter->reset_task); 4380 4381 if (icr & IGC_ICR_DOUTSYNC) { 4382 /* HW is reporting DMA is out of sync */ 4383 adapter->stats.doosync++; 4384 } 4385 4386 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4387 hw->mac.get_link_status = 1; 4388 if (!test_bit(__IGC_DOWN, &adapter->state)) 4389 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4390 } 4391 4392 napi_schedule(&q_vector->napi); 4393 4394 return IRQ_HANDLED; 4395 } 4396 4397 /** 4398 * igc_intr - Legacy Interrupt Handler 4399 * @irq: interrupt number 4400 * @data: pointer to a network interface device structure 4401 */ 4402 static irqreturn_t igc_intr(int irq, void *data) 4403 { 4404 struct igc_adapter *adapter = data; 4405 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4406 struct igc_hw *hw = &adapter->hw; 4407 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 4408 * need for the IMC write 4409 */ 4410 u32 icr = rd32(IGC_ICR); 4411 4412 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 4413 * not set, then the adapter didn't send an interrupt 4414 */ 4415 if (!(icr & IGC_ICR_INT_ASSERTED)) 4416 return IRQ_NONE; 4417 4418 igc_write_itr(q_vector); 4419 4420 if (icr & IGC_ICR_DRSTA) 4421 schedule_work(&adapter->reset_task); 4422 4423 if (icr & IGC_ICR_DOUTSYNC) { 4424 /* HW is reporting DMA is out of sync */ 4425 adapter->stats.doosync++; 4426 } 4427 4428 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4429 hw->mac.get_link_status = 1; 4430 /* guard against interrupt when we're going down */ 4431 if (!test_bit(__IGC_DOWN, &adapter->state)) 4432 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4433 } 4434 4435 napi_schedule(&q_vector->napi); 4436 4437 return IRQ_HANDLED; 4438 } 4439 4440 static void igc_free_irq(struct igc_adapter *adapter) 4441 { 4442 if (adapter->msix_entries) { 4443 int vector = 0, i; 4444 4445 free_irq(adapter->msix_entries[vector++].vector, adapter); 4446 4447 for (i = 0; i < adapter->num_q_vectors; i++) 4448 free_irq(adapter->msix_entries[vector++].vector, 4449 adapter->q_vector[i]); 4450 } else { 4451 free_irq(adapter->pdev->irq, adapter); 4452 } 4453 } 4454 4455 /** 4456 * igc_request_irq - initialize interrupts 4457 * @adapter: Pointer to adapter structure 4458 * 4459 * Attempts to configure interrupts using the best available 4460 * capabilities of the hardware and kernel. 4461 */ 4462 static int igc_request_irq(struct igc_adapter *adapter) 4463 { 4464 struct net_device *netdev = adapter->netdev; 4465 struct pci_dev *pdev = adapter->pdev; 4466 int err = 0; 4467 4468 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4469 err = igc_request_msix(adapter); 4470 if (!err) 4471 goto request_done; 4472 /* fall back to MSI */ 4473 igc_free_all_tx_resources(adapter); 4474 igc_free_all_rx_resources(adapter); 4475 4476 igc_clear_interrupt_scheme(adapter); 4477 err = igc_init_interrupt_scheme(adapter, false); 4478 if (err) 4479 goto request_done; 4480 igc_setup_all_tx_resources(adapter); 4481 igc_setup_all_rx_resources(adapter); 4482 igc_configure(adapter); 4483 } 4484 4485 igc_assign_vector(adapter->q_vector[0], 0); 4486 4487 if (adapter->flags & IGC_FLAG_HAS_MSI) { 4488 err = request_irq(pdev->irq, &igc_intr_msi, 0, 4489 netdev->name, adapter); 4490 if (!err) 4491 goto request_done; 4492 4493 /* fall back to legacy interrupts */ 4494 igc_reset_interrupt_capability(adapter); 4495 adapter->flags &= ~IGC_FLAG_HAS_MSI; 4496 } 4497 4498 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 4499 netdev->name, adapter); 4500 4501 if (err) 4502 netdev_err(netdev, "Error %d getting interrupt\n", err); 4503 4504 request_done: 4505 return err; 4506 } 4507 4508 /** 4509 * __igc_open - Called when a network interface is made active 4510 * @netdev: network interface device structure 4511 * @resuming: boolean indicating if the device is resuming 4512 * 4513 * Returns 0 on success, negative value on failure 4514 * 4515 * The open entry point is called when a network interface is made 4516 * active by the system (IFF_UP). At this point all resources needed 4517 * for transmit and receive operations are allocated, the interrupt 4518 * handler is registered with the OS, the watchdog timer is started, 4519 * and the stack is notified that the interface is ready. 4520 */ 4521 static int __igc_open(struct net_device *netdev, bool resuming) 4522 { 4523 struct igc_adapter *adapter = netdev_priv(netdev); 4524 struct pci_dev *pdev = adapter->pdev; 4525 struct igc_hw *hw = &adapter->hw; 4526 int err = 0; 4527 int i = 0; 4528 4529 /* disallow open during test */ 4530 4531 if (test_bit(__IGC_TESTING, &adapter->state)) { 4532 WARN_ON(resuming); 4533 return -EBUSY; 4534 } 4535 4536 if (!resuming) 4537 pm_runtime_get_sync(&pdev->dev); 4538 4539 netif_carrier_off(netdev); 4540 4541 /* allocate transmit descriptors */ 4542 err = igc_setup_all_tx_resources(adapter); 4543 if (err) 4544 goto err_setup_tx; 4545 4546 /* allocate receive descriptors */ 4547 err = igc_setup_all_rx_resources(adapter); 4548 if (err) 4549 goto err_setup_rx; 4550 4551 igc_power_up_link(adapter); 4552 4553 igc_configure(adapter); 4554 4555 err = igc_request_irq(adapter); 4556 if (err) 4557 goto err_req_irq; 4558 4559 /* Notify the stack of the actual queue counts. */ 4560 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 4561 if (err) 4562 goto err_set_queues; 4563 4564 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 4565 if (err) 4566 goto err_set_queues; 4567 4568 clear_bit(__IGC_DOWN, &adapter->state); 4569 4570 for (i = 0; i < adapter->num_q_vectors; i++) 4571 napi_enable(&adapter->q_vector[i]->napi); 4572 4573 /* Clear any pending interrupts. */ 4574 rd32(IGC_ICR); 4575 igc_irq_enable(adapter); 4576 4577 if (!resuming) 4578 pm_runtime_put(&pdev->dev); 4579 4580 netif_tx_start_all_queues(netdev); 4581 4582 /* start the watchdog. */ 4583 hw->mac.get_link_status = 1; 4584 schedule_work(&adapter->watchdog_task); 4585 4586 return IGC_SUCCESS; 4587 4588 err_set_queues: 4589 igc_free_irq(adapter); 4590 err_req_irq: 4591 igc_release_hw_control(adapter); 4592 igc_power_down_phy_copper_base(&adapter->hw); 4593 igc_free_all_rx_resources(adapter); 4594 err_setup_rx: 4595 igc_free_all_tx_resources(adapter); 4596 err_setup_tx: 4597 igc_reset(adapter); 4598 if (!resuming) 4599 pm_runtime_put(&pdev->dev); 4600 4601 return err; 4602 } 4603 4604 int igc_open(struct net_device *netdev) 4605 { 4606 return __igc_open(netdev, false); 4607 } 4608 4609 /** 4610 * __igc_close - Disables a network interface 4611 * @netdev: network interface device structure 4612 * @suspending: boolean indicating the device is suspending 4613 * 4614 * Returns 0, this is not allowed to fail 4615 * 4616 * The close entry point is called when an interface is de-activated 4617 * by the OS. The hardware is still under the driver's control, but 4618 * needs to be disabled. A global MAC reset is issued to stop the 4619 * hardware, and all transmit and receive resources are freed. 4620 */ 4621 static int __igc_close(struct net_device *netdev, bool suspending) 4622 { 4623 struct igc_adapter *adapter = netdev_priv(netdev); 4624 struct pci_dev *pdev = adapter->pdev; 4625 4626 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 4627 4628 if (!suspending) 4629 pm_runtime_get_sync(&pdev->dev); 4630 4631 igc_down(adapter); 4632 4633 igc_release_hw_control(adapter); 4634 4635 igc_free_irq(adapter); 4636 4637 igc_free_all_tx_resources(adapter); 4638 igc_free_all_rx_resources(adapter); 4639 4640 if (!suspending) 4641 pm_runtime_put_sync(&pdev->dev); 4642 4643 return 0; 4644 } 4645 4646 int igc_close(struct net_device *netdev) 4647 { 4648 if (netif_device_present(netdev) || netdev->dismantle) 4649 return __igc_close(netdev, false); 4650 return 0; 4651 } 4652 4653 /** 4654 * igc_ioctl - Access the hwtstamp interface 4655 * @netdev: network interface device structure 4656 * @ifreq: interface request data 4657 * @cmd: ioctl command 4658 **/ 4659 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 4660 { 4661 switch (cmd) { 4662 case SIOCGHWTSTAMP: 4663 return igc_ptp_get_ts_config(netdev, ifr); 4664 case SIOCSHWTSTAMP: 4665 return igc_ptp_set_ts_config(netdev, ifr); 4666 default: 4667 return -EOPNOTSUPP; 4668 } 4669 } 4670 4671 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, 4672 bool enable) 4673 { 4674 struct igc_ring *ring; 4675 int i; 4676 4677 if (queue < 0 || queue >= adapter->num_tx_queues) 4678 return -EINVAL; 4679 4680 ring = adapter->tx_ring[queue]; 4681 ring->launchtime_enable = enable; 4682 4683 if (adapter->base_time) 4684 return 0; 4685 4686 adapter->cycle_time = NSEC_PER_SEC; 4687 4688 for (i = 0; i < adapter->num_tx_queues; i++) { 4689 ring = adapter->tx_ring[i]; 4690 ring->start_time = 0; 4691 ring->end_time = NSEC_PER_SEC; 4692 } 4693 4694 return 0; 4695 } 4696 4697 static bool validate_schedule(const struct tc_taprio_qopt_offload *qopt) 4698 { 4699 int queue_uses[IGC_MAX_TX_QUEUES] = { }; 4700 size_t n; 4701 4702 if (qopt->cycle_time_extension) 4703 return false; 4704 4705 for (n = 0; n < qopt->num_entries; n++) { 4706 const struct tc_taprio_sched_entry *e; 4707 int i; 4708 4709 e = &qopt->entries[n]; 4710 4711 /* i225 only supports "global" frame preemption 4712 * settings. 4713 */ 4714 if (e->command != TC_TAPRIO_CMD_SET_GATES) 4715 return false; 4716 4717 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4718 if (e->gate_mask & BIT(i)) 4719 queue_uses[i]++; 4720 4721 if (queue_uses[i] > 1) 4722 return false; 4723 } 4724 } 4725 4726 return true; 4727 } 4728 4729 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, 4730 struct tc_etf_qopt_offload *qopt) 4731 { 4732 struct igc_hw *hw = &adapter->hw; 4733 int err; 4734 4735 if (hw->mac.type != igc_i225) 4736 return -EOPNOTSUPP; 4737 4738 err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable); 4739 if (err) 4740 return err; 4741 4742 return igc_tsn_offload_apply(adapter); 4743 } 4744 4745 static int igc_save_qbv_schedule(struct igc_adapter *adapter, 4746 struct tc_taprio_qopt_offload *qopt) 4747 { 4748 u32 start_time = 0, end_time = 0; 4749 size_t n; 4750 4751 if (!qopt->enable) { 4752 adapter->base_time = 0; 4753 return 0; 4754 } 4755 4756 if (adapter->base_time) 4757 return -EALREADY; 4758 4759 if (!validate_schedule(qopt)) 4760 return -EINVAL; 4761 4762 adapter->cycle_time = qopt->cycle_time; 4763 adapter->base_time = qopt->base_time; 4764 4765 /* FIXME: be a little smarter about cases when the gate for a 4766 * queue stays open for more than one entry. 4767 */ 4768 for (n = 0; n < qopt->num_entries; n++) { 4769 struct tc_taprio_sched_entry *e = &qopt->entries[n]; 4770 int i; 4771 4772 end_time += e->interval; 4773 4774 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4775 struct igc_ring *ring = adapter->tx_ring[i]; 4776 4777 if (!(e->gate_mask & BIT(i))) 4778 continue; 4779 4780 ring->start_time = start_time; 4781 ring->end_time = end_time; 4782 } 4783 4784 start_time += e->interval; 4785 } 4786 4787 return 0; 4788 } 4789 4790 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, 4791 struct tc_taprio_qopt_offload *qopt) 4792 { 4793 struct igc_hw *hw = &adapter->hw; 4794 int err; 4795 4796 if (hw->mac.type != igc_i225) 4797 return -EOPNOTSUPP; 4798 4799 err = igc_save_qbv_schedule(adapter, qopt); 4800 if (err) 4801 return err; 4802 4803 return igc_tsn_offload_apply(adapter); 4804 } 4805 4806 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, 4807 void *type_data) 4808 { 4809 struct igc_adapter *adapter = netdev_priv(dev); 4810 4811 switch (type) { 4812 case TC_SETUP_QDISC_TAPRIO: 4813 return igc_tsn_enable_qbv_scheduling(adapter, type_data); 4814 4815 case TC_SETUP_QDISC_ETF: 4816 return igc_tsn_enable_launchtime(adapter, type_data); 4817 4818 default: 4819 return -EOPNOTSUPP; 4820 } 4821 } 4822 4823 static const struct net_device_ops igc_netdev_ops = { 4824 .ndo_open = igc_open, 4825 .ndo_stop = igc_close, 4826 .ndo_start_xmit = igc_xmit_frame, 4827 .ndo_set_rx_mode = igc_set_rx_mode, 4828 .ndo_set_mac_address = igc_set_mac, 4829 .ndo_change_mtu = igc_change_mtu, 4830 .ndo_get_stats = igc_get_stats, 4831 .ndo_fix_features = igc_fix_features, 4832 .ndo_set_features = igc_set_features, 4833 .ndo_features_check = igc_features_check, 4834 .ndo_do_ioctl = igc_ioctl, 4835 .ndo_setup_tc = igc_setup_tc, 4836 }; 4837 4838 /* PCIe configuration access */ 4839 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4840 { 4841 struct igc_adapter *adapter = hw->back; 4842 4843 pci_read_config_word(adapter->pdev, reg, value); 4844 } 4845 4846 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4847 { 4848 struct igc_adapter *adapter = hw->back; 4849 4850 pci_write_config_word(adapter->pdev, reg, *value); 4851 } 4852 4853 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4854 { 4855 struct igc_adapter *adapter = hw->back; 4856 4857 if (!pci_is_pcie(adapter->pdev)) 4858 return -IGC_ERR_CONFIG; 4859 4860 pcie_capability_read_word(adapter->pdev, reg, value); 4861 4862 return IGC_SUCCESS; 4863 } 4864 4865 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4866 { 4867 struct igc_adapter *adapter = hw->back; 4868 4869 if (!pci_is_pcie(adapter->pdev)) 4870 return -IGC_ERR_CONFIG; 4871 4872 pcie_capability_write_word(adapter->pdev, reg, *value); 4873 4874 return IGC_SUCCESS; 4875 } 4876 4877 u32 igc_rd32(struct igc_hw *hw, u32 reg) 4878 { 4879 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 4880 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 4881 u32 value = 0; 4882 4883 value = readl(&hw_addr[reg]); 4884 4885 /* reads should not return all F's */ 4886 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 4887 struct net_device *netdev = igc->netdev; 4888 4889 hw->hw_addr = NULL; 4890 netif_device_detach(netdev); 4891 netdev_err(netdev, "PCIe link lost, device now detached\n"); 4892 WARN(pci_device_is_present(igc->pdev), 4893 "igc: Failed to read reg 0x%x!\n", reg); 4894 } 4895 4896 return value; 4897 } 4898 4899 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) 4900 { 4901 struct igc_mac_info *mac = &adapter->hw.mac; 4902 4903 mac->autoneg = 0; 4904 4905 /* Make sure dplx is at most 1 bit and lsb of speed is not set 4906 * for the switch() below to work 4907 */ 4908 if ((spd & 1) || (dplx & ~1)) 4909 goto err_inval; 4910 4911 switch (spd + dplx) { 4912 case SPEED_10 + DUPLEX_HALF: 4913 mac->forced_speed_duplex = ADVERTISE_10_HALF; 4914 break; 4915 case SPEED_10 + DUPLEX_FULL: 4916 mac->forced_speed_duplex = ADVERTISE_10_FULL; 4917 break; 4918 case SPEED_100 + DUPLEX_HALF: 4919 mac->forced_speed_duplex = ADVERTISE_100_HALF; 4920 break; 4921 case SPEED_100 + DUPLEX_FULL: 4922 mac->forced_speed_duplex = ADVERTISE_100_FULL; 4923 break; 4924 case SPEED_1000 + DUPLEX_FULL: 4925 mac->autoneg = 1; 4926 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 4927 break; 4928 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 4929 goto err_inval; 4930 case SPEED_2500 + DUPLEX_FULL: 4931 mac->autoneg = 1; 4932 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 4933 break; 4934 case SPEED_2500 + DUPLEX_HALF: /* not supported */ 4935 default: 4936 goto err_inval; 4937 } 4938 4939 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 4940 adapter->hw.phy.mdix = AUTO_ALL_MODES; 4941 4942 return 0; 4943 4944 err_inval: 4945 netdev_err(adapter->netdev, "Unsupported Speed/Duplex configuration\n"); 4946 return -EINVAL; 4947 } 4948 4949 /** 4950 * igc_probe - Device Initialization Routine 4951 * @pdev: PCI device information struct 4952 * @ent: entry in igc_pci_tbl 4953 * 4954 * Returns 0 on success, negative on failure 4955 * 4956 * igc_probe initializes an adapter identified by a pci_dev structure. 4957 * The OS initialization, configuring the adapter private structure, 4958 * and a hardware reset occur. 4959 */ 4960 static int igc_probe(struct pci_dev *pdev, 4961 const struct pci_device_id *ent) 4962 { 4963 struct igc_adapter *adapter; 4964 struct net_device *netdev; 4965 struct igc_hw *hw; 4966 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 4967 int err, pci_using_dac; 4968 4969 err = pci_enable_device_mem(pdev); 4970 if (err) 4971 return err; 4972 4973 pci_using_dac = 0; 4974 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4975 if (!err) { 4976 pci_using_dac = 1; 4977 } else { 4978 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 4979 if (err) { 4980 dev_err(&pdev->dev, 4981 "No usable DMA configuration, aborting\n"); 4982 goto err_dma; 4983 } 4984 } 4985 4986 err = pci_request_mem_regions(pdev, igc_driver_name); 4987 if (err) 4988 goto err_pci_reg; 4989 4990 pci_enable_pcie_error_reporting(pdev); 4991 4992 pci_set_master(pdev); 4993 4994 err = -ENOMEM; 4995 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 4996 IGC_MAX_TX_QUEUES); 4997 4998 if (!netdev) 4999 goto err_alloc_etherdev; 5000 5001 SET_NETDEV_DEV(netdev, &pdev->dev); 5002 5003 pci_set_drvdata(pdev, netdev); 5004 adapter = netdev_priv(netdev); 5005 adapter->netdev = netdev; 5006 adapter->pdev = pdev; 5007 hw = &adapter->hw; 5008 hw->back = adapter; 5009 adapter->port_num = hw->bus.func; 5010 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 5011 5012 err = pci_save_state(pdev); 5013 if (err) 5014 goto err_ioremap; 5015 5016 err = -EIO; 5017 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 5018 pci_resource_len(pdev, 0)); 5019 if (!adapter->io_addr) 5020 goto err_ioremap; 5021 5022 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 5023 hw->hw_addr = adapter->io_addr; 5024 5025 netdev->netdev_ops = &igc_netdev_ops; 5026 igc_ethtool_set_ops(netdev); 5027 netdev->watchdog_timeo = 5 * HZ; 5028 5029 netdev->mem_start = pci_resource_start(pdev, 0); 5030 netdev->mem_end = pci_resource_end(pdev, 0); 5031 5032 /* PCI config space info */ 5033 hw->vendor_id = pdev->vendor; 5034 hw->device_id = pdev->device; 5035 hw->revision_id = pdev->revision; 5036 hw->subsystem_vendor_id = pdev->subsystem_vendor; 5037 hw->subsystem_device_id = pdev->subsystem_device; 5038 5039 /* Copy the default MAC and PHY function pointers */ 5040 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 5041 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 5042 5043 /* Initialize skew-specific constants */ 5044 err = ei->get_invariants(hw); 5045 if (err) 5046 goto err_sw_init; 5047 5048 /* Add supported features to the features list*/ 5049 netdev->features |= NETIF_F_SG; 5050 netdev->features |= NETIF_F_TSO; 5051 netdev->features |= NETIF_F_TSO6; 5052 netdev->features |= NETIF_F_TSO_ECN; 5053 netdev->features |= NETIF_F_RXCSUM; 5054 netdev->features |= NETIF_F_HW_CSUM; 5055 netdev->features |= NETIF_F_SCTP_CRC; 5056 netdev->features |= NETIF_F_HW_TC; 5057 5058 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 5059 NETIF_F_GSO_GRE_CSUM | \ 5060 NETIF_F_GSO_IPXIP4 | \ 5061 NETIF_F_GSO_IPXIP6 | \ 5062 NETIF_F_GSO_UDP_TUNNEL | \ 5063 NETIF_F_GSO_UDP_TUNNEL_CSUM) 5064 5065 netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; 5066 netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; 5067 5068 /* setup the private structure */ 5069 err = igc_sw_init(adapter); 5070 if (err) 5071 goto err_sw_init; 5072 5073 /* copy netdev features into list of user selectable features */ 5074 netdev->hw_features |= NETIF_F_NTUPLE; 5075 netdev->hw_features |= netdev->features; 5076 5077 if (pci_using_dac) 5078 netdev->features |= NETIF_F_HIGHDMA; 5079 5080 /* MTU range: 68 - 9216 */ 5081 netdev->min_mtu = ETH_MIN_MTU; 5082 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 5083 5084 /* before reading the NVM, reset the controller to put the device in a 5085 * known good starting state 5086 */ 5087 hw->mac.ops.reset_hw(hw); 5088 5089 if (igc_get_flash_presence_i225(hw)) { 5090 if (hw->nvm.ops.validate(hw) < 0) { 5091 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 5092 err = -EIO; 5093 goto err_eeprom; 5094 } 5095 } 5096 5097 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 5098 /* copy the MAC address out of the NVM */ 5099 if (hw->mac.ops.read_mac_addr(hw)) 5100 dev_err(&pdev->dev, "NVM Read Error\n"); 5101 } 5102 5103 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 5104 5105 if (!is_valid_ether_addr(netdev->dev_addr)) { 5106 dev_err(&pdev->dev, "Invalid MAC Address\n"); 5107 err = -EIO; 5108 goto err_eeprom; 5109 } 5110 5111 /* configure RXPBSIZE and TXPBSIZE */ 5112 wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT); 5113 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); 5114 5115 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 5116 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 5117 5118 INIT_WORK(&adapter->reset_task, igc_reset_task); 5119 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 5120 5121 /* Initialize link properties that are user-changeable */ 5122 adapter->fc_autoneg = true; 5123 hw->mac.autoneg = true; 5124 hw->phy.autoneg_advertised = 0xaf; 5125 5126 hw->fc.requested_mode = igc_fc_default; 5127 hw->fc.current_mode = igc_fc_default; 5128 5129 /* By default, support wake on port A */ 5130 adapter->flags |= IGC_FLAG_WOL_SUPPORTED; 5131 5132 /* initialize the wol settings based on the eeprom settings */ 5133 if (adapter->flags & IGC_FLAG_WOL_SUPPORTED) 5134 adapter->wol |= IGC_WUFC_MAG; 5135 5136 device_set_wakeup_enable(&adapter->pdev->dev, 5137 adapter->flags & IGC_FLAG_WOL_SUPPORTED); 5138 5139 igc_ptp_init(adapter); 5140 5141 /* reset the hardware with the new settings */ 5142 igc_reset(adapter); 5143 5144 /* let the f/w know that the h/w is now under the control of the 5145 * driver. 5146 */ 5147 igc_get_hw_control(adapter); 5148 5149 strncpy(netdev->name, "eth%d", IFNAMSIZ); 5150 err = register_netdev(netdev); 5151 if (err) 5152 goto err_register; 5153 5154 /* carrier off reporting is important to ethtool even BEFORE open */ 5155 netif_carrier_off(netdev); 5156 5157 /* Check if Media Autosense is enabled */ 5158 adapter->ei = *ei; 5159 5160 /* print pcie link status and MAC address */ 5161 pcie_print_link_status(pdev); 5162 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 5163 5164 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); 5165 /* Disable EEE for internal PHY devices */ 5166 hw->dev_spec._base.eee_enable = false; 5167 adapter->flags &= ~IGC_FLAG_EEE; 5168 igc_set_eee_i225(hw, false, false, false); 5169 5170 pm_runtime_put_noidle(&pdev->dev); 5171 5172 return 0; 5173 5174 err_register: 5175 igc_release_hw_control(adapter); 5176 err_eeprom: 5177 if (!igc_check_reset_block(hw)) 5178 igc_reset_phy(hw); 5179 err_sw_init: 5180 igc_clear_interrupt_scheme(adapter); 5181 iounmap(adapter->io_addr); 5182 err_ioremap: 5183 free_netdev(netdev); 5184 err_alloc_etherdev: 5185 pci_release_mem_regions(pdev); 5186 err_pci_reg: 5187 err_dma: 5188 pci_disable_device(pdev); 5189 return err; 5190 } 5191 5192 /** 5193 * igc_remove - Device Removal Routine 5194 * @pdev: PCI device information struct 5195 * 5196 * igc_remove is called by the PCI subsystem to alert the driver 5197 * that it should release a PCI device. This could be caused by a 5198 * Hot-Plug event, or because the driver is going to be removed from 5199 * memory. 5200 */ 5201 static void igc_remove(struct pci_dev *pdev) 5202 { 5203 struct net_device *netdev = pci_get_drvdata(pdev); 5204 struct igc_adapter *adapter = netdev_priv(netdev); 5205 5206 pm_runtime_get_noresume(&pdev->dev); 5207 5208 igc_flush_nfc_rules(adapter); 5209 5210 igc_ptp_stop(adapter); 5211 5212 set_bit(__IGC_DOWN, &adapter->state); 5213 5214 del_timer_sync(&adapter->watchdog_timer); 5215 del_timer_sync(&adapter->phy_info_timer); 5216 5217 cancel_work_sync(&adapter->reset_task); 5218 cancel_work_sync(&adapter->watchdog_task); 5219 5220 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5221 * would have already happened in close and is redundant. 5222 */ 5223 igc_release_hw_control(adapter); 5224 unregister_netdev(netdev); 5225 5226 igc_clear_interrupt_scheme(adapter); 5227 pci_iounmap(pdev, adapter->io_addr); 5228 pci_release_mem_regions(pdev); 5229 5230 free_netdev(netdev); 5231 5232 pci_disable_pcie_error_reporting(pdev); 5233 5234 pci_disable_device(pdev); 5235 } 5236 5237 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, 5238 bool runtime) 5239 { 5240 struct net_device *netdev = pci_get_drvdata(pdev); 5241 struct igc_adapter *adapter = netdev_priv(netdev); 5242 u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol; 5243 struct igc_hw *hw = &adapter->hw; 5244 u32 ctrl, rctl, status; 5245 bool wake; 5246 5247 rtnl_lock(); 5248 netif_device_detach(netdev); 5249 5250 if (netif_running(netdev)) 5251 __igc_close(netdev, true); 5252 5253 igc_ptp_suspend(adapter); 5254 5255 igc_clear_interrupt_scheme(adapter); 5256 rtnl_unlock(); 5257 5258 status = rd32(IGC_STATUS); 5259 if (status & IGC_STATUS_LU) 5260 wufc &= ~IGC_WUFC_LNKC; 5261 5262 if (wufc) { 5263 igc_setup_rctl(adapter); 5264 igc_set_rx_mode(netdev); 5265 5266 /* turn on all-multi mode if wake on multicast is enabled */ 5267 if (wufc & IGC_WUFC_MC) { 5268 rctl = rd32(IGC_RCTL); 5269 rctl |= IGC_RCTL_MPE; 5270 wr32(IGC_RCTL, rctl); 5271 } 5272 5273 ctrl = rd32(IGC_CTRL); 5274 ctrl |= IGC_CTRL_ADVD3WUC; 5275 wr32(IGC_CTRL, ctrl); 5276 5277 /* Allow time for pending master requests to run */ 5278 igc_disable_pcie_master(hw); 5279 5280 wr32(IGC_WUC, IGC_WUC_PME_EN); 5281 wr32(IGC_WUFC, wufc); 5282 } else { 5283 wr32(IGC_WUC, 0); 5284 wr32(IGC_WUFC, 0); 5285 } 5286 5287 wake = wufc || adapter->en_mng_pt; 5288 if (!wake) 5289 igc_power_down_phy_copper_base(&adapter->hw); 5290 else 5291 igc_power_up_link(adapter); 5292 5293 if (enable_wake) 5294 *enable_wake = wake; 5295 5296 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5297 * would have already happened in close and is redundant. 5298 */ 5299 igc_release_hw_control(adapter); 5300 5301 pci_disable_device(pdev); 5302 5303 return 0; 5304 } 5305 5306 #ifdef CONFIG_PM 5307 static int __maybe_unused igc_runtime_suspend(struct device *dev) 5308 { 5309 return __igc_shutdown(to_pci_dev(dev), NULL, 1); 5310 } 5311 5312 static void igc_deliver_wake_packet(struct net_device *netdev) 5313 { 5314 struct igc_adapter *adapter = netdev_priv(netdev); 5315 struct igc_hw *hw = &adapter->hw; 5316 struct sk_buff *skb; 5317 u32 wupl; 5318 5319 wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK; 5320 5321 /* WUPM stores only the first 128 bytes of the wake packet. 5322 * Read the packet only if we have the whole thing. 5323 */ 5324 if (wupl == 0 || wupl > IGC_WUPM_BYTES) 5325 return; 5326 5327 skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES); 5328 if (!skb) 5329 return; 5330 5331 skb_put(skb, wupl); 5332 5333 /* Ensure reads are 32-bit aligned */ 5334 wupl = roundup(wupl, 4); 5335 5336 memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl); 5337 5338 skb->protocol = eth_type_trans(skb, netdev); 5339 netif_rx(skb); 5340 } 5341 5342 static int __maybe_unused igc_resume(struct device *dev) 5343 { 5344 struct pci_dev *pdev = to_pci_dev(dev); 5345 struct net_device *netdev = pci_get_drvdata(pdev); 5346 struct igc_adapter *adapter = netdev_priv(netdev); 5347 struct igc_hw *hw = &adapter->hw; 5348 u32 err, val; 5349 5350 pci_set_power_state(pdev, PCI_D0); 5351 pci_restore_state(pdev); 5352 pci_save_state(pdev); 5353 5354 if (!pci_device_is_present(pdev)) 5355 return -ENODEV; 5356 err = pci_enable_device_mem(pdev); 5357 if (err) { 5358 netdev_err(netdev, "Cannot enable PCI device from suspend\n"); 5359 return err; 5360 } 5361 pci_set_master(pdev); 5362 5363 pci_enable_wake(pdev, PCI_D3hot, 0); 5364 pci_enable_wake(pdev, PCI_D3cold, 0); 5365 5366 if (igc_init_interrupt_scheme(adapter, true)) { 5367 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5368 return -ENOMEM; 5369 } 5370 5371 igc_reset(adapter); 5372 5373 /* let the f/w know that the h/w is now under the control of the 5374 * driver. 5375 */ 5376 igc_get_hw_control(adapter); 5377 5378 val = rd32(IGC_WUS); 5379 if (val & WAKE_PKT_WUS) 5380 igc_deliver_wake_packet(netdev); 5381 5382 wr32(IGC_WUS, ~0); 5383 5384 rtnl_lock(); 5385 if (!err && netif_running(netdev)) 5386 err = __igc_open(netdev, true); 5387 5388 if (!err) 5389 netif_device_attach(netdev); 5390 rtnl_unlock(); 5391 5392 return err; 5393 } 5394 5395 static int __maybe_unused igc_runtime_resume(struct device *dev) 5396 { 5397 return igc_resume(dev); 5398 } 5399 5400 static int __maybe_unused igc_suspend(struct device *dev) 5401 { 5402 return __igc_shutdown(to_pci_dev(dev), NULL, 0); 5403 } 5404 5405 static int __maybe_unused igc_runtime_idle(struct device *dev) 5406 { 5407 struct net_device *netdev = dev_get_drvdata(dev); 5408 struct igc_adapter *adapter = netdev_priv(netdev); 5409 5410 if (!igc_has_link(adapter)) 5411 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 5412 5413 return -EBUSY; 5414 } 5415 #endif /* CONFIG_PM */ 5416 5417 static void igc_shutdown(struct pci_dev *pdev) 5418 { 5419 bool wake; 5420 5421 __igc_shutdown(pdev, &wake, 0); 5422 5423 if (system_state == SYSTEM_POWER_OFF) { 5424 pci_wake_from_d3(pdev, wake); 5425 pci_set_power_state(pdev, PCI_D3hot); 5426 } 5427 } 5428 5429 /** 5430 * igc_io_error_detected - called when PCI error is detected 5431 * @pdev: Pointer to PCI device 5432 * @state: The current PCI connection state 5433 * 5434 * This function is called after a PCI bus error affecting 5435 * this device has been detected. 5436 **/ 5437 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, 5438 pci_channel_state_t state) 5439 { 5440 struct net_device *netdev = pci_get_drvdata(pdev); 5441 struct igc_adapter *adapter = netdev_priv(netdev); 5442 5443 netif_device_detach(netdev); 5444 5445 if (state == pci_channel_io_perm_failure) 5446 return PCI_ERS_RESULT_DISCONNECT; 5447 5448 if (netif_running(netdev)) 5449 igc_down(adapter); 5450 pci_disable_device(pdev); 5451 5452 /* Request a slot reset. */ 5453 return PCI_ERS_RESULT_NEED_RESET; 5454 } 5455 5456 /** 5457 * igc_io_slot_reset - called after the PCI bus has been reset. 5458 * @pdev: Pointer to PCI device 5459 * 5460 * Restart the card from scratch, as if from a cold-boot. Implementation 5461 * resembles the first-half of the igc_resume routine. 5462 **/ 5463 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) 5464 { 5465 struct net_device *netdev = pci_get_drvdata(pdev); 5466 struct igc_adapter *adapter = netdev_priv(netdev); 5467 struct igc_hw *hw = &adapter->hw; 5468 pci_ers_result_t result; 5469 5470 if (pci_enable_device_mem(pdev)) { 5471 netdev_err(netdev, "Could not re-enable PCI device after reset\n"); 5472 result = PCI_ERS_RESULT_DISCONNECT; 5473 } else { 5474 pci_set_master(pdev); 5475 pci_restore_state(pdev); 5476 pci_save_state(pdev); 5477 5478 pci_enable_wake(pdev, PCI_D3hot, 0); 5479 pci_enable_wake(pdev, PCI_D3cold, 0); 5480 5481 /* In case of PCI error, adapter loses its HW address 5482 * so we should re-assign it here. 5483 */ 5484 hw->hw_addr = adapter->io_addr; 5485 5486 igc_reset(adapter); 5487 wr32(IGC_WUS, ~0); 5488 result = PCI_ERS_RESULT_RECOVERED; 5489 } 5490 5491 return result; 5492 } 5493 5494 /** 5495 * igc_io_resume - called when traffic can start to flow again. 5496 * @pdev: Pointer to PCI device 5497 * 5498 * This callback is called when the error recovery driver tells us that 5499 * its OK to resume normal operation. Implementation resembles the 5500 * second-half of the igc_resume routine. 5501 */ 5502 static void igc_io_resume(struct pci_dev *pdev) 5503 { 5504 struct net_device *netdev = pci_get_drvdata(pdev); 5505 struct igc_adapter *adapter = netdev_priv(netdev); 5506 5507 rtnl_lock(); 5508 if (netif_running(netdev)) { 5509 if (igc_open(netdev)) { 5510 netdev_err(netdev, "igc_open failed after reset\n"); 5511 return; 5512 } 5513 } 5514 5515 netif_device_attach(netdev); 5516 5517 /* let the f/w know that the h/w is now under the control of the 5518 * driver. 5519 */ 5520 igc_get_hw_control(adapter); 5521 rtnl_unlock(); 5522 } 5523 5524 static const struct pci_error_handlers igc_err_handler = { 5525 .error_detected = igc_io_error_detected, 5526 .slot_reset = igc_io_slot_reset, 5527 .resume = igc_io_resume, 5528 }; 5529 5530 #ifdef CONFIG_PM 5531 static const struct dev_pm_ops igc_pm_ops = { 5532 SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume) 5533 SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume, 5534 igc_runtime_idle) 5535 }; 5536 #endif 5537 5538 static struct pci_driver igc_driver = { 5539 .name = igc_driver_name, 5540 .id_table = igc_pci_tbl, 5541 .probe = igc_probe, 5542 .remove = igc_remove, 5543 #ifdef CONFIG_PM 5544 .driver.pm = &igc_pm_ops, 5545 #endif 5546 .shutdown = igc_shutdown, 5547 .err_handler = &igc_err_handler, 5548 }; 5549 5550 /** 5551 * igc_reinit_queues - return error 5552 * @adapter: pointer to adapter structure 5553 */ 5554 int igc_reinit_queues(struct igc_adapter *adapter) 5555 { 5556 struct net_device *netdev = adapter->netdev; 5557 int err = 0; 5558 5559 if (netif_running(netdev)) 5560 igc_close(netdev); 5561 5562 igc_reset_interrupt_capability(adapter); 5563 5564 if (igc_init_interrupt_scheme(adapter, true)) { 5565 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5566 return -ENOMEM; 5567 } 5568 5569 if (netif_running(netdev)) 5570 err = igc_open(netdev); 5571 5572 return err; 5573 } 5574 5575 /** 5576 * igc_get_hw_dev - return device 5577 * @hw: pointer to hardware structure 5578 * 5579 * used by hardware layer to print debugging information 5580 */ 5581 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 5582 { 5583 struct igc_adapter *adapter = hw->back; 5584 5585 return adapter->netdev; 5586 } 5587 5588 /** 5589 * igc_init_module - Driver Registration Routine 5590 * 5591 * igc_init_module is the first routine called when the driver is 5592 * loaded. All it does is register with the PCI subsystem. 5593 */ 5594 static int __init igc_init_module(void) 5595 { 5596 int ret; 5597 5598 pr_info("%s\n", igc_driver_string); 5599 pr_info("%s\n", igc_copyright); 5600 5601 ret = pci_register_driver(&igc_driver); 5602 return ret; 5603 } 5604 5605 module_init(igc_init_module); 5606 5607 /** 5608 * igc_exit_module - Driver Exit Cleanup Routine 5609 * 5610 * igc_exit_module is called just before the driver is removed 5611 * from memory. 5612 */ 5613 static void __exit igc_exit_module(void) 5614 { 5615 pci_unregister_driver(&igc_driver); 5616 } 5617 5618 module_exit(igc_exit_module); 5619 /* igc_main.c */ 5620