1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/aer.h> 8 #include <linux/tcp.h> 9 #include <linux/udp.h> 10 #include <linux/ip.h> 11 #include <linux/pm_runtime.h> 12 #include <net/pkt_sched.h> 13 14 #include <net/ipv6.h> 15 16 #include "igc.h" 17 #include "igc_hw.h" 18 #include "igc_tsn.h" 19 20 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 21 22 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 23 24 static int debug = -1; 25 26 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 27 MODULE_DESCRIPTION(DRV_SUMMARY); 28 MODULE_LICENSE("GPL v2"); 29 module_param(debug, int, 0); 30 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 31 32 char igc_driver_name[] = "igc"; 33 static const char igc_driver_string[] = DRV_SUMMARY; 34 static const char igc_copyright[] = 35 "Copyright(c) 2018 Intel Corporation."; 36 37 static const struct igc_info *igc_info_tbl[] = { 38 [board_base] = &igc_base_info, 39 }; 40 41 static const struct pci_device_id igc_pci_tbl[] = { 42 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base }, 43 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base }, 44 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, 45 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, 46 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, 47 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base }, 48 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base }, 49 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base }, 50 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, 51 /* required last entry */ 52 {0, } 53 }; 54 55 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 56 57 enum latency_range { 58 lowest_latency = 0, 59 low_latency = 1, 60 bulk_latency = 2, 61 latency_invalid = 255 62 }; 63 64 void igc_reset(struct igc_adapter *adapter) 65 { 66 struct net_device *dev = adapter->netdev; 67 struct igc_hw *hw = &adapter->hw; 68 struct igc_fc_info *fc = &hw->fc; 69 u32 pba, hwm; 70 71 /* Repartition PBA for greater than 9k MTU if required */ 72 pba = IGC_PBA_34K; 73 74 /* flow control settings 75 * The high water mark must be low enough to fit one full frame 76 * after transmitting the pause frame. As such we must have enough 77 * space to allow for us to complete our current transmit and then 78 * receive the frame that is in progress from the link partner. 79 * Set it to: 80 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 81 */ 82 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 83 84 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 85 fc->low_water = fc->high_water - 16; 86 fc->pause_time = 0xFFFF; 87 fc->send_xon = 1; 88 fc->current_mode = fc->requested_mode; 89 90 hw->mac.ops.reset_hw(hw); 91 92 if (hw->mac.ops.init_hw(hw)) 93 netdev_err(dev, "Error on hardware initialization\n"); 94 95 /* Re-establish EEE setting */ 96 igc_set_eee_i225(hw, true, true, true); 97 98 if (!netif_running(adapter->netdev)) 99 igc_power_down_phy_copper_base(&adapter->hw); 100 101 /* Re-enable PTP, where applicable. */ 102 igc_ptp_reset(adapter); 103 104 /* Re-enable TSN offloading, where applicable. */ 105 igc_tsn_offload_apply(adapter); 106 107 igc_get_phy_info(hw); 108 } 109 110 /** 111 * igc_power_up_link - Power up the phy link 112 * @adapter: address of board private structure 113 */ 114 static void igc_power_up_link(struct igc_adapter *adapter) 115 { 116 igc_reset_phy(&adapter->hw); 117 118 igc_power_up_phy_copper(&adapter->hw); 119 120 igc_setup_link(&adapter->hw); 121 } 122 123 /** 124 * igc_release_hw_control - release control of the h/w to f/w 125 * @adapter: address of board private structure 126 * 127 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 128 * For ASF and Pass Through versions of f/w this means that the 129 * driver is no longer loaded. 130 */ 131 static void igc_release_hw_control(struct igc_adapter *adapter) 132 { 133 struct igc_hw *hw = &adapter->hw; 134 u32 ctrl_ext; 135 136 /* Let firmware take over control of h/w */ 137 ctrl_ext = rd32(IGC_CTRL_EXT); 138 wr32(IGC_CTRL_EXT, 139 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 140 } 141 142 /** 143 * igc_get_hw_control - get control of the h/w from f/w 144 * @adapter: address of board private structure 145 * 146 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 147 * For ASF and Pass Through versions of f/w this means that 148 * the driver is loaded. 149 */ 150 static void igc_get_hw_control(struct igc_adapter *adapter) 151 { 152 struct igc_hw *hw = &adapter->hw; 153 u32 ctrl_ext; 154 155 /* Let firmware know the driver has taken over */ 156 ctrl_ext = rd32(IGC_CTRL_EXT); 157 wr32(IGC_CTRL_EXT, 158 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 159 } 160 161 /** 162 * igc_clean_tx_ring - Free Tx Buffers 163 * @tx_ring: ring to be cleaned 164 */ 165 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 166 { 167 u16 i = tx_ring->next_to_clean; 168 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 169 170 while (i != tx_ring->next_to_use) { 171 union igc_adv_tx_desc *eop_desc, *tx_desc; 172 173 /* Free all the Tx ring sk_buffs */ 174 dev_kfree_skb_any(tx_buffer->skb); 175 176 /* unmap skb header data */ 177 dma_unmap_single(tx_ring->dev, 178 dma_unmap_addr(tx_buffer, dma), 179 dma_unmap_len(tx_buffer, len), 180 DMA_TO_DEVICE); 181 182 /* check for eop_desc to determine the end of the packet */ 183 eop_desc = tx_buffer->next_to_watch; 184 tx_desc = IGC_TX_DESC(tx_ring, i); 185 186 /* unmap remaining buffers */ 187 while (tx_desc != eop_desc) { 188 tx_buffer++; 189 tx_desc++; 190 i++; 191 if (unlikely(i == tx_ring->count)) { 192 i = 0; 193 tx_buffer = tx_ring->tx_buffer_info; 194 tx_desc = IGC_TX_DESC(tx_ring, 0); 195 } 196 197 /* unmap any remaining paged data */ 198 if (dma_unmap_len(tx_buffer, len)) 199 dma_unmap_page(tx_ring->dev, 200 dma_unmap_addr(tx_buffer, dma), 201 dma_unmap_len(tx_buffer, len), 202 DMA_TO_DEVICE); 203 } 204 205 /* move us one more past the eop_desc for start of next pkt */ 206 tx_buffer++; 207 i++; 208 if (unlikely(i == tx_ring->count)) { 209 i = 0; 210 tx_buffer = tx_ring->tx_buffer_info; 211 } 212 } 213 214 /* reset BQL for queue */ 215 netdev_tx_reset_queue(txring_txq(tx_ring)); 216 217 /* reset next_to_use and next_to_clean */ 218 tx_ring->next_to_use = 0; 219 tx_ring->next_to_clean = 0; 220 } 221 222 /** 223 * igc_free_tx_resources - Free Tx Resources per Queue 224 * @tx_ring: Tx descriptor ring for a specific queue 225 * 226 * Free all transmit software resources 227 */ 228 void igc_free_tx_resources(struct igc_ring *tx_ring) 229 { 230 igc_clean_tx_ring(tx_ring); 231 232 vfree(tx_ring->tx_buffer_info); 233 tx_ring->tx_buffer_info = NULL; 234 235 /* if not set, then don't free */ 236 if (!tx_ring->desc) 237 return; 238 239 dma_free_coherent(tx_ring->dev, tx_ring->size, 240 tx_ring->desc, tx_ring->dma); 241 242 tx_ring->desc = NULL; 243 } 244 245 /** 246 * igc_free_all_tx_resources - Free Tx Resources for All Queues 247 * @adapter: board private structure 248 * 249 * Free all transmit software resources 250 */ 251 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 252 { 253 int i; 254 255 for (i = 0; i < adapter->num_tx_queues; i++) 256 igc_free_tx_resources(adapter->tx_ring[i]); 257 } 258 259 /** 260 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 261 * @adapter: board private structure 262 */ 263 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 264 { 265 int i; 266 267 for (i = 0; i < adapter->num_tx_queues; i++) 268 if (adapter->tx_ring[i]) 269 igc_clean_tx_ring(adapter->tx_ring[i]); 270 } 271 272 /** 273 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 274 * @tx_ring: tx descriptor ring (for a specific queue) to setup 275 * 276 * Return 0 on success, negative on failure 277 */ 278 int igc_setup_tx_resources(struct igc_ring *tx_ring) 279 { 280 struct net_device *ndev = tx_ring->netdev; 281 struct device *dev = tx_ring->dev; 282 int size = 0; 283 284 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 285 tx_ring->tx_buffer_info = vzalloc(size); 286 if (!tx_ring->tx_buffer_info) 287 goto err; 288 289 /* round up to nearest 4K */ 290 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 291 tx_ring->size = ALIGN(tx_ring->size, 4096); 292 293 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 294 &tx_ring->dma, GFP_KERNEL); 295 296 if (!tx_ring->desc) 297 goto err; 298 299 tx_ring->next_to_use = 0; 300 tx_ring->next_to_clean = 0; 301 302 return 0; 303 304 err: 305 vfree(tx_ring->tx_buffer_info); 306 netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n"); 307 return -ENOMEM; 308 } 309 310 /** 311 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 312 * @adapter: board private structure 313 * 314 * Return 0 on success, negative on failure 315 */ 316 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 317 { 318 struct net_device *dev = adapter->netdev; 319 int i, err = 0; 320 321 for (i = 0; i < adapter->num_tx_queues; i++) { 322 err = igc_setup_tx_resources(adapter->tx_ring[i]); 323 if (err) { 324 netdev_err(dev, "Error on Tx queue %u setup\n", i); 325 for (i--; i >= 0; i--) 326 igc_free_tx_resources(adapter->tx_ring[i]); 327 break; 328 } 329 } 330 331 return err; 332 } 333 334 /** 335 * igc_clean_rx_ring - Free Rx Buffers per Queue 336 * @rx_ring: ring to free buffers from 337 */ 338 static void igc_clean_rx_ring(struct igc_ring *rx_ring) 339 { 340 u16 i = rx_ring->next_to_clean; 341 342 dev_kfree_skb(rx_ring->skb); 343 rx_ring->skb = NULL; 344 345 /* Free all the Rx ring sk_buffs */ 346 while (i != rx_ring->next_to_alloc) { 347 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 348 349 /* Invalidate cache lines that may have been written to by 350 * device so that we avoid corrupting memory. 351 */ 352 dma_sync_single_range_for_cpu(rx_ring->dev, 353 buffer_info->dma, 354 buffer_info->page_offset, 355 igc_rx_bufsz(rx_ring), 356 DMA_FROM_DEVICE); 357 358 /* free resources associated with mapping */ 359 dma_unmap_page_attrs(rx_ring->dev, 360 buffer_info->dma, 361 igc_rx_pg_size(rx_ring), 362 DMA_FROM_DEVICE, 363 IGC_RX_DMA_ATTR); 364 __page_frag_cache_drain(buffer_info->page, 365 buffer_info->pagecnt_bias); 366 367 i++; 368 if (i == rx_ring->count) 369 i = 0; 370 } 371 372 rx_ring->next_to_alloc = 0; 373 rx_ring->next_to_clean = 0; 374 rx_ring->next_to_use = 0; 375 } 376 377 /** 378 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 379 * @adapter: board private structure 380 */ 381 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 382 { 383 int i; 384 385 for (i = 0; i < adapter->num_rx_queues; i++) 386 if (adapter->rx_ring[i]) 387 igc_clean_rx_ring(adapter->rx_ring[i]); 388 } 389 390 /** 391 * igc_free_rx_resources - Free Rx Resources 392 * @rx_ring: ring to clean the resources from 393 * 394 * Free all receive software resources 395 */ 396 void igc_free_rx_resources(struct igc_ring *rx_ring) 397 { 398 igc_clean_rx_ring(rx_ring); 399 400 vfree(rx_ring->rx_buffer_info); 401 rx_ring->rx_buffer_info = NULL; 402 403 /* if not set, then don't free */ 404 if (!rx_ring->desc) 405 return; 406 407 dma_free_coherent(rx_ring->dev, rx_ring->size, 408 rx_ring->desc, rx_ring->dma); 409 410 rx_ring->desc = NULL; 411 } 412 413 /** 414 * igc_free_all_rx_resources - Free Rx Resources for All Queues 415 * @adapter: board private structure 416 * 417 * Free all receive software resources 418 */ 419 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 420 { 421 int i; 422 423 for (i = 0; i < adapter->num_rx_queues; i++) 424 igc_free_rx_resources(adapter->rx_ring[i]); 425 } 426 427 /** 428 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 429 * @rx_ring: rx descriptor ring (for a specific queue) to setup 430 * 431 * Returns 0 on success, negative on failure 432 */ 433 int igc_setup_rx_resources(struct igc_ring *rx_ring) 434 { 435 struct net_device *ndev = rx_ring->netdev; 436 struct device *dev = rx_ring->dev; 437 int size, desc_len; 438 439 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 440 rx_ring->rx_buffer_info = vzalloc(size); 441 if (!rx_ring->rx_buffer_info) 442 goto err; 443 444 desc_len = sizeof(union igc_adv_rx_desc); 445 446 /* Round up to nearest 4K */ 447 rx_ring->size = rx_ring->count * desc_len; 448 rx_ring->size = ALIGN(rx_ring->size, 4096); 449 450 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 451 &rx_ring->dma, GFP_KERNEL); 452 453 if (!rx_ring->desc) 454 goto err; 455 456 rx_ring->next_to_alloc = 0; 457 rx_ring->next_to_clean = 0; 458 rx_ring->next_to_use = 0; 459 460 return 0; 461 462 err: 463 vfree(rx_ring->rx_buffer_info); 464 rx_ring->rx_buffer_info = NULL; 465 netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n"); 466 return -ENOMEM; 467 } 468 469 /** 470 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 471 * (Descriptors) for all queues 472 * @adapter: board private structure 473 * 474 * Return 0 on success, negative on failure 475 */ 476 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 477 { 478 struct net_device *dev = adapter->netdev; 479 int i, err = 0; 480 481 for (i = 0; i < adapter->num_rx_queues; i++) { 482 err = igc_setup_rx_resources(adapter->rx_ring[i]); 483 if (err) { 484 netdev_err(dev, "Error on Rx queue %u setup\n", i); 485 for (i--; i >= 0; i--) 486 igc_free_rx_resources(adapter->rx_ring[i]); 487 break; 488 } 489 } 490 491 return err; 492 } 493 494 /** 495 * igc_configure_rx_ring - Configure a receive ring after Reset 496 * @adapter: board private structure 497 * @ring: receive ring to be configured 498 * 499 * Configure the Rx unit of the MAC after a reset. 500 */ 501 static void igc_configure_rx_ring(struct igc_adapter *adapter, 502 struct igc_ring *ring) 503 { 504 struct igc_hw *hw = &adapter->hw; 505 union igc_adv_rx_desc *rx_desc; 506 int reg_idx = ring->reg_idx; 507 u32 srrctl = 0, rxdctl = 0; 508 u64 rdba = ring->dma; 509 510 /* disable the queue */ 511 wr32(IGC_RXDCTL(reg_idx), 0); 512 513 /* Set DMA base address registers */ 514 wr32(IGC_RDBAL(reg_idx), 515 rdba & 0x00000000ffffffffULL); 516 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 517 wr32(IGC_RDLEN(reg_idx), 518 ring->count * sizeof(union igc_adv_rx_desc)); 519 520 /* initialize head and tail */ 521 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 522 wr32(IGC_RDH(reg_idx), 0); 523 writel(0, ring->tail); 524 525 /* reset next-to- use/clean to place SW in sync with hardware */ 526 ring->next_to_clean = 0; 527 ring->next_to_use = 0; 528 529 /* set descriptor configuration */ 530 srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; 531 if (ring_uses_large_buffer(ring)) 532 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 533 else 534 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 535 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 536 537 wr32(IGC_SRRCTL(reg_idx), srrctl); 538 539 rxdctl |= IGC_RX_PTHRESH; 540 rxdctl |= IGC_RX_HTHRESH << 8; 541 rxdctl |= IGC_RX_WTHRESH << 16; 542 543 /* initialize rx_buffer_info */ 544 memset(ring->rx_buffer_info, 0, 545 sizeof(struct igc_rx_buffer) * ring->count); 546 547 /* initialize Rx descriptor 0 */ 548 rx_desc = IGC_RX_DESC(ring, 0); 549 rx_desc->wb.upper.length = 0; 550 551 /* enable receive descriptor fetching */ 552 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 553 554 wr32(IGC_RXDCTL(reg_idx), rxdctl); 555 } 556 557 /** 558 * igc_configure_rx - Configure receive Unit after Reset 559 * @adapter: board private structure 560 * 561 * Configure the Rx unit of the MAC after a reset. 562 */ 563 static void igc_configure_rx(struct igc_adapter *adapter) 564 { 565 int i; 566 567 /* Setup the HW Rx Head and Tail Descriptor Pointers and 568 * the Base and Length of the Rx Descriptor Ring 569 */ 570 for (i = 0; i < adapter->num_rx_queues; i++) 571 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 572 } 573 574 /** 575 * igc_configure_tx_ring - Configure transmit ring after Reset 576 * @adapter: board private structure 577 * @ring: tx ring to configure 578 * 579 * Configure a transmit ring after a reset. 580 */ 581 static void igc_configure_tx_ring(struct igc_adapter *adapter, 582 struct igc_ring *ring) 583 { 584 struct igc_hw *hw = &adapter->hw; 585 int reg_idx = ring->reg_idx; 586 u64 tdba = ring->dma; 587 u32 txdctl = 0; 588 589 /* disable the queue */ 590 wr32(IGC_TXDCTL(reg_idx), 0); 591 wrfl(); 592 mdelay(10); 593 594 wr32(IGC_TDLEN(reg_idx), 595 ring->count * sizeof(union igc_adv_tx_desc)); 596 wr32(IGC_TDBAL(reg_idx), 597 tdba & 0x00000000ffffffffULL); 598 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 599 600 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 601 wr32(IGC_TDH(reg_idx), 0); 602 writel(0, ring->tail); 603 604 txdctl |= IGC_TX_PTHRESH; 605 txdctl |= IGC_TX_HTHRESH << 8; 606 txdctl |= IGC_TX_WTHRESH << 16; 607 608 txdctl |= IGC_TXDCTL_QUEUE_ENABLE; 609 wr32(IGC_TXDCTL(reg_idx), txdctl); 610 } 611 612 /** 613 * igc_configure_tx - Configure transmit Unit after Reset 614 * @adapter: board private structure 615 * 616 * Configure the Tx unit of the MAC after a reset. 617 */ 618 static void igc_configure_tx(struct igc_adapter *adapter) 619 { 620 int i; 621 622 for (i = 0; i < adapter->num_tx_queues; i++) 623 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 624 } 625 626 /** 627 * igc_setup_mrqc - configure the multiple receive queue control registers 628 * @adapter: Board private structure 629 */ 630 static void igc_setup_mrqc(struct igc_adapter *adapter) 631 { 632 struct igc_hw *hw = &adapter->hw; 633 u32 j, num_rx_queues; 634 u32 mrqc, rxcsum; 635 u32 rss_key[10]; 636 637 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 638 for (j = 0; j < 10; j++) 639 wr32(IGC_RSSRK(j), rss_key[j]); 640 641 num_rx_queues = adapter->rss_queues; 642 643 if (adapter->rss_indir_tbl_init != num_rx_queues) { 644 for (j = 0; j < IGC_RETA_SIZE; j++) 645 adapter->rss_indir_tbl[j] = 646 (j * num_rx_queues) / IGC_RETA_SIZE; 647 adapter->rss_indir_tbl_init = num_rx_queues; 648 } 649 igc_write_rss_indir_tbl(adapter); 650 651 /* Disable raw packet checksumming so that RSS hash is placed in 652 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 653 * offloads as they are enabled by default 654 */ 655 rxcsum = rd32(IGC_RXCSUM); 656 rxcsum |= IGC_RXCSUM_PCSD; 657 658 /* Enable Receive Checksum Offload for SCTP */ 659 rxcsum |= IGC_RXCSUM_CRCOFL; 660 661 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 662 wr32(IGC_RXCSUM, rxcsum); 663 664 /* Generate RSS hash based on packet types, TCP/UDP 665 * port numbers and/or IPv4/v6 src and dst addresses 666 */ 667 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 668 IGC_MRQC_RSS_FIELD_IPV4_TCP | 669 IGC_MRQC_RSS_FIELD_IPV6 | 670 IGC_MRQC_RSS_FIELD_IPV6_TCP | 671 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 672 673 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 674 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 675 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 676 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 677 678 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 679 680 wr32(IGC_MRQC, mrqc); 681 } 682 683 /** 684 * igc_setup_rctl - configure the receive control registers 685 * @adapter: Board private structure 686 */ 687 static void igc_setup_rctl(struct igc_adapter *adapter) 688 { 689 struct igc_hw *hw = &adapter->hw; 690 u32 rctl; 691 692 rctl = rd32(IGC_RCTL); 693 694 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 695 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 696 697 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 698 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 699 700 /* enable stripping of CRC. Newer features require 701 * that the HW strips the CRC. 702 */ 703 rctl |= IGC_RCTL_SECRC; 704 705 /* disable store bad packets and clear size bits. */ 706 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 707 708 /* enable LPE to allow for reception of jumbo frames */ 709 rctl |= IGC_RCTL_LPE; 710 711 /* disable queue 0 to prevent tail write w/o re-config */ 712 wr32(IGC_RXDCTL(0), 0); 713 714 /* This is useful for sniffing bad packets. */ 715 if (adapter->netdev->features & NETIF_F_RXALL) { 716 /* UPE and MPE will be handled by normal PROMISC logic 717 * in set_rx_mode 718 */ 719 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 720 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 721 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 722 723 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 724 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 725 } 726 727 wr32(IGC_RCTL, rctl); 728 } 729 730 /** 731 * igc_setup_tctl - configure the transmit control registers 732 * @adapter: Board private structure 733 */ 734 static void igc_setup_tctl(struct igc_adapter *adapter) 735 { 736 struct igc_hw *hw = &adapter->hw; 737 u32 tctl; 738 739 /* disable queue 0 which icould be enabled by default */ 740 wr32(IGC_TXDCTL(0), 0); 741 742 /* Program the Transmit Control Register */ 743 tctl = rd32(IGC_TCTL); 744 tctl &= ~IGC_TCTL_CT; 745 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 746 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 747 748 /* Enable transmits */ 749 tctl |= IGC_TCTL_EN; 750 751 wr32(IGC_TCTL, tctl); 752 } 753 754 /** 755 * igc_set_mac_filter_hw() - Set MAC address filter in hardware 756 * @adapter: Pointer to adapter where the filter should be set 757 * @index: Filter index 758 * @type: MAC address filter type (source or destination) 759 * @addr: MAC address 760 * @queue: If non-negative, queue assignment feature is enabled and frames 761 * matching the filter are enqueued onto 'queue'. Otherwise, queue 762 * assignment is disabled. 763 */ 764 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, 765 enum igc_mac_filter_type type, 766 const u8 *addr, int queue) 767 { 768 struct net_device *dev = adapter->netdev; 769 struct igc_hw *hw = &adapter->hw; 770 u32 ral, rah; 771 772 if (WARN_ON(index >= hw->mac.rar_entry_count)) 773 return; 774 775 ral = le32_to_cpup((__le32 *)(addr)); 776 rah = le16_to_cpup((__le16 *)(addr + 4)); 777 778 if (type == IGC_MAC_FILTER_TYPE_SRC) { 779 rah &= ~IGC_RAH_ASEL_MASK; 780 rah |= IGC_RAH_ASEL_SRC_ADDR; 781 } 782 783 if (queue >= 0) { 784 rah &= ~IGC_RAH_QSEL_MASK; 785 rah |= (queue << IGC_RAH_QSEL_SHIFT); 786 rah |= IGC_RAH_QSEL_ENABLE; 787 } 788 789 rah |= IGC_RAH_AV; 790 791 wr32(IGC_RAL(index), ral); 792 wr32(IGC_RAH(index), rah); 793 794 netdev_dbg(dev, "MAC address filter set in HW: index %d", index); 795 } 796 797 /** 798 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware 799 * @adapter: Pointer to adapter where the filter should be cleared 800 * @index: Filter index 801 */ 802 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) 803 { 804 struct net_device *dev = adapter->netdev; 805 struct igc_hw *hw = &adapter->hw; 806 807 if (WARN_ON(index >= hw->mac.rar_entry_count)) 808 return; 809 810 wr32(IGC_RAL(index), 0); 811 wr32(IGC_RAH(index), 0); 812 813 netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index); 814 } 815 816 /* Set default MAC address for the PF in the first RAR entry */ 817 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 818 { 819 struct net_device *dev = adapter->netdev; 820 u8 *addr = adapter->hw.mac.addr; 821 822 netdev_dbg(dev, "Set default MAC address filter: address %pM", addr); 823 824 igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1); 825 } 826 827 /** 828 * igc_set_mac - Change the Ethernet Address of the NIC 829 * @netdev: network interface device structure 830 * @p: pointer to an address structure 831 * 832 * Returns 0 on success, negative on failure 833 */ 834 static int igc_set_mac(struct net_device *netdev, void *p) 835 { 836 struct igc_adapter *adapter = netdev_priv(netdev); 837 struct igc_hw *hw = &adapter->hw; 838 struct sockaddr *addr = p; 839 840 if (!is_valid_ether_addr(addr->sa_data)) 841 return -EADDRNOTAVAIL; 842 843 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 844 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 845 846 /* set the correct pool for the new PF MAC address in entry 0 */ 847 igc_set_default_mac_filter(adapter); 848 849 return 0; 850 } 851 852 /** 853 * igc_write_mc_addr_list - write multicast addresses to MTA 854 * @netdev: network interface device structure 855 * 856 * Writes multicast address list to the MTA hash table. 857 * Returns: -ENOMEM on failure 858 * 0 on no addresses written 859 * X on writing X addresses to MTA 860 **/ 861 static int igc_write_mc_addr_list(struct net_device *netdev) 862 { 863 struct igc_adapter *adapter = netdev_priv(netdev); 864 struct igc_hw *hw = &adapter->hw; 865 struct netdev_hw_addr *ha; 866 u8 *mta_list; 867 int i; 868 869 if (netdev_mc_empty(netdev)) { 870 /* nothing to program, so clear mc list */ 871 igc_update_mc_addr_list(hw, NULL, 0); 872 return 0; 873 } 874 875 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 876 if (!mta_list) 877 return -ENOMEM; 878 879 /* The shared function expects a packed array of only addresses. */ 880 i = 0; 881 netdev_for_each_mc_addr(ha, netdev) 882 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 883 884 igc_update_mc_addr_list(hw, mta_list, i); 885 kfree(mta_list); 886 887 return netdev_mc_count(netdev); 888 } 889 890 static __le32 igc_tx_launchtime(struct igc_adapter *adapter, ktime_t txtime) 891 { 892 ktime_t cycle_time = adapter->cycle_time; 893 ktime_t base_time = adapter->base_time; 894 u32 launchtime; 895 896 /* FIXME: when using ETF together with taprio, we may have a 897 * case where 'delta' is larger than the cycle_time, this may 898 * cause problems if we don't read the current value of 899 * IGC_BASET, as the value writen into the launchtime 900 * descriptor field may be misinterpreted. 901 */ 902 div_s64_rem(ktime_sub_ns(txtime, base_time), cycle_time, &launchtime); 903 904 return cpu_to_le32(launchtime); 905 } 906 907 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, 908 struct igc_tx_buffer *first, 909 u32 vlan_macip_lens, u32 type_tucmd, 910 u32 mss_l4len_idx) 911 { 912 struct igc_adv_tx_context_desc *context_desc; 913 u16 i = tx_ring->next_to_use; 914 915 context_desc = IGC_TX_CTXTDESC(tx_ring, i); 916 917 i++; 918 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 919 920 /* set bits to identify this as an advanced context descriptor */ 921 type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 922 923 /* For i225, context index must be unique per ring. */ 924 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 925 mss_l4len_idx |= tx_ring->reg_idx << 4; 926 927 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 928 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 929 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 930 931 /* We assume there is always a valid Tx time available. Invalid times 932 * should have been handled by the upper layers. 933 */ 934 if (tx_ring->launchtime_enable) { 935 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 936 ktime_t txtime = first->skb->tstamp; 937 938 first->skb->tstamp = ktime_set(0, 0); 939 context_desc->launch_time = igc_tx_launchtime(adapter, 940 txtime); 941 } else { 942 context_desc->launch_time = 0; 943 } 944 } 945 946 static inline bool igc_ipv6_csum_is_sctp(struct sk_buff *skb) 947 { 948 unsigned int offset = 0; 949 950 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 951 952 return offset == skb_checksum_start_offset(skb); 953 } 954 955 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first) 956 { 957 struct sk_buff *skb = first->skb; 958 u32 vlan_macip_lens = 0; 959 u32 type_tucmd = 0; 960 961 if (skb->ip_summed != CHECKSUM_PARTIAL) { 962 csum_failed: 963 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && 964 !tx_ring->launchtime_enable) 965 return; 966 goto no_csum; 967 } 968 969 switch (skb->csum_offset) { 970 case offsetof(struct tcphdr, check): 971 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 972 fallthrough; 973 case offsetof(struct udphdr, check): 974 break; 975 case offsetof(struct sctphdr, checksum): 976 /* validate that this is actually an SCTP request */ 977 if ((first->protocol == htons(ETH_P_IP) && 978 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 979 (first->protocol == htons(ETH_P_IPV6) && 980 igc_ipv6_csum_is_sctp(skb))) { 981 type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; 982 break; 983 } 984 fallthrough; 985 default: 986 skb_checksum_help(skb); 987 goto csum_failed; 988 } 989 990 /* update TX checksum flag */ 991 first->tx_flags |= IGC_TX_FLAGS_CSUM; 992 vlan_macip_lens = skb_checksum_start_offset(skb) - 993 skb_network_offset(skb); 994 no_csum: 995 vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; 996 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 997 998 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0); 999 } 1000 1001 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1002 { 1003 struct net_device *netdev = tx_ring->netdev; 1004 1005 netif_stop_subqueue(netdev, tx_ring->queue_index); 1006 1007 /* memory barriier comment */ 1008 smp_mb(); 1009 1010 /* We need to check again in a case another CPU has just 1011 * made room available. 1012 */ 1013 if (igc_desc_unused(tx_ring) < size) 1014 return -EBUSY; 1015 1016 /* A reprieve! */ 1017 netif_wake_subqueue(netdev, tx_ring->queue_index); 1018 1019 u64_stats_update_begin(&tx_ring->tx_syncp2); 1020 tx_ring->tx_stats.restart_queue2++; 1021 u64_stats_update_end(&tx_ring->tx_syncp2); 1022 1023 return 0; 1024 } 1025 1026 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1027 { 1028 if (igc_desc_unused(tx_ring) >= size) 1029 return 0; 1030 return __igc_maybe_stop_tx(tx_ring, size); 1031 } 1032 1033 #define IGC_SET_FLAG(_input, _flag, _result) \ 1034 (((_flag) <= (_result)) ? \ 1035 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ 1036 ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) 1037 1038 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 1039 { 1040 /* set type for advanced descriptor with frame checksum insertion */ 1041 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 1042 IGC_ADVTXD_DCMD_DEXT | 1043 IGC_ADVTXD_DCMD_IFCS; 1044 1045 /* set segmentation bits for TSO */ 1046 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, 1047 (IGC_ADVTXD_DCMD_TSE)); 1048 1049 /* set timestamp bit if present */ 1050 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, 1051 (IGC_ADVTXD_MAC_TSTAMP)); 1052 1053 return cmd_type; 1054 } 1055 1056 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 1057 union igc_adv_tx_desc *tx_desc, 1058 u32 tx_flags, unsigned int paylen) 1059 { 1060 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 1061 1062 /* insert L4 checksum */ 1063 olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) * 1064 ((IGC_TXD_POPTS_TXSM << 8) / 1065 IGC_TX_FLAGS_CSUM); 1066 1067 /* insert IPv4 checksum */ 1068 olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) * 1069 (((IGC_TXD_POPTS_IXSM << 8)) / 1070 IGC_TX_FLAGS_IPV4); 1071 1072 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 1073 } 1074 1075 static int igc_tx_map(struct igc_ring *tx_ring, 1076 struct igc_tx_buffer *first, 1077 const u8 hdr_len) 1078 { 1079 struct sk_buff *skb = first->skb; 1080 struct igc_tx_buffer *tx_buffer; 1081 union igc_adv_tx_desc *tx_desc; 1082 u32 tx_flags = first->tx_flags; 1083 skb_frag_t *frag; 1084 u16 i = tx_ring->next_to_use; 1085 unsigned int data_len, size; 1086 dma_addr_t dma; 1087 u32 cmd_type = igc_tx_cmd_type(skb, tx_flags); 1088 1089 tx_desc = IGC_TX_DESC(tx_ring, i); 1090 1091 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 1092 1093 size = skb_headlen(skb); 1094 data_len = skb->data_len; 1095 1096 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1097 1098 tx_buffer = first; 1099 1100 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1101 if (dma_mapping_error(tx_ring->dev, dma)) 1102 goto dma_error; 1103 1104 /* record length, and DMA address */ 1105 dma_unmap_len_set(tx_buffer, len, size); 1106 dma_unmap_addr_set(tx_buffer, dma, dma); 1107 1108 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1109 1110 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 1111 tx_desc->read.cmd_type_len = 1112 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 1113 1114 i++; 1115 tx_desc++; 1116 if (i == tx_ring->count) { 1117 tx_desc = IGC_TX_DESC(tx_ring, 0); 1118 i = 0; 1119 } 1120 tx_desc->read.olinfo_status = 0; 1121 1122 dma += IGC_MAX_DATA_PER_TXD; 1123 size -= IGC_MAX_DATA_PER_TXD; 1124 1125 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1126 } 1127 1128 if (likely(!data_len)) 1129 break; 1130 1131 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 1132 1133 i++; 1134 tx_desc++; 1135 if (i == tx_ring->count) { 1136 tx_desc = IGC_TX_DESC(tx_ring, 0); 1137 i = 0; 1138 } 1139 tx_desc->read.olinfo_status = 0; 1140 1141 size = skb_frag_size(frag); 1142 data_len -= size; 1143 1144 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 1145 size, DMA_TO_DEVICE); 1146 1147 tx_buffer = &tx_ring->tx_buffer_info[i]; 1148 } 1149 1150 /* write last descriptor with RS and EOP bits */ 1151 cmd_type |= size | IGC_TXD_DCMD; 1152 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1153 1154 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1155 1156 /* set the timestamp */ 1157 first->time_stamp = jiffies; 1158 1159 skb_tx_timestamp(skb); 1160 1161 /* Force memory writes to complete before letting h/w know there 1162 * are new descriptors to fetch. (Only applicable for weak-ordered 1163 * memory model archs, such as IA-64). 1164 * 1165 * We also need this memory barrier to make certain all of the 1166 * status bits have been updated before next_to_watch is written. 1167 */ 1168 wmb(); 1169 1170 /* set next_to_watch value indicating a packet is present */ 1171 first->next_to_watch = tx_desc; 1172 1173 i++; 1174 if (i == tx_ring->count) 1175 i = 0; 1176 1177 tx_ring->next_to_use = i; 1178 1179 /* Make sure there is space in the ring for the next send. */ 1180 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 1181 1182 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1183 writel(i, tx_ring->tail); 1184 } 1185 1186 return 0; 1187 dma_error: 1188 netdev_err(tx_ring->netdev, "TX DMA map failed\n"); 1189 tx_buffer = &tx_ring->tx_buffer_info[i]; 1190 1191 /* clear dma mappings for failed tx_buffer_info map */ 1192 while (tx_buffer != first) { 1193 if (dma_unmap_len(tx_buffer, len)) 1194 dma_unmap_page(tx_ring->dev, 1195 dma_unmap_addr(tx_buffer, dma), 1196 dma_unmap_len(tx_buffer, len), 1197 DMA_TO_DEVICE); 1198 dma_unmap_len_set(tx_buffer, len, 0); 1199 1200 if (i-- == 0) 1201 i += tx_ring->count; 1202 tx_buffer = &tx_ring->tx_buffer_info[i]; 1203 } 1204 1205 if (dma_unmap_len(tx_buffer, len)) 1206 dma_unmap_single(tx_ring->dev, 1207 dma_unmap_addr(tx_buffer, dma), 1208 dma_unmap_len(tx_buffer, len), 1209 DMA_TO_DEVICE); 1210 dma_unmap_len_set(tx_buffer, len, 0); 1211 1212 dev_kfree_skb_any(tx_buffer->skb); 1213 tx_buffer->skb = NULL; 1214 1215 tx_ring->next_to_use = i; 1216 1217 return -1; 1218 } 1219 1220 static int igc_tso(struct igc_ring *tx_ring, 1221 struct igc_tx_buffer *first, 1222 u8 *hdr_len) 1223 { 1224 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 1225 struct sk_buff *skb = first->skb; 1226 union { 1227 struct iphdr *v4; 1228 struct ipv6hdr *v6; 1229 unsigned char *hdr; 1230 } ip; 1231 union { 1232 struct tcphdr *tcp; 1233 struct udphdr *udp; 1234 unsigned char *hdr; 1235 } l4; 1236 u32 paylen, l4_offset; 1237 int err; 1238 1239 if (skb->ip_summed != CHECKSUM_PARTIAL) 1240 return 0; 1241 1242 if (!skb_is_gso(skb)) 1243 return 0; 1244 1245 err = skb_cow_head(skb, 0); 1246 if (err < 0) 1247 return err; 1248 1249 ip.hdr = skb_network_header(skb); 1250 l4.hdr = skb_checksum_start(skb); 1251 1252 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 1253 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 1254 1255 /* initialize outer IP header fields */ 1256 if (ip.v4->version == 4) { 1257 unsigned char *csum_start = skb_checksum_start(skb); 1258 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 1259 1260 /* IP header will have to cancel out any data that 1261 * is not a part of the outer IP header 1262 */ 1263 ip.v4->check = csum_fold(csum_partial(trans_start, 1264 csum_start - trans_start, 1265 0)); 1266 type_tucmd |= IGC_ADVTXD_TUCMD_IPV4; 1267 1268 ip.v4->tot_len = 0; 1269 first->tx_flags |= IGC_TX_FLAGS_TSO | 1270 IGC_TX_FLAGS_CSUM | 1271 IGC_TX_FLAGS_IPV4; 1272 } else { 1273 ip.v6->payload_len = 0; 1274 first->tx_flags |= IGC_TX_FLAGS_TSO | 1275 IGC_TX_FLAGS_CSUM; 1276 } 1277 1278 /* determine offset of inner transport header */ 1279 l4_offset = l4.hdr - skb->data; 1280 1281 /* remove payload length from inner checksum */ 1282 paylen = skb->len - l4_offset; 1283 if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) { 1284 /* compute length of segmentation header */ 1285 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 1286 csum_replace_by_diff(&l4.tcp->check, 1287 (__force __wsum)htonl(paylen)); 1288 } else { 1289 /* compute length of segmentation header */ 1290 *hdr_len = sizeof(*l4.udp) + l4_offset; 1291 csum_replace_by_diff(&l4.udp->check, 1292 (__force __wsum)htonl(paylen)); 1293 } 1294 1295 /* update gso size and bytecount with header size */ 1296 first->gso_segs = skb_shinfo(skb)->gso_segs; 1297 first->bytecount += (first->gso_segs - 1) * *hdr_len; 1298 1299 /* MSS L4LEN IDX */ 1300 mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT; 1301 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT; 1302 1303 /* VLAN MACLEN IPLEN */ 1304 vlan_macip_lens = l4.hdr - ip.hdr; 1305 vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT; 1306 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1307 1308 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, 1309 type_tucmd, mss_l4len_idx); 1310 1311 return 1; 1312 } 1313 1314 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1315 struct igc_ring *tx_ring) 1316 { 1317 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1318 __be16 protocol = vlan_get_protocol(skb); 1319 struct igc_tx_buffer *first; 1320 u32 tx_flags = 0; 1321 unsigned short f; 1322 u8 hdr_len = 0; 1323 int tso = 0; 1324 1325 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1326 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1327 * + 2 desc gap to keep tail from touching head, 1328 * + 1 desc for context descriptor, 1329 * otherwise try next time 1330 */ 1331 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1332 count += TXD_USE_COUNT(skb_frag_size( 1333 &skb_shinfo(skb)->frags[f])); 1334 1335 if (igc_maybe_stop_tx(tx_ring, count + 3)) { 1336 /* this is a hard error */ 1337 return NETDEV_TX_BUSY; 1338 } 1339 1340 /* record the location of the first descriptor for this packet */ 1341 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1342 first->skb = skb; 1343 first->bytecount = skb->len; 1344 first->gso_segs = 1; 1345 1346 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 1347 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 1348 1349 /* FIXME: add support for retrieving timestamps from 1350 * the other timer registers before skipping the 1351 * timestamping request. 1352 */ 1353 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && 1354 !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS, 1355 &adapter->state)) { 1356 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1357 tx_flags |= IGC_TX_FLAGS_TSTAMP; 1358 1359 adapter->ptp_tx_skb = skb_get(skb); 1360 adapter->ptp_tx_start = jiffies; 1361 } else { 1362 adapter->tx_hwtstamp_skipped++; 1363 } 1364 } 1365 1366 /* record initial flags and protocol */ 1367 first->tx_flags = tx_flags; 1368 first->protocol = protocol; 1369 1370 tso = igc_tso(tx_ring, first, &hdr_len); 1371 if (tso < 0) 1372 goto out_drop; 1373 else if (!tso) 1374 igc_tx_csum(tx_ring, first); 1375 1376 igc_tx_map(tx_ring, first, hdr_len); 1377 1378 return NETDEV_TX_OK; 1379 1380 out_drop: 1381 dev_kfree_skb_any(first->skb); 1382 first->skb = NULL; 1383 1384 return NETDEV_TX_OK; 1385 } 1386 1387 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1388 struct sk_buff *skb) 1389 { 1390 unsigned int r_idx = skb->queue_mapping; 1391 1392 if (r_idx >= adapter->num_tx_queues) 1393 r_idx = r_idx % adapter->num_tx_queues; 1394 1395 return adapter->tx_ring[r_idx]; 1396 } 1397 1398 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1399 struct net_device *netdev) 1400 { 1401 struct igc_adapter *adapter = netdev_priv(netdev); 1402 1403 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1404 * in order to meet this minimum size requirement. 1405 */ 1406 if (skb->len < 17) { 1407 if (skb_padto(skb, 17)) 1408 return NETDEV_TX_OK; 1409 skb->len = 17; 1410 } 1411 1412 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1413 } 1414 1415 static void igc_rx_checksum(struct igc_ring *ring, 1416 union igc_adv_rx_desc *rx_desc, 1417 struct sk_buff *skb) 1418 { 1419 skb_checksum_none_assert(skb); 1420 1421 /* Ignore Checksum bit is set */ 1422 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) 1423 return; 1424 1425 /* Rx checksum disabled via ethtool */ 1426 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1427 return; 1428 1429 /* TCP/UDP checksum error bit is set */ 1430 if (igc_test_staterr(rx_desc, 1431 IGC_RXDEXT_STATERR_TCPE | 1432 IGC_RXDEXT_STATERR_IPE)) { 1433 /* work around errata with sctp packets where the TCPE aka 1434 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 1435 * packets (aka let the stack check the crc32c) 1436 */ 1437 if (!(skb->len == 60 && 1438 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 1439 u64_stats_update_begin(&ring->rx_syncp); 1440 ring->rx_stats.csum_err++; 1441 u64_stats_update_end(&ring->rx_syncp); 1442 } 1443 /* let the stack verify checksum errors */ 1444 return; 1445 } 1446 /* It must be a TCP or UDP packet with a valid checksum */ 1447 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | 1448 IGC_RXD_STAT_UDPCS)) 1449 skb->ip_summed = CHECKSUM_UNNECESSARY; 1450 1451 netdev_dbg(ring->netdev, "cksum success: bits %08X\n", 1452 le32_to_cpu(rx_desc->wb.upper.status_error)); 1453 } 1454 1455 static inline void igc_rx_hash(struct igc_ring *ring, 1456 union igc_adv_rx_desc *rx_desc, 1457 struct sk_buff *skb) 1458 { 1459 if (ring->netdev->features & NETIF_F_RXHASH) 1460 skb_set_hash(skb, 1461 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 1462 PKT_HASH_TYPE_L3); 1463 } 1464 1465 /** 1466 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1467 * @rx_ring: rx descriptor ring packet is being transacted on 1468 * @rx_desc: pointer to the EOP Rx descriptor 1469 * @skb: pointer to current skb being populated 1470 * 1471 * This function checks the ring, descriptor, and packet information in order 1472 * to populate the hash, checksum, VLAN, protocol, and other fields within the 1473 * skb. 1474 */ 1475 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1476 union igc_adv_rx_desc *rx_desc, 1477 struct sk_buff *skb) 1478 { 1479 igc_rx_hash(rx_ring, rx_desc, skb); 1480 1481 igc_rx_checksum(rx_ring, rx_desc, skb); 1482 1483 skb_record_rx_queue(skb, rx_ring->queue_index); 1484 1485 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1486 } 1487 1488 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1489 const unsigned int size) 1490 { 1491 struct igc_rx_buffer *rx_buffer; 1492 1493 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1494 prefetchw(rx_buffer->page); 1495 1496 /* we are reusing so sync this buffer for CPU use */ 1497 dma_sync_single_range_for_cpu(rx_ring->dev, 1498 rx_buffer->dma, 1499 rx_buffer->page_offset, 1500 size, 1501 DMA_FROM_DEVICE); 1502 1503 rx_buffer->pagecnt_bias--; 1504 1505 return rx_buffer; 1506 } 1507 1508 /** 1509 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1510 * @rx_ring: rx descriptor ring to transact packets on 1511 * @rx_buffer: buffer containing page to add 1512 * @skb: sk_buff to place the data into 1513 * @size: size of buffer to be added 1514 * 1515 * This function will add the data contained in rx_buffer->page to the skb. 1516 */ 1517 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1518 struct igc_rx_buffer *rx_buffer, 1519 struct sk_buff *skb, 1520 unsigned int size) 1521 { 1522 #if (PAGE_SIZE < 8192) 1523 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1524 1525 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1526 rx_buffer->page_offset, size, truesize); 1527 rx_buffer->page_offset ^= truesize; 1528 #else 1529 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1530 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1531 SKB_DATA_ALIGN(size); 1532 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1533 rx_buffer->page_offset, size, truesize); 1534 rx_buffer->page_offset += truesize; 1535 #endif 1536 } 1537 1538 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1539 struct igc_rx_buffer *rx_buffer, 1540 union igc_adv_rx_desc *rx_desc, 1541 unsigned int size) 1542 { 1543 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1544 #if (PAGE_SIZE < 8192) 1545 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1546 #else 1547 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1548 SKB_DATA_ALIGN(IGC_SKB_PAD + size); 1549 #endif 1550 struct sk_buff *skb; 1551 1552 /* prefetch first cache line of first page */ 1553 prefetch(va); 1554 #if L1_CACHE_BYTES < 128 1555 prefetch(va + L1_CACHE_BYTES); 1556 #endif 1557 1558 /* build an skb around the page buffer */ 1559 skb = build_skb(va - IGC_SKB_PAD, truesize); 1560 if (unlikely(!skb)) 1561 return NULL; 1562 1563 /* update pointers within the skb to store the data */ 1564 skb_reserve(skb, IGC_SKB_PAD); 1565 __skb_put(skb, size); 1566 1567 /* update buffer offset */ 1568 #if (PAGE_SIZE < 8192) 1569 rx_buffer->page_offset ^= truesize; 1570 #else 1571 rx_buffer->page_offset += truesize; 1572 #endif 1573 1574 return skb; 1575 } 1576 1577 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1578 struct igc_rx_buffer *rx_buffer, 1579 union igc_adv_rx_desc *rx_desc, 1580 unsigned int size) 1581 { 1582 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1583 #if (PAGE_SIZE < 8192) 1584 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1585 #else 1586 unsigned int truesize = SKB_DATA_ALIGN(size); 1587 #endif 1588 unsigned int headlen; 1589 struct sk_buff *skb; 1590 1591 /* prefetch first cache line of first page */ 1592 prefetch(va); 1593 #if L1_CACHE_BYTES < 128 1594 prefetch(va + L1_CACHE_BYTES); 1595 #endif 1596 1597 /* allocate a skb to store the frags */ 1598 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); 1599 if (unlikely(!skb)) 1600 return NULL; 1601 1602 if (unlikely(igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP))) { 1603 igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb); 1604 va += IGC_TS_HDR_LEN; 1605 size -= IGC_TS_HDR_LEN; 1606 } 1607 1608 /* Determine available headroom for copy */ 1609 headlen = size; 1610 if (headlen > IGC_RX_HDR_LEN) 1611 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 1612 1613 /* align pull length to size of long to optimize memcpy performance */ 1614 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 1615 1616 /* update all of the pointers */ 1617 size -= headlen; 1618 if (size) { 1619 skb_add_rx_frag(skb, 0, rx_buffer->page, 1620 (va + headlen) - page_address(rx_buffer->page), 1621 size, truesize); 1622 #if (PAGE_SIZE < 8192) 1623 rx_buffer->page_offset ^= truesize; 1624 #else 1625 rx_buffer->page_offset += truesize; 1626 #endif 1627 } else { 1628 rx_buffer->pagecnt_bias++; 1629 } 1630 1631 return skb; 1632 } 1633 1634 /** 1635 * igc_reuse_rx_page - page flip buffer and store it back on the ring 1636 * @rx_ring: rx descriptor ring to store buffers on 1637 * @old_buff: donor buffer to have page reused 1638 * 1639 * Synchronizes page for reuse by the adapter 1640 */ 1641 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 1642 struct igc_rx_buffer *old_buff) 1643 { 1644 u16 nta = rx_ring->next_to_alloc; 1645 struct igc_rx_buffer *new_buff; 1646 1647 new_buff = &rx_ring->rx_buffer_info[nta]; 1648 1649 /* update, and store next to alloc */ 1650 nta++; 1651 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1652 1653 /* Transfer page from old buffer to new buffer. 1654 * Move each member individually to avoid possible store 1655 * forwarding stalls. 1656 */ 1657 new_buff->dma = old_buff->dma; 1658 new_buff->page = old_buff->page; 1659 new_buff->page_offset = old_buff->page_offset; 1660 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1661 } 1662 1663 static inline bool igc_page_is_reserved(struct page *page) 1664 { 1665 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 1666 } 1667 1668 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) 1669 { 1670 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1671 struct page *page = rx_buffer->page; 1672 1673 /* avoid re-using remote pages */ 1674 if (unlikely(igc_page_is_reserved(page))) 1675 return false; 1676 1677 #if (PAGE_SIZE < 8192) 1678 /* if we are only owner of page we can reuse it */ 1679 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 1680 return false; 1681 #else 1682 #define IGC_LAST_OFFSET \ 1683 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 1684 1685 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 1686 return false; 1687 #endif 1688 1689 /* If we have drained the page fragment pool we need to update 1690 * the pagecnt_bias and page count so that we fully restock the 1691 * number of references the driver holds. 1692 */ 1693 if (unlikely(!pagecnt_bias)) { 1694 page_ref_add(page, USHRT_MAX); 1695 rx_buffer->pagecnt_bias = USHRT_MAX; 1696 } 1697 1698 return true; 1699 } 1700 1701 /** 1702 * igc_is_non_eop - process handling of non-EOP buffers 1703 * @rx_ring: Rx ring being processed 1704 * @rx_desc: Rx descriptor for current buffer 1705 * 1706 * This function updates next to clean. If the buffer is an EOP buffer 1707 * this function exits returning false, otherwise it will place the 1708 * sk_buff in the next buffer to be chained and return true indicating 1709 * that this is in fact a non-EOP buffer. 1710 */ 1711 static bool igc_is_non_eop(struct igc_ring *rx_ring, 1712 union igc_adv_rx_desc *rx_desc) 1713 { 1714 u32 ntc = rx_ring->next_to_clean + 1; 1715 1716 /* fetch, update, and store next to clean */ 1717 ntc = (ntc < rx_ring->count) ? ntc : 0; 1718 rx_ring->next_to_clean = ntc; 1719 1720 prefetch(IGC_RX_DESC(rx_ring, ntc)); 1721 1722 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 1723 return false; 1724 1725 return true; 1726 } 1727 1728 /** 1729 * igc_cleanup_headers - Correct corrupted or empty headers 1730 * @rx_ring: rx descriptor ring packet is being transacted on 1731 * @rx_desc: pointer to the EOP Rx descriptor 1732 * @skb: pointer to current skb being fixed 1733 * 1734 * Address the case where we are pulling data in on pages only 1735 * and as such no data is present in the skb header. 1736 * 1737 * In addition if skb is not at least 60 bytes we need to pad it so that 1738 * it is large enough to qualify as a valid Ethernet frame. 1739 * 1740 * Returns true if an error was encountered and skb was freed. 1741 */ 1742 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 1743 union igc_adv_rx_desc *rx_desc, 1744 struct sk_buff *skb) 1745 { 1746 if (unlikely((igc_test_staterr(rx_desc, 1747 IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) { 1748 struct net_device *netdev = rx_ring->netdev; 1749 1750 if (!(netdev->features & NETIF_F_RXALL)) { 1751 dev_kfree_skb_any(skb); 1752 return true; 1753 } 1754 } 1755 1756 /* if eth_skb_pad returns an error the skb was freed */ 1757 if (eth_skb_pad(skb)) 1758 return true; 1759 1760 return false; 1761 } 1762 1763 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 1764 struct igc_rx_buffer *rx_buffer) 1765 { 1766 if (igc_can_reuse_rx_page(rx_buffer)) { 1767 /* hand second half of page back to the ring */ 1768 igc_reuse_rx_page(rx_ring, rx_buffer); 1769 } else { 1770 /* We are not reusing the buffer so unmap it and free 1771 * any references we are holding to it 1772 */ 1773 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1774 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 1775 IGC_RX_DMA_ATTR); 1776 __page_frag_cache_drain(rx_buffer->page, 1777 rx_buffer->pagecnt_bias); 1778 } 1779 1780 /* clear contents of rx_buffer */ 1781 rx_buffer->page = NULL; 1782 } 1783 1784 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 1785 { 1786 return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; 1787 } 1788 1789 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 1790 struct igc_rx_buffer *bi) 1791 { 1792 struct page *page = bi->page; 1793 dma_addr_t dma; 1794 1795 /* since we are recycling buffers we should seldom need to alloc */ 1796 if (likely(page)) 1797 return true; 1798 1799 /* alloc new page for storage */ 1800 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 1801 if (unlikely(!page)) { 1802 rx_ring->rx_stats.alloc_failed++; 1803 return false; 1804 } 1805 1806 /* map page for use */ 1807 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1808 igc_rx_pg_size(rx_ring), 1809 DMA_FROM_DEVICE, 1810 IGC_RX_DMA_ATTR); 1811 1812 /* if mapping failed free memory back to system since 1813 * there isn't much point in holding memory we can't use 1814 */ 1815 if (dma_mapping_error(rx_ring->dev, dma)) { 1816 __free_page(page); 1817 1818 rx_ring->rx_stats.alloc_failed++; 1819 return false; 1820 } 1821 1822 bi->dma = dma; 1823 bi->page = page; 1824 bi->page_offset = igc_rx_offset(rx_ring); 1825 bi->pagecnt_bias = 1; 1826 1827 return true; 1828 } 1829 1830 /** 1831 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 1832 * @rx_ring: rx descriptor ring 1833 * @cleaned_count: number of buffers to clean 1834 */ 1835 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 1836 { 1837 union igc_adv_rx_desc *rx_desc; 1838 u16 i = rx_ring->next_to_use; 1839 struct igc_rx_buffer *bi; 1840 u16 bufsz; 1841 1842 /* nothing to do */ 1843 if (!cleaned_count) 1844 return; 1845 1846 rx_desc = IGC_RX_DESC(rx_ring, i); 1847 bi = &rx_ring->rx_buffer_info[i]; 1848 i -= rx_ring->count; 1849 1850 bufsz = igc_rx_bufsz(rx_ring); 1851 1852 do { 1853 if (!igc_alloc_mapped_page(rx_ring, bi)) 1854 break; 1855 1856 /* sync the buffer for use by the device */ 1857 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1858 bi->page_offset, bufsz, 1859 DMA_FROM_DEVICE); 1860 1861 /* Refresh the desc even if buffer_addrs didn't change 1862 * because each write-back erases this info. 1863 */ 1864 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1865 1866 rx_desc++; 1867 bi++; 1868 i++; 1869 if (unlikely(!i)) { 1870 rx_desc = IGC_RX_DESC(rx_ring, 0); 1871 bi = rx_ring->rx_buffer_info; 1872 i -= rx_ring->count; 1873 } 1874 1875 /* clear the length for the next_to_use descriptor */ 1876 rx_desc->wb.upper.length = 0; 1877 1878 cleaned_count--; 1879 } while (cleaned_count); 1880 1881 i += rx_ring->count; 1882 1883 if (rx_ring->next_to_use != i) { 1884 /* record the next descriptor to use */ 1885 rx_ring->next_to_use = i; 1886 1887 /* update next to alloc since we have filled the ring */ 1888 rx_ring->next_to_alloc = i; 1889 1890 /* Force memory writes to complete before letting h/w 1891 * know there are new descriptors to fetch. (Only 1892 * applicable for weak-ordered memory model archs, 1893 * such as IA-64). 1894 */ 1895 wmb(); 1896 writel(i, rx_ring->tail); 1897 } 1898 } 1899 1900 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 1901 { 1902 unsigned int total_bytes = 0, total_packets = 0; 1903 struct igc_ring *rx_ring = q_vector->rx.ring; 1904 struct sk_buff *skb = rx_ring->skb; 1905 u16 cleaned_count = igc_desc_unused(rx_ring); 1906 1907 while (likely(total_packets < budget)) { 1908 union igc_adv_rx_desc *rx_desc; 1909 struct igc_rx_buffer *rx_buffer; 1910 unsigned int size; 1911 1912 /* return some buffers to hardware, one at a time is too slow */ 1913 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 1914 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1915 cleaned_count = 0; 1916 } 1917 1918 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 1919 size = le16_to_cpu(rx_desc->wb.upper.length); 1920 if (!size) 1921 break; 1922 1923 /* This memory barrier is needed to keep us from reading 1924 * any other fields out of the rx_desc until we know the 1925 * descriptor has been written back 1926 */ 1927 dma_rmb(); 1928 1929 rx_buffer = igc_get_rx_buffer(rx_ring, size); 1930 1931 /* retrieve a buffer from the ring */ 1932 if (skb) 1933 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 1934 else if (ring_uses_build_skb(rx_ring)) 1935 skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); 1936 else 1937 skb = igc_construct_skb(rx_ring, rx_buffer, 1938 rx_desc, size); 1939 1940 /* exit if we failed to retrieve a buffer */ 1941 if (!skb) { 1942 rx_ring->rx_stats.alloc_failed++; 1943 rx_buffer->pagecnt_bias++; 1944 break; 1945 } 1946 1947 igc_put_rx_buffer(rx_ring, rx_buffer); 1948 cleaned_count++; 1949 1950 /* fetch next buffer in frame if non-eop */ 1951 if (igc_is_non_eop(rx_ring, rx_desc)) 1952 continue; 1953 1954 /* verify the packet layout is correct */ 1955 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) { 1956 skb = NULL; 1957 continue; 1958 } 1959 1960 /* probably a little skewed due to removing CRC */ 1961 total_bytes += skb->len; 1962 1963 /* populate checksum, VLAN, and protocol */ 1964 igc_process_skb_fields(rx_ring, rx_desc, skb); 1965 1966 napi_gro_receive(&q_vector->napi, skb); 1967 1968 /* reset skb pointer */ 1969 skb = NULL; 1970 1971 /* update budget accounting */ 1972 total_packets++; 1973 } 1974 1975 /* place incomplete frames back on ring for completion */ 1976 rx_ring->skb = skb; 1977 1978 u64_stats_update_begin(&rx_ring->rx_syncp); 1979 rx_ring->rx_stats.packets += total_packets; 1980 rx_ring->rx_stats.bytes += total_bytes; 1981 u64_stats_update_end(&rx_ring->rx_syncp); 1982 q_vector->rx.total_packets += total_packets; 1983 q_vector->rx.total_bytes += total_bytes; 1984 1985 if (cleaned_count) 1986 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1987 1988 return total_packets; 1989 } 1990 1991 /** 1992 * igc_clean_tx_irq - Reclaim resources after transmit completes 1993 * @q_vector: pointer to q_vector containing needed info 1994 * @napi_budget: Used to determine if we are in netpoll 1995 * 1996 * returns true if ring is completely cleaned 1997 */ 1998 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 1999 { 2000 struct igc_adapter *adapter = q_vector->adapter; 2001 unsigned int total_bytes = 0, total_packets = 0; 2002 unsigned int budget = q_vector->tx.work_limit; 2003 struct igc_ring *tx_ring = q_vector->tx.ring; 2004 unsigned int i = tx_ring->next_to_clean; 2005 struct igc_tx_buffer *tx_buffer; 2006 union igc_adv_tx_desc *tx_desc; 2007 2008 if (test_bit(__IGC_DOWN, &adapter->state)) 2009 return true; 2010 2011 tx_buffer = &tx_ring->tx_buffer_info[i]; 2012 tx_desc = IGC_TX_DESC(tx_ring, i); 2013 i -= tx_ring->count; 2014 2015 do { 2016 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 2017 2018 /* if next_to_watch is not set then there is no work pending */ 2019 if (!eop_desc) 2020 break; 2021 2022 /* prevent any other reads prior to eop_desc */ 2023 smp_rmb(); 2024 2025 /* if DD is not set pending work has not been completed */ 2026 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 2027 break; 2028 2029 /* clear next_to_watch to prevent false hangs */ 2030 tx_buffer->next_to_watch = NULL; 2031 2032 /* update the statistics for this packet */ 2033 total_bytes += tx_buffer->bytecount; 2034 total_packets += tx_buffer->gso_segs; 2035 2036 /* free the skb */ 2037 napi_consume_skb(tx_buffer->skb, napi_budget); 2038 2039 /* unmap skb header data */ 2040 dma_unmap_single(tx_ring->dev, 2041 dma_unmap_addr(tx_buffer, dma), 2042 dma_unmap_len(tx_buffer, len), 2043 DMA_TO_DEVICE); 2044 2045 /* clear tx_buffer data */ 2046 dma_unmap_len_set(tx_buffer, len, 0); 2047 2048 /* clear last DMA location and unmap remaining buffers */ 2049 while (tx_desc != eop_desc) { 2050 tx_buffer++; 2051 tx_desc++; 2052 i++; 2053 if (unlikely(!i)) { 2054 i -= tx_ring->count; 2055 tx_buffer = tx_ring->tx_buffer_info; 2056 tx_desc = IGC_TX_DESC(tx_ring, 0); 2057 } 2058 2059 /* unmap any remaining paged data */ 2060 if (dma_unmap_len(tx_buffer, len)) { 2061 dma_unmap_page(tx_ring->dev, 2062 dma_unmap_addr(tx_buffer, dma), 2063 dma_unmap_len(tx_buffer, len), 2064 DMA_TO_DEVICE); 2065 dma_unmap_len_set(tx_buffer, len, 0); 2066 } 2067 } 2068 2069 /* move us one more past the eop_desc for start of next pkt */ 2070 tx_buffer++; 2071 tx_desc++; 2072 i++; 2073 if (unlikely(!i)) { 2074 i -= tx_ring->count; 2075 tx_buffer = tx_ring->tx_buffer_info; 2076 tx_desc = IGC_TX_DESC(tx_ring, 0); 2077 } 2078 2079 /* issue prefetch for next Tx descriptor */ 2080 prefetch(tx_desc); 2081 2082 /* update budget accounting */ 2083 budget--; 2084 } while (likely(budget)); 2085 2086 netdev_tx_completed_queue(txring_txq(tx_ring), 2087 total_packets, total_bytes); 2088 2089 i += tx_ring->count; 2090 tx_ring->next_to_clean = i; 2091 u64_stats_update_begin(&tx_ring->tx_syncp); 2092 tx_ring->tx_stats.bytes += total_bytes; 2093 tx_ring->tx_stats.packets += total_packets; 2094 u64_stats_update_end(&tx_ring->tx_syncp); 2095 q_vector->tx.total_bytes += total_bytes; 2096 q_vector->tx.total_packets += total_packets; 2097 2098 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 2099 struct igc_hw *hw = &adapter->hw; 2100 2101 /* Detect a transmit hang in hardware, this serializes the 2102 * check with the clearing of time_stamp and movement of i 2103 */ 2104 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 2105 if (tx_buffer->next_to_watch && 2106 time_after(jiffies, tx_buffer->time_stamp + 2107 (adapter->tx_timeout_factor * HZ)) && 2108 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { 2109 /* detected Tx unit hang */ 2110 netdev_err(tx_ring->netdev, 2111 "Detected Tx Unit Hang\n" 2112 " Tx Queue <%d>\n" 2113 " TDH <%x>\n" 2114 " TDT <%x>\n" 2115 " next_to_use <%x>\n" 2116 " next_to_clean <%x>\n" 2117 "buffer_info[next_to_clean]\n" 2118 " time_stamp <%lx>\n" 2119 " next_to_watch <%p>\n" 2120 " jiffies <%lx>\n" 2121 " desc.status <%x>\n", 2122 tx_ring->queue_index, 2123 rd32(IGC_TDH(tx_ring->reg_idx)), 2124 readl(tx_ring->tail), 2125 tx_ring->next_to_use, 2126 tx_ring->next_to_clean, 2127 tx_buffer->time_stamp, 2128 tx_buffer->next_to_watch, 2129 jiffies, 2130 tx_buffer->next_to_watch->wb.status); 2131 netif_stop_subqueue(tx_ring->netdev, 2132 tx_ring->queue_index); 2133 2134 /* we are about to reset, no point in enabling stuff */ 2135 return true; 2136 } 2137 } 2138 2139 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 2140 if (unlikely(total_packets && 2141 netif_carrier_ok(tx_ring->netdev) && 2142 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 2143 /* Make sure that anybody stopping the queue after this 2144 * sees the new next_to_clean. 2145 */ 2146 smp_mb(); 2147 if (__netif_subqueue_stopped(tx_ring->netdev, 2148 tx_ring->queue_index) && 2149 !(test_bit(__IGC_DOWN, &adapter->state))) { 2150 netif_wake_subqueue(tx_ring->netdev, 2151 tx_ring->queue_index); 2152 2153 u64_stats_update_begin(&tx_ring->tx_syncp); 2154 tx_ring->tx_stats.restart_queue++; 2155 u64_stats_update_end(&tx_ring->tx_syncp); 2156 } 2157 } 2158 2159 return !!budget; 2160 } 2161 2162 static int igc_find_mac_filter(struct igc_adapter *adapter, 2163 enum igc_mac_filter_type type, const u8 *addr) 2164 { 2165 struct igc_hw *hw = &adapter->hw; 2166 int max_entries = hw->mac.rar_entry_count; 2167 u32 ral, rah; 2168 int i; 2169 2170 for (i = 0; i < max_entries; i++) { 2171 ral = rd32(IGC_RAL(i)); 2172 rah = rd32(IGC_RAH(i)); 2173 2174 if (!(rah & IGC_RAH_AV)) 2175 continue; 2176 if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type) 2177 continue; 2178 if ((rah & IGC_RAH_RAH_MASK) != 2179 le16_to_cpup((__le16 *)(addr + 4))) 2180 continue; 2181 if (ral != le32_to_cpup((__le32 *)(addr))) 2182 continue; 2183 2184 return i; 2185 } 2186 2187 return -1; 2188 } 2189 2190 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) 2191 { 2192 struct igc_hw *hw = &adapter->hw; 2193 int max_entries = hw->mac.rar_entry_count; 2194 u32 rah; 2195 int i; 2196 2197 for (i = 0; i < max_entries; i++) { 2198 rah = rd32(IGC_RAH(i)); 2199 2200 if (!(rah & IGC_RAH_AV)) 2201 return i; 2202 } 2203 2204 return -1; 2205 } 2206 2207 /** 2208 * igc_add_mac_filter() - Add MAC address filter 2209 * @adapter: Pointer to adapter where the filter should be added 2210 * @type: MAC address filter type (source or destination) 2211 * @addr: MAC address 2212 * @queue: If non-negative, queue assignment feature is enabled and frames 2213 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2214 * assignment is disabled. 2215 * 2216 * Return: 0 in case of success, negative errno code otherwise. 2217 */ 2218 static int igc_add_mac_filter(struct igc_adapter *adapter, 2219 enum igc_mac_filter_type type, const u8 *addr, 2220 int queue) 2221 { 2222 struct net_device *dev = adapter->netdev; 2223 int index; 2224 2225 index = igc_find_mac_filter(adapter, type, addr); 2226 if (index >= 0) 2227 goto update_filter; 2228 2229 index = igc_get_avail_mac_filter_slot(adapter); 2230 if (index < 0) 2231 return -ENOSPC; 2232 2233 netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n", 2234 index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2235 addr, queue); 2236 2237 update_filter: 2238 igc_set_mac_filter_hw(adapter, index, type, addr, queue); 2239 return 0; 2240 } 2241 2242 /** 2243 * igc_del_mac_filter() - Delete MAC address filter 2244 * @adapter: Pointer to adapter where the filter should be deleted from 2245 * @type: MAC address filter type (source or destination) 2246 * @addr: MAC address 2247 */ 2248 static void igc_del_mac_filter(struct igc_adapter *adapter, 2249 enum igc_mac_filter_type type, const u8 *addr) 2250 { 2251 struct net_device *dev = adapter->netdev; 2252 int index; 2253 2254 index = igc_find_mac_filter(adapter, type, addr); 2255 if (index < 0) 2256 return; 2257 2258 if (index == 0) { 2259 /* If this is the default filter, we don't actually delete it. 2260 * We just reset to its default value i.e. disable queue 2261 * assignment. 2262 */ 2263 netdev_dbg(dev, "Disable default MAC filter queue assignment"); 2264 2265 igc_set_mac_filter_hw(adapter, 0, type, addr, -1); 2266 } else { 2267 netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n", 2268 index, 2269 type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2270 addr); 2271 2272 igc_clear_mac_filter_hw(adapter, index); 2273 } 2274 } 2275 2276 /** 2277 * igc_add_vlan_prio_filter() - Add VLAN priority filter 2278 * @adapter: Pointer to adapter where the filter should be added 2279 * @prio: VLAN priority value 2280 * @queue: Queue number which matching frames are assigned to 2281 * 2282 * Return: 0 in case of success, negative errno code otherwise. 2283 */ 2284 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, 2285 int queue) 2286 { 2287 struct net_device *dev = adapter->netdev; 2288 struct igc_hw *hw = &adapter->hw; 2289 u32 vlanpqf; 2290 2291 vlanpqf = rd32(IGC_VLANPQF); 2292 2293 if (vlanpqf & IGC_VLANPQF_VALID(prio)) { 2294 netdev_dbg(dev, "VLAN priority filter already in use\n"); 2295 return -EEXIST; 2296 } 2297 2298 vlanpqf |= IGC_VLANPQF_QSEL(prio, queue); 2299 vlanpqf |= IGC_VLANPQF_VALID(prio); 2300 2301 wr32(IGC_VLANPQF, vlanpqf); 2302 2303 netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n", 2304 prio, queue); 2305 return 0; 2306 } 2307 2308 /** 2309 * igc_del_vlan_prio_filter() - Delete VLAN priority filter 2310 * @adapter: Pointer to adapter where the filter should be deleted from 2311 * @prio: VLAN priority value 2312 */ 2313 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) 2314 { 2315 struct igc_hw *hw = &adapter->hw; 2316 u32 vlanpqf; 2317 2318 vlanpqf = rd32(IGC_VLANPQF); 2319 2320 vlanpqf &= ~IGC_VLANPQF_VALID(prio); 2321 vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK); 2322 2323 wr32(IGC_VLANPQF, vlanpqf); 2324 2325 netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n", 2326 prio); 2327 } 2328 2329 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) 2330 { 2331 struct igc_hw *hw = &adapter->hw; 2332 int i; 2333 2334 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2335 u32 etqf = rd32(IGC_ETQF(i)); 2336 2337 if (!(etqf & IGC_ETQF_FILTER_ENABLE)) 2338 return i; 2339 } 2340 2341 return -1; 2342 } 2343 2344 /** 2345 * igc_add_etype_filter() - Add ethertype filter 2346 * @adapter: Pointer to adapter where the filter should be added 2347 * @etype: Ethertype value 2348 * @queue: If non-negative, queue assignment feature is enabled and frames 2349 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2350 * assignment is disabled. 2351 * 2352 * Return: 0 in case of success, negative errno code otherwise. 2353 */ 2354 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, 2355 int queue) 2356 { 2357 struct igc_hw *hw = &adapter->hw; 2358 int index; 2359 u32 etqf; 2360 2361 index = igc_get_avail_etype_filter_slot(adapter); 2362 if (index < 0) 2363 return -ENOSPC; 2364 2365 etqf = rd32(IGC_ETQF(index)); 2366 2367 etqf &= ~IGC_ETQF_ETYPE_MASK; 2368 etqf |= etype; 2369 2370 if (queue >= 0) { 2371 etqf &= ~IGC_ETQF_QUEUE_MASK; 2372 etqf |= (queue << IGC_ETQF_QUEUE_SHIFT); 2373 etqf |= IGC_ETQF_QUEUE_ENABLE; 2374 } 2375 2376 etqf |= IGC_ETQF_FILTER_ENABLE; 2377 2378 wr32(IGC_ETQF(index), etqf); 2379 2380 netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n", 2381 etype, queue); 2382 return 0; 2383 } 2384 2385 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) 2386 { 2387 struct igc_hw *hw = &adapter->hw; 2388 int i; 2389 2390 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2391 u32 etqf = rd32(IGC_ETQF(i)); 2392 2393 if ((etqf & IGC_ETQF_ETYPE_MASK) == etype) 2394 return i; 2395 } 2396 2397 return -1; 2398 } 2399 2400 /** 2401 * igc_del_etype_filter() - Delete ethertype filter 2402 * @adapter: Pointer to adapter where the filter should be deleted from 2403 * @etype: Ethertype value 2404 */ 2405 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) 2406 { 2407 struct igc_hw *hw = &adapter->hw; 2408 int index; 2409 2410 index = igc_find_etype_filter(adapter, etype); 2411 if (index < 0) 2412 return; 2413 2414 wr32(IGC_ETQF(index), 0); 2415 2416 netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n", 2417 etype); 2418 } 2419 2420 static int igc_enable_nfc_rule(struct igc_adapter *adapter, 2421 const struct igc_nfc_rule *rule) 2422 { 2423 int err; 2424 2425 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { 2426 err = igc_add_etype_filter(adapter, rule->filter.etype, 2427 rule->action); 2428 if (err) 2429 return err; 2430 } 2431 2432 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { 2433 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2434 rule->filter.src_addr, rule->action); 2435 if (err) 2436 return err; 2437 } 2438 2439 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { 2440 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2441 rule->filter.dst_addr, rule->action); 2442 if (err) 2443 return err; 2444 } 2445 2446 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2447 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2448 VLAN_PRIO_SHIFT; 2449 2450 err = igc_add_vlan_prio_filter(adapter, prio, rule->action); 2451 if (err) 2452 return err; 2453 } 2454 2455 return 0; 2456 } 2457 2458 static void igc_disable_nfc_rule(struct igc_adapter *adapter, 2459 const struct igc_nfc_rule *rule) 2460 { 2461 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) 2462 igc_del_etype_filter(adapter, rule->filter.etype); 2463 2464 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2465 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2466 VLAN_PRIO_SHIFT; 2467 2468 igc_del_vlan_prio_filter(adapter, prio); 2469 } 2470 2471 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) 2472 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2473 rule->filter.src_addr); 2474 2475 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) 2476 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2477 rule->filter.dst_addr); 2478 } 2479 2480 /** 2481 * igc_get_nfc_rule() - Get NFC rule 2482 * @adapter: Pointer to adapter 2483 * @location: Rule location 2484 * 2485 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2486 * 2487 * Return: Pointer to NFC rule at @location. If not found, NULL. 2488 */ 2489 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, 2490 u32 location) 2491 { 2492 struct igc_nfc_rule *rule; 2493 2494 list_for_each_entry(rule, &adapter->nfc_rule_list, list) { 2495 if (rule->location == location) 2496 return rule; 2497 if (rule->location > location) 2498 break; 2499 } 2500 2501 return NULL; 2502 } 2503 2504 /** 2505 * igc_del_nfc_rule() - Delete NFC rule 2506 * @adapter: Pointer to adapter 2507 * @rule: Pointer to rule to be deleted 2508 * 2509 * Disable NFC rule in hardware and delete it from adapter. 2510 * 2511 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2512 */ 2513 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2514 { 2515 igc_disable_nfc_rule(adapter, rule); 2516 2517 list_del(&rule->list); 2518 adapter->nfc_rule_count--; 2519 2520 kfree(rule); 2521 } 2522 2523 static void igc_flush_nfc_rules(struct igc_adapter *adapter) 2524 { 2525 struct igc_nfc_rule *rule, *tmp; 2526 2527 mutex_lock(&adapter->nfc_rule_lock); 2528 2529 list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list) 2530 igc_del_nfc_rule(adapter, rule); 2531 2532 mutex_unlock(&adapter->nfc_rule_lock); 2533 } 2534 2535 /** 2536 * igc_add_nfc_rule() - Add NFC rule 2537 * @adapter: Pointer to adapter 2538 * @rule: Pointer to rule to be added 2539 * 2540 * Enable NFC rule in hardware and add it to adapter. 2541 * 2542 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2543 * 2544 * Return: 0 on success, negative errno on failure. 2545 */ 2546 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2547 { 2548 struct igc_nfc_rule *pred, *cur; 2549 int err; 2550 2551 err = igc_enable_nfc_rule(adapter, rule); 2552 if (err) 2553 return err; 2554 2555 pred = NULL; 2556 list_for_each_entry(cur, &adapter->nfc_rule_list, list) { 2557 if (cur->location >= rule->location) 2558 break; 2559 pred = cur; 2560 } 2561 2562 list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list); 2563 adapter->nfc_rule_count++; 2564 return 0; 2565 } 2566 2567 static void igc_restore_nfc_rules(struct igc_adapter *adapter) 2568 { 2569 struct igc_nfc_rule *rule; 2570 2571 mutex_lock(&adapter->nfc_rule_lock); 2572 2573 list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list) 2574 igc_enable_nfc_rule(adapter, rule); 2575 2576 mutex_unlock(&adapter->nfc_rule_lock); 2577 } 2578 2579 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) 2580 { 2581 struct igc_adapter *adapter = netdev_priv(netdev); 2582 2583 return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1); 2584 } 2585 2586 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) 2587 { 2588 struct igc_adapter *adapter = netdev_priv(netdev); 2589 2590 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr); 2591 return 0; 2592 } 2593 2594 /** 2595 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 2596 * @netdev: network interface device structure 2597 * 2598 * The set_rx_mode entry point is called whenever the unicast or multicast 2599 * address lists or the network interface flags are updated. This routine is 2600 * responsible for configuring the hardware for proper unicast, multicast, 2601 * promiscuous mode, and all-multi behavior. 2602 */ 2603 static void igc_set_rx_mode(struct net_device *netdev) 2604 { 2605 struct igc_adapter *adapter = netdev_priv(netdev); 2606 struct igc_hw *hw = &adapter->hw; 2607 u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 2608 int count; 2609 2610 /* Check for Promiscuous and All Multicast modes */ 2611 if (netdev->flags & IFF_PROMISC) { 2612 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; 2613 } else { 2614 if (netdev->flags & IFF_ALLMULTI) { 2615 rctl |= IGC_RCTL_MPE; 2616 } else { 2617 /* Write addresses to the MTA, if the attempt fails 2618 * then we should just turn on promiscuous mode so 2619 * that we can at least receive multicast traffic 2620 */ 2621 count = igc_write_mc_addr_list(netdev); 2622 if (count < 0) 2623 rctl |= IGC_RCTL_MPE; 2624 } 2625 } 2626 2627 /* Write addresses to available RAR registers, if there is not 2628 * sufficient space to store all the addresses then enable 2629 * unicast promiscuous mode 2630 */ 2631 if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) 2632 rctl |= IGC_RCTL_UPE; 2633 2634 /* update state of unicast and multicast */ 2635 rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 2636 wr32(IGC_RCTL, rctl); 2637 2638 #if (PAGE_SIZE < 8192) 2639 if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) 2640 rlpml = IGC_MAX_FRAME_BUILD_SKB; 2641 #endif 2642 wr32(IGC_RLPML, rlpml); 2643 } 2644 2645 /** 2646 * igc_configure - configure the hardware for RX and TX 2647 * @adapter: private board structure 2648 */ 2649 static void igc_configure(struct igc_adapter *adapter) 2650 { 2651 struct net_device *netdev = adapter->netdev; 2652 int i = 0; 2653 2654 igc_get_hw_control(adapter); 2655 igc_set_rx_mode(netdev); 2656 2657 igc_setup_tctl(adapter); 2658 igc_setup_mrqc(adapter); 2659 igc_setup_rctl(adapter); 2660 2661 igc_set_default_mac_filter(adapter); 2662 igc_restore_nfc_rules(adapter); 2663 2664 igc_configure_tx(adapter); 2665 igc_configure_rx(adapter); 2666 2667 igc_rx_fifo_flush_base(&adapter->hw); 2668 2669 /* call igc_desc_unused which always leaves 2670 * at least 1 descriptor unused to make sure 2671 * next_to_use != next_to_clean 2672 */ 2673 for (i = 0; i < adapter->num_rx_queues; i++) { 2674 struct igc_ring *ring = adapter->rx_ring[i]; 2675 2676 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 2677 } 2678 } 2679 2680 /** 2681 * igc_write_ivar - configure ivar for given MSI-X vector 2682 * @hw: pointer to the HW structure 2683 * @msix_vector: vector number we are allocating to a given ring 2684 * @index: row index of IVAR register to write within IVAR table 2685 * @offset: column offset of in IVAR, should be multiple of 8 2686 * 2687 * The IVAR table consists of 2 columns, 2688 * each containing an cause allocation for an Rx and Tx ring, and a 2689 * variable number of rows depending on the number of queues supported. 2690 */ 2691 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 2692 int index, int offset) 2693 { 2694 u32 ivar = array_rd32(IGC_IVAR0, index); 2695 2696 /* clear any bits that are currently set */ 2697 ivar &= ~((u32)0xFF << offset); 2698 2699 /* write vector and valid bit */ 2700 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 2701 2702 array_wr32(IGC_IVAR0, index, ivar); 2703 } 2704 2705 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 2706 { 2707 struct igc_adapter *adapter = q_vector->adapter; 2708 struct igc_hw *hw = &adapter->hw; 2709 int rx_queue = IGC_N0_QUEUE; 2710 int tx_queue = IGC_N0_QUEUE; 2711 2712 if (q_vector->rx.ring) 2713 rx_queue = q_vector->rx.ring->reg_idx; 2714 if (q_vector->tx.ring) 2715 tx_queue = q_vector->tx.ring->reg_idx; 2716 2717 switch (hw->mac.type) { 2718 case igc_i225: 2719 if (rx_queue > IGC_N0_QUEUE) 2720 igc_write_ivar(hw, msix_vector, 2721 rx_queue >> 1, 2722 (rx_queue & 0x1) << 4); 2723 if (tx_queue > IGC_N0_QUEUE) 2724 igc_write_ivar(hw, msix_vector, 2725 tx_queue >> 1, 2726 ((tx_queue & 0x1) << 4) + 8); 2727 q_vector->eims_value = BIT(msix_vector); 2728 break; 2729 default: 2730 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 2731 break; 2732 } 2733 2734 /* add q_vector eims value to global eims_enable_mask */ 2735 adapter->eims_enable_mask |= q_vector->eims_value; 2736 2737 /* configure q_vector to set itr on first interrupt */ 2738 q_vector->set_itr = 1; 2739 } 2740 2741 /** 2742 * igc_configure_msix - Configure MSI-X hardware 2743 * @adapter: Pointer to adapter structure 2744 * 2745 * igc_configure_msix sets up the hardware to properly 2746 * generate MSI-X interrupts. 2747 */ 2748 static void igc_configure_msix(struct igc_adapter *adapter) 2749 { 2750 struct igc_hw *hw = &adapter->hw; 2751 int i, vector = 0; 2752 u32 tmp; 2753 2754 adapter->eims_enable_mask = 0; 2755 2756 /* set vector for other causes, i.e. link changes */ 2757 switch (hw->mac.type) { 2758 case igc_i225: 2759 /* Turn on MSI-X capability first, or our settings 2760 * won't stick. And it will take days to debug. 2761 */ 2762 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 2763 IGC_GPIE_PBA | IGC_GPIE_EIAME | 2764 IGC_GPIE_NSICR); 2765 2766 /* enable msix_other interrupt */ 2767 adapter->eims_other = BIT(vector); 2768 tmp = (vector++ | IGC_IVAR_VALID) << 8; 2769 2770 wr32(IGC_IVAR_MISC, tmp); 2771 break; 2772 default: 2773 /* do nothing, since nothing else supports MSI-X */ 2774 break; 2775 } /* switch (hw->mac.type) */ 2776 2777 adapter->eims_enable_mask |= adapter->eims_other; 2778 2779 for (i = 0; i < adapter->num_q_vectors; i++) 2780 igc_assign_vector(adapter->q_vector[i], vector++); 2781 2782 wrfl(); 2783 } 2784 2785 /** 2786 * igc_irq_enable - Enable default interrupt generation settings 2787 * @adapter: board private structure 2788 */ 2789 static void igc_irq_enable(struct igc_adapter *adapter) 2790 { 2791 struct igc_hw *hw = &adapter->hw; 2792 2793 if (adapter->msix_entries) { 2794 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 2795 u32 regval = rd32(IGC_EIAC); 2796 2797 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 2798 regval = rd32(IGC_EIAM); 2799 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 2800 wr32(IGC_EIMS, adapter->eims_enable_mask); 2801 wr32(IGC_IMS, ims); 2802 } else { 2803 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2804 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2805 } 2806 } 2807 2808 /** 2809 * igc_irq_disable - Mask off interrupt generation on the NIC 2810 * @adapter: board private structure 2811 */ 2812 static void igc_irq_disable(struct igc_adapter *adapter) 2813 { 2814 struct igc_hw *hw = &adapter->hw; 2815 2816 if (adapter->msix_entries) { 2817 u32 regval = rd32(IGC_EIAM); 2818 2819 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 2820 wr32(IGC_EIMC, adapter->eims_enable_mask); 2821 regval = rd32(IGC_EIAC); 2822 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 2823 } 2824 2825 wr32(IGC_IAM, 0); 2826 wr32(IGC_IMC, ~0); 2827 wrfl(); 2828 2829 if (adapter->msix_entries) { 2830 int vector = 0, i; 2831 2832 synchronize_irq(adapter->msix_entries[vector++].vector); 2833 2834 for (i = 0; i < adapter->num_q_vectors; i++) 2835 synchronize_irq(adapter->msix_entries[vector++].vector); 2836 } else { 2837 synchronize_irq(adapter->pdev->irq); 2838 } 2839 } 2840 2841 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 2842 const u32 max_rss_queues) 2843 { 2844 /* Determine if we need to pair queues. */ 2845 /* If rss_queues > half of max_rss_queues, pair the queues in 2846 * order to conserve interrupts due to limited supply. 2847 */ 2848 if (adapter->rss_queues > (max_rss_queues / 2)) 2849 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 2850 else 2851 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 2852 } 2853 2854 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 2855 { 2856 return IGC_MAX_RX_QUEUES; 2857 } 2858 2859 static void igc_init_queue_configuration(struct igc_adapter *adapter) 2860 { 2861 u32 max_rss_queues; 2862 2863 max_rss_queues = igc_get_max_rss_queues(adapter); 2864 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 2865 2866 igc_set_flag_queue_pairs(adapter, max_rss_queues); 2867 } 2868 2869 /** 2870 * igc_reset_q_vector - Reset config for interrupt vector 2871 * @adapter: board private structure to initialize 2872 * @v_idx: Index of vector to be reset 2873 * 2874 * If NAPI is enabled it will delete any references to the 2875 * NAPI struct. This is preparation for igc_free_q_vector. 2876 */ 2877 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 2878 { 2879 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2880 2881 /* if we're coming from igc_set_interrupt_capability, the vectors are 2882 * not yet allocated 2883 */ 2884 if (!q_vector) 2885 return; 2886 2887 if (q_vector->tx.ring) 2888 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 2889 2890 if (q_vector->rx.ring) 2891 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 2892 2893 netif_napi_del(&q_vector->napi); 2894 } 2895 2896 /** 2897 * igc_free_q_vector - Free memory allocated for specific interrupt vector 2898 * @adapter: board private structure to initialize 2899 * @v_idx: Index of vector to be freed 2900 * 2901 * This function frees the memory allocated to the q_vector. 2902 */ 2903 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 2904 { 2905 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2906 2907 adapter->q_vector[v_idx] = NULL; 2908 2909 /* igc_get_stats64() might access the rings on this vector, 2910 * we must wait a grace period before freeing it. 2911 */ 2912 if (q_vector) 2913 kfree_rcu(q_vector, rcu); 2914 } 2915 2916 /** 2917 * igc_free_q_vectors - Free memory allocated for interrupt vectors 2918 * @adapter: board private structure to initialize 2919 * 2920 * This function frees the memory allocated to the q_vectors. In addition if 2921 * NAPI is enabled it will delete any references to the NAPI struct prior 2922 * to freeing the q_vector. 2923 */ 2924 static void igc_free_q_vectors(struct igc_adapter *adapter) 2925 { 2926 int v_idx = adapter->num_q_vectors; 2927 2928 adapter->num_tx_queues = 0; 2929 adapter->num_rx_queues = 0; 2930 adapter->num_q_vectors = 0; 2931 2932 while (v_idx--) { 2933 igc_reset_q_vector(adapter, v_idx); 2934 igc_free_q_vector(adapter, v_idx); 2935 } 2936 } 2937 2938 /** 2939 * igc_update_itr - update the dynamic ITR value based on statistics 2940 * @q_vector: pointer to q_vector 2941 * @ring_container: ring info to update the itr for 2942 * 2943 * Stores a new ITR value based on packets and byte 2944 * counts during the last interrupt. The advantage of per interrupt 2945 * computation is faster updates and more accurate ITR for the current 2946 * traffic pattern. Constants in this function were computed 2947 * based on theoretical maximum wire speed and thresholds were set based 2948 * on testing data as well as attempting to minimize response time 2949 * while increasing bulk throughput. 2950 * NOTE: These calculations are only valid when operating in a single- 2951 * queue environment. 2952 */ 2953 static void igc_update_itr(struct igc_q_vector *q_vector, 2954 struct igc_ring_container *ring_container) 2955 { 2956 unsigned int packets = ring_container->total_packets; 2957 unsigned int bytes = ring_container->total_bytes; 2958 u8 itrval = ring_container->itr; 2959 2960 /* no packets, exit with status unchanged */ 2961 if (packets == 0) 2962 return; 2963 2964 switch (itrval) { 2965 case lowest_latency: 2966 /* handle TSO and jumbo frames */ 2967 if (bytes / packets > 8000) 2968 itrval = bulk_latency; 2969 else if ((packets < 5) && (bytes > 512)) 2970 itrval = low_latency; 2971 break; 2972 case low_latency: /* 50 usec aka 20000 ints/s */ 2973 if (bytes > 10000) { 2974 /* this if handles the TSO accounting */ 2975 if (bytes / packets > 8000) 2976 itrval = bulk_latency; 2977 else if ((packets < 10) || ((bytes / packets) > 1200)) 2978 itrval = bulk_latency; 2979 else if ((packets > 35)) 2980 itrval = lowest_latency; 2981 } else if (bytes / packets > 2000) { 2982 itrval = bulk_latency; 2983 } else if (packets <= 2 && bytes < 512) { 2984 itrval = lowest_latency; 2985 } 2986 break; 2987 case bulk_latency: /* 250 usec aka 4000 ints/s */ 2988 if (bytes > 25000) { 2989 if (packets > 35) 2990 itrval = low_latency; 2991 } else if (bytes < 1500) { 2992 itrval = low_latency; 2993 } 2994 break; 2995 } 2996 2997 /* clear work counters since we have the values we need */ 2998 ring_container->total_bytes = 0; 2999 ring_container->total_packets = 0; 3000 3001 /* write updated itr to ring container */ 3002 ring_container->itr = itrval; 3003 } 3004 3005 static void igc_set_itr(struct igc_q_vector *q_vector) 3006 { 3007 struct igc_adapter *adapter = q_vector->adapter; 3008 u32 new_itr = q_vector->itr_val; 3009 u8 current_itr = 0; 3010 3011 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 3012 switch (adapter->link_speed) { 3013 case SPEED_10: 3014 case SPEED_100: 3015 current_itr = 0; 3016 new_itr = IGC_4K_ITR; 3017 goto set_itr_now; 3018 default: 3019 break; 3020 } 3021 3022 igc_update_itr(q_vector, &q_vector->tx); 3023 igc_update_itr(q_vector, &q_vector->rx); 3024 3025 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 3026 3027 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3028 if (current_itr == lowest_latency && 3029 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3030 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3031 current_itr = low_latency; 3032 3033 switch (current_itr) { 3034 /* counts and packets in update_itr are dependent on these numbers */ 3035 case lowest_latency: 3036 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 3037 break; 3038 case low_latency: 3039 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 3040 break; 3041 case bulk_latency: 3042 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 3043 break; 3044 default: 3045 break; 3046 } 3047 3048 set_itr_now: 3049 if (new_itr != q_vector->itr_val) { 3050 /* this attempts to bias the interrupt rate towards Bulk 3051 * by adding intermediate steps when interrupt rate is 3052 * increasing 3053 */ 3054 new_itr = new_itr > q_vector->itr_val ? 3055 max((new_itr * q_vector->itr_val) / 3056 (new_itr + (q_vector->itr_val >> 2)), 3057 new_itr) : new_itr; 3058 /* Don't write the value here; it resets the adapter's 3059 * internal timer, and causes us to delay far longer than 3060 * we should between interrupts. Instead, we write the ITR 3061 * value at the beginning of the next interrupt so the timing 3062 * ends up being correct. 3063 */ 3064 q_vector->itr_val = new_itr; 3065 q_vector->set_itr = 1; 3066 } 3067 } 3068 3069 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 3070 { 3071 int v_idx = adapter->num_q_vectors; 3072 3073 if (adapter->msix_entries) { 3074 pci_disable_msix(adapter->pdev); 3075 kfree(adapter->msix_entries); 3076 adapter->msix_entries = NULL; 3077 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 3078 pci_disable_msi(adapter->pdev); 3079 } 3080 3081 while (v_idx--) 3082 igc_reset_q_vector(adapter, v_idx); 3083 } 3084 3085 /** 3086 * igc_set_interrupt_capability - set MSI or MSI-X if supported 3087 * @adapter: Pointer to adapter structure 3088 * @msix: boolean value for MSI-X capability 3089 * 3090 * Attempt to configure interrupts using the best available 3091 * capabilities of the hardware and kernel. 3092 */ 3093 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 3094 bool msix) 3095 { 3096 int numvecs, i; 3097 int err; 3098 3099 if (!msix) 3100 goto msi_only; 3101 adapter->flags |= IGC_FLAG_HAS_MSIX; 3102 3103 /* Number of supported queues. */ 3104 adapter->num_rx_queues = adapter->rss_queues; 3105 3106 adapter->num_tx_queues = adapter->rss_queues; 3107 3108 /* start with one vector for every Rx queue */ 3109 numvecs = adapter->num_rx_queues; 3110 3111 /* if Tx handler is separate add 1 for every Tx queue */ 3112 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 3113 numvecs += adapter->num_tx_queues; 3114 3115 /* store the number of vectors reserved for queues */ 3116 adapter->num_q_vectors = numvecs; 3117 3118 /* add 1 vector for link status interrupts */ 3119 numvecs++; 3120 3121 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), 3122 GFP_KERNEL); 3123 3124 if (!adapter->msix_entries) 3125 return; 3126 3127 /* populate entry values */ 3128 for (i = 0; i < numvecs; i++) 3129 adapter->msix_entries[i].entry = i; 3130 3131 err = pci_enable_msix_range(adapter->pdev, 3132 adapter->msix_entries, 3133 numvecs, 3134 numvecs); 3135 if (err > 0) 3136 return; 3137 3138 kfree(adapter->msix_entries); 3139 adapter->msix_entries = NULL; 3140 3141 igc_reset_interrupt_capability(adapter); 3142 3143 msi_only: 3144 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 3145 3146 adapter->rss_queues = 1; 3147 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 3148 adapter->num_rx_queues = 1; 3149 adapter->num_tx_queues = 1; 3150 adapter->num_q_vectors = 1; 3151 if (!pci_enable_msi(adapter->pdev)) 3152 adapter->flags |= IGC_FLAG_HAS_MSI; 3153 } 3154 3155 /** 3156 * igc_update_ring_itr - update the dynamic ITR value based on packet size 3157 * @q_vector: pointer to q_vector 3158 * 3159 * Stores a new ITR value based on strictly on packet size. This 3160 * algorithm is less sophisticated than that used in igc_update_itr, 3161 * due to the difficulty of synchronizing statistics across multiple 3162 * receive rings. The divisors and thresholds used by this function 3163 * were determined based on theoretical maximum wire speed and testing 3164 * data, in order to minimize response time while increasing bulk 3165 * throughput. 3166 * NOTE: This function is called only when operating in a multiqueue 3167 * receive environment. 3168 */ 3169 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 3170 { 3171 struct igc_adapter *adapter = q_vector->adapter; 3172 int new_val = q_vector->itr_val; 3173 int avg_wire_size = 0; 3174 unsigned int packets; 3175 3176 /* For non-gigabit speeds, just fix the interrupt rate at 4000 3177 * ints/sec - ITR timer value of 120 ticks. 3178 */ 3179 switch (adapter->link_speed) { 3180 case SPEED_10: 3181 case SPEED_100: 3182 new_val = IGC_4K_ITR; 3183 goto set_itr_val; 3184 default: 3185 break; 3186 } 3187 3188 packets = q_vector->rx.total_packets; 3189 if (packets) 3190 avg_wire_size = q_vector->rx.total_bytes / packets; 3191 3192 packets = q_vector->tx.total_packets; 3193 if (packets) 3194 avg_wire_size = max_t(u32, avg_wire_size, 3195 q_vector->tx.total_bytes / packets); 3196 3197 /* if avg_wire_size isn't set no work was done */ 3198 if (!avg_wire_size) 3199 goto clear_counts; 3200 3201 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 3202 avg_wire_size += 24; 3203 3204 /* Don't starve jumbo frames */ 3205 avg_wire_size = min(avg_wire_size, 3000); 3206 3207 /* Give a little boost to mid-size frames */ 3208 if (avg_wire_size > 300 && avg_wire_size < 1200) 3209 new_val = avg_wire_size / 3; 3210 else 3211 new_val = avg_wire_size / 2; 3212 3213 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3214 if (new_val < IGC_20K_ITR && 3215 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3216 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3217 new_val = IGC_20K_ITR; 3218 3219 set_itr_val: 3220 if (new_val != q_vector->itr_val) { 3221 q_vector->itr_val = new_val; 3222 q_vector->set_itr = 1; 3223 } 3224 clear_counts: 3225 q_vector->rx.total_bytes = 0; 3226 q_vector->rx.total_packets = 0; 3227 q_vector->tx.total_bytes = 0; 3228 q_vector->tx.total_packets = 0; 3229 } 3230 3231 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 3232 { 3233 struct igc_adapter *adapter = q_vector->adapter; 3234 struct igc_hw *hw = &adapter->hw; 3235 3236 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 3237 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 3238 if (adapter->num_q_vectors == 1) 3239 igc_set_itr(q_vector); 3240 else 3241 igc_update_ring_itr(q_vector); 3242 } 3243 3244 if (!test_bit(__IGC_DOWN, &adapter->state)) { 3245 if (adapter->msix_entries) 3246 wr32(IGC_EIMS, q_vector->eims_value); 3247 else 3248 igc_irq_enable(adapter); 3249 } 3250 } 3251 3252 static void igc_add_ring(struct igc_ring *ring, 3253 struct igc_ring_container *head) 3254 { 3255 head->ring = ring; 3256 head->count++; 3257 } 3258 3259 /** 3260 * igc_cache_ring_register - Descriptor ring to register mapping 3261 * @adapter: board private structure to initialize 3262 * 3263 * Once we know the feature-set enabled for the device, we'll cache 3264 * the register offset the descriptor ring is assigned to. 3265 */ 3266 static void igc_cache_ring_register(struct igc_adapter *adapter) 3267 { 3268 int i = 0, j = 0; 3269 3270 switch (adapter->hw.mac.type) { 3271 case igc_i225: 3272 default: 3273 for (; i < adapter->num_rx_queues; i++) 3274 adapter->rx_ring[i]->reg_idx = i; 3275 for (; j < adapter->num_tx_queues; j++) 3276 adapter->tx_ring[j]->reg_idx = j; 3277 break; 3278 } 3279 } 3280 3281 /** 3282 * igc_poll - NAPI Rx polling callback 3283 * @napi: napi polling structure 3284 * @budget: count of how many packets we should handle 3285 */ 3286 static int igc_poll(struct napi_struct *napi, int budget) 3287 { 3288 struct igc_q_vector *q_vector = container_of(napi, 3289 struct igc_q_vector, 3290 napi); 3291 bool clean_complete = true; 3292 int work_done = 0; 3293 3294 if (q_vector->tx.ring) 3295 clean_complete = igc_clean_tx_irq(q_vector, budget); 3296 3297 if (q_vector->rx.ring) { 3298 int cleaned = igc_clean_rx_irq(q_vector, budget); 3299 3300 work_done += cleaned; 3301 if (cleaned >= budget) 3302 clean_complete = false; 3303 } 3304 3305 /* If all work not completed, return budget and keep polling */ 3306 if (!clean_complete) 3307 return budget; 3308 3309 /* Exit the polling mode, but don't re-enable interrupts if stack might 3310 * poll us due to busy-polling 3311 */ 3312 if (likely(napi_complete_done(napi, work_done))) 3313 igc_ring_irq_enable(q_vector); 3314 3315 return min(work_done, budget - 1); 3316 } 3317 3318 /** 3319 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 3320 * @adapter: board private structure to initialize 3321 * @v_count: q_vectors allocated on adapter, used for ring interleaving 3322 * @v_idx: index of vector in adapter struct 3323 * @txr_count: total number of Tx rings to allocate 3324 * @txr_idx: index of first Tx ring to allocate 3325 * @rxr_count: total number of Rx rings to allocate 3326 * @rxr_idx: index of first Rx ring to allocate 3327 * 3328 * We allocate one q_vector. If allocation fails we return -ENOMEM. 3329 */ 3330 static int igc_alloc_q_vector(struct igc_adapter *adapter, 3331 unsigned int v_count, unsigned int v_idx, 3332 unsigned int txr_count, unsigned int txr_idx, 3333 unsigned int rxr_count, unsigned int rxr_idx) 3334 { 3335 struct igc_q_vector *q_vector; 3336 struct igc_ring *ring; 3337 int ring_count; 3338 3339 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 3340 if (txr_count > 1 || rxr_count > 1) 3341 return -ENOMEM; 3342 3343 ring_count = txr_count + rxr_count; 3344 3345 /* allocate q_vector and rings */ 3346 q_vector = adapter->q_vector[v_idx]; 3347 if (!q_vector) 3348 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), 3349 GFP_KERNEL); 3350 else 3351 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 3352 if (!q_vector) 3353 return -ENOMEM; 3354 3355 /* initialize NAPI */ 3356 netif_napi_add(adapter->netdev, &q_vector->napi, 3357 igc_poll, 64); 3358 3359 /* tie q_vector and adapter together */ 3360 adapter->q_vector[v_idx] = q_vector; 3361 q_vector->adapter = adapter; 3362 3363 /* initialize work limits */ 3364 q_vector->tx.work_limit = adapter->tx_work_limit; 3365 3366 /* initialize ITR configuration */ 3367 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 3368 q_vector->itr_val = IGC_START_ITR; 3369 3370 /* initialize pointer to rings */ 3371 ring = q_vector->ring; 3372 3373 /* initialize ITR */ 3374 if (rxr_count) { 3375 /* rx or rx/tx vector */ 3376 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 3377 q_vector->itr_val = adapter->rx_itr_setting; 3378 } else { 3379 /* tx only vector */ 3380 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 3381 q_vector->itr_val = adapter->tx_itr_setting; 3382 } 3383 3384 if (txr_count) { 3385 /* assign generic ring traits */ 3386 ring->dev = &adapter->pdev->dev; 3387 ring->netdev = adapter->netdev; 3388 3389 /* configure backlink on ring */ 3390 ring->q_vector = q_vector; 3391 3392 /* update q_vector Tx values */ 3393 igc_add_ring(ring, &q_vector->tx); 3394 3395 /* apply Tx specific ring traits */ 3396 ring->count = adapter->tx_ring_count; 3397 ring->queue_index = txr_idx; 3398 3399 /* assign ring to adapter */ 3400 adapter->tx_ring[txr_idx] = ring; 3401 3402 /* push pointer to next ring */ 3403 ring++; 3404 } 3405 3406 if (rxr_count) { 3407 /* assign generic ring traits */ 3408 ring->dev = &adapter->pdev->dev; 3409 ring->netdev = adapter->netdev; 3410 3411 /* configure backlink on ring */ 3412 ring->q_vector = q_vector; 3413 3414 /* update q_vector Rx values */ 3415 igc_add_ring(ring, &q_vector->rx); 3416 3417 /* apply Rx specific ring traits */ 3418 ring->count = adapter->rx_ring_count; 3419 ring->queue_index = rxr_idx; 3420 3421 /* assign ring to adapter */ 3422 adapter->rx_ring[rxr_idx] = ring; 3423 } 3424 3425 return 0; 3426 } 3427 3428 /** 3429 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 3430 * @adapter: board private structure to initialize 3431 * 3432 * We allocate one q_vector per queue interrupt. If allocation fails we 3433 * return -ENOMEM. 3434 */ 3435 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 3436 { 3437 int rxr_remaining = adapter->num_rx_queues; 3438 int txr_remaining = adapter->num_tx_queues; 3439 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 3440 int q_vectors = adapter->num_q_vectors; 3441 int err; 3442 3443 if (q_vectors >= (rxr_remaining + txr_remaining)) { 3444 for (; rxr_remaining; v_idx++) { 3445 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3446 0, 0, 1, rxr_idx); 3447 3448 if (err) 3449 goto err_out; 3450 3451 /* update counts and index */ 3452 rxr_remaining--; 3453 rxr_idx++; 3454 } 3455 } 3456 3457 for (; v_idx < q_vectors; v_idx++) { 3458 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 3459 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 3460 3461 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3462 tqpv, txr_idx, rqpv, rxr_idx); 3463 3464 if (err) 3465 goto err_out; 3466 3467 /* update counts and index */ 3468 rxr_remaining -= rqpv; 3469 txr_remaining -= tqpv; 3470 rxr_idx++; 3471 txr_idx++; 3472 } 3473 3474 return 0; 3475 3476 err_out: 3477 adapter->num_tx_queues = 0; 3478 adapter->num_rx_queues = 0; 3479 adapter->num_q_vectors = 0; 3480 3481 while (v_idx--) 3482 igc_free_q_vector(adapter, v_idx); 3483 3484 return -ENOMEM; 3485 } 3486 3487 /** 3488 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 3489 * @adapter: Pointer to adapter structure 3490 * @msix: boolean for MSI-X capability 3491 * 3492 * This function initializes the interrupts and allocates all of the queues. 3493 */ 3494 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 3495 { 3496 struct net_device *dev = adapter->netdev; 3497 int err = 0; 3498 3499 igc_set_interrupt_capability(adapter, msix); 3500 3501 err = igc_alloc_q_vectors(adapter); 3502 if (err) { 3503 netdev_err(dev, "Unable to allocate memory for vectors\n"); 3504 goto err_alloc_q_vectors; 3505 } 3506 3507 igc_cache_ring_register(adapter); 3508 3509 return 0; 3510 3511 err_alloc_q_vectors: 3512 igc_reset_interrupt_capability(adapter); 3513 return err; 3514 } 3515 3516 /** 3517 * igc_sw_init - Initialize general software structures (struct igc_adapter) 3518 * @adapter: board private structure to initialize 3519 * 3520 * igc_sw_init initializes the Adapter private data structure. 3521 * Fields are initialized based on PCI device information and 3522 * OS network device settings (MTU size). 3523 */ 3524 static int igc_sw_init(struct igc_adapter *adapter) 3525 { 3526 struct net_device *netdev = adapter->netdev; 3527 struct pci_dev *pdev = adapter->pdev; 3528 struct igc_hw *hw = &adapter->hw; 3529 3530 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 3531 3532 /* set default ring sizes */ 3533 adapter->tx_ring_count = IGC_DEFAULT_TXD; 3534 adapter->rx_ring_count = IGC_DEFAULT_RXD; 3535 3536 /* set default ITR values */ 3537 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 3538 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 3539 3540 /* set default work limits */ 3541 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 3542 3543 /* adjust max frame to be at least the size of a standard frame */ 3544 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 3545 VLAN_HLEN; 3546 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 3547 3548 mutex_init(&adapter->nfc_rule_lock); 3549 INIT_LIST_HEAD(&adapter->nfc_rule_list); 3550 adapter->nfc_rule_count = 0; 3551 3552 spin_lock_init(&adapter->stats64_lock); 3553 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 3554 adapter->flags |= IGC_FLAG_HAS_MSIX; 3555 3556 igc_init_queue_configuration(adapter); 3557 3558 /* This call may decrease the number of queues */ 3559 if (igc_init_interrupt_scheme(adapter, true)) { 3560 netdev_err(netdev, "Unable to allocate memory for queues\n"); 3561 return -ENOMEM; 3562 } 3563 3564 /* Explicitly disable IRQ since the NIC can be in any state. */ 3565 igc_irq_disable(adapter); 3566 3567 set_bit(__IGC_DOWN, &adapter->state); 3568 3569 return 0; 3570 } 3571 3572 /** 3573 * igc_up - Open the interface and prepare it to handle traffic 3574 * @adapter: board private structure 3575 */ 3576 void igc_up(struct igc_adapter *adapter) 3577 { 3578 struct igc_hw *hw = &adapter->hw; 3579 int i = 0; 3580 3581 /* hardware has been reset, we need to reload some things */ 3582 igc_configure(adapter); 3583 3584 clear_bit(__IGC_DOWN, &adapter->state); 3585 3586 for (i = 0; i < adapter->num_q_vectors; i++) 3587 napi_enable(&adapter->q_vector[i]->napi); 3588 3589 if (adapter->msix_entries) 3590 igc_configure_msix(adapter); 3591 else 3592 igc_assign_vector(adapter->q_vector[0], 0); 3593 3594 /* Clear any pending interrupts. */ 3595 rd32(IGC_ICR); 3596 igc_irq_enable(adapter); 3597 3598 netif_tx_start_all_queues(adapter->netdev); 3599 3600 /* start the watchdog. */ 3601 hw->mac.get_link_status = 1; 3602 schedule_work(&adapter->watchdog_task); 3603 } 3604 3605 /** 3606 * igc_update_stats - Update the board statistics counters 3607 * @adapter: board private structure 3608 */ 3609 void igc_update_stats(struct igc_adapter *adapter) 3610 { 3611 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 3612 struct pci_dev *pdev = adapter->pdev; 3613 struct igc_hw *hw = &adapter->hw; 3614 u64 _bytes, _packets; 3615 u64 bytes, packets; 3616 unsigned int start; 3617 u32 mpc; 3618 int i; 3619 3620 /* Prevent stats update while adapter is being reset, or if the pci 3621 * connection is down. 3622 */ 3623 if (adapter->link_speed == 0) 3624 return; 3625 if (pci_channel_offline(pdev)) 3626 return; 3627 3628 packets = 0; 3629 bytes = 0; 3630 3631 rcu_read_lock(); 3632 for (i = 0; i < adapter->num_rx_queues; i++) { 3633 struct igc_ring *ring = adapter->rx_ring[i]; 3634 u32 rqdpc = rd32(IGC_RQDPC(i)); 3635 3636 if (hw->mac.type >= igc_i225) 3637 wr32(IGC_RQDPC(i), 0); 3638 3639 if (rqdpc) { 3640 ring->rx_stats.drops += rqdpc; 3641 net_stats->rx_fifo_errors += rqdpc; 3642 } 3643 3644 do { 3645 start = u64_stats_fetch_begin_irq(&ring->rx_syncp); 3646 _bytes = ring->rx_stats.bytes; 3647 _packets = ring->rx_stats.packets; 3648 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); 3649 bytes += _bytes; 3650 packets += _packets; 3651 } 3652 3653 net_stats->rx_bytes = bytes; 3654 net_stats->rx_packets = packets; 3655 3656 packets = 0; 3657 bytes = 0; 3658 for (i = 0; i < adapter->num_tx_queues; i++) { 3659 struct igc_ring *ring = adapter->tx_ring[i]; 3660 3661 do { 3662 start = u64_stats_fetch_begin_irq(&ring->tx_syncp); 3663 _bytes = ring->tx_stats.bytes; 3664 _packets = ring->tx_stats.packets; 3665 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); 3666 bytes += _bytes; 3667 packets += _packets; 3668 } 3669 net_stats->tx_bytes = bytes; 3670 net_stats->tx_packets = packets; 3671 rcu_read_unlock(); 3672 3673 /* read stats registers */ 3674 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 3675 adapter->stats.gprc += rd32(IGC_GPRC); 3676 adapter->stats.gorc += rd32(IGC_GORCL); 3677 rd32(IGC_GORCH); /* clear GORCL */ 3678 adapter->stats.bprc += rd32(IGC_BPRC); 3679 adapter->stats.mprc += rd32(IGC_MPRC); 3680 adapter->stats.roc += rd32(IGC_ROC); 3681 3682 adapter->stats.prc64 += rd32(IGC_PRC64); 3683 adapter->stats.prc127 += rd32(IGC_PRC127); 3684 adapter->stats.prc255 += rd32(IGC_PRC255); 3685 adapter->stats.prc511 += rd32(IGC_PRC511); 3686 adapter->stats.prc1023 += rd32(IGC_PRC1023); 3687 adapter->stats.prc1522 += rd32(IGC_PRC1522); 3688 3689 mpc = rd32(IGC_MPC); 3690 adapter->stats.mpc += mpc; 3691 net_stats->rx_fifo_errors += mpc; 3692 adapter->stats.scc += rd32(IGC_SCC); 3693 adapter->stats.ecol += rd32(IGC_ECOL); 3694 adapter->stats.mcc += rd32(IGC_MCC); 3695 adapter->stats.latecol += rd32(IGC_LATECOL); 3696 adapter->stats.dc += rd32(IGC_DC); 3697 adapter->stats.rlec += rd32(IGC_RLEC); 3698 adapter->stats.xonrxc += rd32(IGC_XONRXC); 3699 adapter->stats.xontxc += rd32(IGC_XONTXC); 3700 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 3701 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 3702 adapter->stats.fcruc += rd32(IGC_FCRUC); 3703 adapter->stats.gptc += rd32(IGC_GPTC); 3704 adapter->stats.gotc += rd32(IGC_GOTCL); 3705 rd32(IGC_GOTCH); /* clear GOTCL */ 3706 adapter->stats.rnbc += rd32(IGC_RNBC); 3707 adapter->stats.ruc += rd32(IGC_RUC); 3708 adapter->stats.rfc += rd32(IGC_RFC); 3709 adapter->stats.rjc += rd32(IGC_RJC); 3710 adapter->stats.tor += rd32(IGC_TORH); 3711 adapter->stats.tot += rd32(IGC_TOTH); 3712 adapter->stats.tpr += rd32(IGC_TPR); 3713 3714 adapter->stats.ptc64 += rd32(IGC_PTC64); 3715 adapter->stats.ptc127 += rd32(IGC_PTC127); 3716 adapter->stats.ptc255 += rd32(IGC_PTC255); 3717 adapter->stats.ptc511 += rd32(IGC_PTC511); 3718 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 3719 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 3720 3721 adapter->stats.mptc += rd32(IGC_MPTC); 3722 adapter->stats.bptc += rd32(IGC_BPTC); 3723 3724 adapter->stats.tpt += rd32(IGC_TPT); 3725 adapter->stats.colc += rd32(IGC_COLC); 3726 adapter->stats.colc += rd32(IGC_RERC); 3727 3728 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 3729 3730 adapter->stats.tsctc += rd32(IGC_TSCTC); 3731 3732 adapter->stats.iac += rd32(IGC_IAC); 3733 adapter->stats.icrxoc += rd32(IGC_ICRXOC); 3734 adapter->stats.icrxptc += rd32(IGC_ICRXPTC); 3735 adapter->stats.icrxatc += rd32(IGC_ICRXATC); 3736 adapter->stats.ictxptc += rd32(IGC_ICTXPTC); 3737 adapter->stats.ictxatc += rd32(IGC_ICTXATC); 3738 adapter->stats.ictxqec += rd32(IGC_ICTXQEC); 3739 adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC); 3740 adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC); 3741 3742 /* Fill out the OS statistics structure */ 3743 net_stats->multicast = adapter->stats.mprc; 3744 net_stats->collisions = adapter->stats.colc; 3745 3746 /* Rx Errors */ 3747 3748 /* RLEC on some newer hardware can be incorrect so build 3749 * our own version based on RUC and ROC 3750 */ 3751 net_stats->rx_errors = adapter->stats.rxerrc + 3752 adapter->stats.crcerrs + adapter->stats.algnerrc + 3753 adapter->stats.ruc + adapter->stats.roc + 3754 adapter->stats.cexterr; 3755 net_stats->rx_length_errors = adapter->stats.ruc + 3756 adapter->stats.roc; 3757 net_stats->rx_crc_errors = adapter->stats.crcerrs; 3758 net_stats->rx_frame_errors = adapter->stats.algnerrc; 3759 net_stats->rx_missed_errors = adapter->stats.mpc; 3760 3761 /* Tx Errors */ 3762 net_stats->tx_errors = adapter->stats.ecol + 3763 adapter->stats.latecol; 3764 net_stats->tx_aborted_errors = adapter->stats.ecol; 3765 net_stats->tx_window_errors = adapter->stats.latecol; 3766 net_stats->tx_carrier_errors = adapter->stats.tncrs; 3767 3768 /* Tx Dropped needs to be maintained elsewhere */ 3769 3770 /* Management Stats */ 3771 adapter->stats.mgptc += rd32(IGC_MGTPTC); 3772 adapter->stats.mgprc += rd32(IGC_MGTPRC); 3773 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 3774 } 3775 3776 /** 3777 * igc_down - Close the interface 3778 * @adapter: board private structure 3779 */ 3780 void igc_down(struct igc_adapter *adapter) 3781 { 3782 struct net_device *netdev = adapter->netdev; 3783 struct igc_hw *hw = &adapter->hw; 3784 u32 tctl, rctl; 3785 int i = 0; 3786 3787 set_bit(__IGC_DOWN, &adapter->state); 3788 3789 /* disable receives in the hardware */ 3790 rctl = rd32(IGC_RCTL); 3791 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 3792 /* flush and sleep below */ 3793 3794 /* set trans_start so we don't get spurious watchdogs during reset */ 3795 netif_trans_update(netdev); 3796 3797 netif_carrier_off(netdev); 3798 netif_tx_stop_all_queues(netdev); 3799 3800 /* disable transmits in the hardware */ 3801 tctl = rd32(IGC_TCTL); 3802 tctl &= ~IGC_TCTL_EN; 3803 wr32(IGC_TCTL, tctl); 3804 /* flush both disables and wait for them to finish */ 3805 wrfl(); 3806 usleep_range(10000, 20000); 3807 3808 igc_irq_disable(adapter); 3809 3810 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3811 3812 for (i = 0; i < adapter->num_q_vectors; i++) { 3813 if (adapter->q_vector[i]) { 3814 napi_synchronize(&adapter->q_vector[i]->napi); 3815 napi_disable(&adapter->q_vector[i]->napi); 3816 } 3817 } 3818 3819 del_timer_sync(&adapter->watchdog_timer); 3820 del_timer_sync(&adapter->phy_info_timer); 3821 3822 /* record the stats before reset*/ 3823 spin_lock(&adapter->stats64_lock); 3824 igc_update_stats(adapter); 3825 spin_unlock(&adapter->stats64_lock); 3826 3827 adapter->link_speed = 0; 3828 adapter->link_duplex = 0; 3829 3830 if (!pci_channel_offline(adapter->pdev)) 3831 igc_reset(adapter); 3832 3833 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 3834 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 3835 3836 igc_clean_all_tx_rings(adapter); 3837 igc_clean_all_rx_rings(adapter); 3838 } 3839 3840 void igc_reinit_locked(struct igc_adapter *adapter) 3841 { 3842 WARN_ON(in_interrupt()); 3843 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3844 usleep_range(1000, 2000); 3845 igc_down(adapter); 3846 igc_up(adapter); 3847 clear_bit(__IGC_RESETTING, &adapter->state); 3848 } 3849 3850 static void igc_reset_task(struct work_struct *work) 3851 { 3852 struct igc_adapter *adapter; 3853 3854 adapter = container_of(work, struct igc_adapter, reset_task); 3855 3856 igc_rings_dump(adapter); 3857 igc_regs_dump(adapter); 3858 netdev_err(adapter->netdev, "Reset adapter\n"); 3859 igc_reinit_locked(adapter); 3860 } 3861 3862 /** 3863 * igc_change_mtu - Change the Maximum Transfer Unit 3864 * @netdev: network interface device structure 3865 * @new_mtu: new value for maximum frame size 3866 * 3867 * Returns 0 on success, negative on failure 3868 */ 3869 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 3870 { 3871 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 3872 struct igc_adapter *adapter = netdev_priv(netdev); 3873 3874 /* adjust max frame to be at least the size of a standard frame */ 3875 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 3876 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 3877 3878 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3879 usleep_range(1000, 2000); 3880 3881 /* igc_down has a dependency on max_frame_size */ 3882 adapter->max_frame_size = max_frame; 3883 3884 if (netif_running(netdev)) 3885 igc_down(adapter); 3886 3887 netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 3888 netdev->mtu = new_mtu; 3889 3890 if (netif_running(netdev)) 3891 igc_up(adapter); 3892 else 3893 igc_reset(adapter); 3894 3895 clear_bit(__IGC_RESETTING, &adapter->state); 3896 3897 return 0; 3898 } 3899 3900 /** 3901 * igc_get_stats - Get System Network Statistics 3902 * @netdev: network interface device structure 3903 * 3904 * Returns the address of the device statistics structure. 3905 * The statistics are updated here and also from the timer callback. 3906 */ 3907 static struct net_device_stats *igc_get_stats(struct net_device *netdev) 3908 { 3909 struct igc_adapter *adapter = netdev_priv(netdev); 3910 3911 if (!test_bit(__IGC_RESETTING, &adapter->state)) 3912 igc_update_stats(adapter); 3913 3914 /* only return the current stats */ 3915 return &netdev->stats; 3916 } 3917 3918 static netdev_features_t igc_fix_features(struct net_device *netdev, 3919 netdev_features_t features) 3920 { 3921 /* Since there is no support for separate Rx/Tx vlan accel 3922 * enable/disable make sure Tx flag is always in same state as Rx. 3923 */ 3924 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3925 features |= NETIF_F_HW_VLAN_CTAG_TX; 3926 else 3927 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 3928 3929 return features; 3930 } 3931 3932 static int igc_set_features(struct net_device *netdev, 3933 netdev_features_t features) 3934 { 3935 netdev_features_t changed = netdev->features ^ features; 3936 struct igc_adapter *adapter = netdev_priv(netdev); 3937 3938 /* Add VLAN support */ 3939 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 3940 return 0; 3941 3942 if (!(features & NETIF_F_NTUPLE)) 3943 igc_flush_nfc_rules(adapter); 3944 3945 netdev->features = features; 3946 3947 if (netif_running(netdev)) 3948 igc_reinit_locked(adapter); 3949 else 3950 igc_reset(adapter); 3951 3952 return 1; 3953 } 3954 3955 static netdev_features_t 3956 igc_features_check(struct sk_buff *skb, struct net_device *dev, 3957 netdev_features_t features) 3958 { 3959 unsigned int network_hdr_len, mac_hdr_len; 3960 3961 /* Make certain the headers can be described by a context descriptor */ 3962 mac_hdr_len = skb_network_header(skb) - skb->data; 3963 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 3964 return features & ~(NETIF_F_HW_CSUM | 3965 NETIF_F_SCTP_CRC | 3966 NETIF_F_HW_VLAN_CTAG_TX | 3967 NETIF_F_TSO | 3968 NETIF_F_TSO6); 3969 3970 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 3971 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 3972 return features & ~(NETIF_F_HW_CSUM | 3973 NETIF_F_SCTP_CRC | 3974 NETIF_F_TSO | 3975 NETIF_F_TSO6); 3976 3977 /* We can only support IPv4 TSO in tunnels if we can mangle the 3978 * inner IP ID field, so strip TSO if MANGLEID is not supported. 3979 */ 3980 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 3981 features &= ~NETIF_F_TSO; 3982 3983 return features; 3984 } 3985 3986 static void igc_tsync_interrupt(struct igc_adapter *adapter) 3987 { 3988 struct igc_hw *hw = &adapter->hw; 3989 u32 tsicr = rd32(IGC_TSICR); 3990 u32 ack = 0; 3991 3992 if (tsicr & IGC_TSICR_TXTS) { 3993 /* retrieve hardware timestamp */ 3994 schedule_work(&adapter->ptp_tx_work); 3995 ack |= IGC_TSICR_TXTS; 3996 } 3997 3998 /* acknowledge the interrupts */ 3999 wr32(IGC_TSICR, ack); 4000 } 4001 4002 /** 4003 * igc_msix_other - msix other interrupt handler 4004 * @irq: interrupt number 4005 * @data: pointer to a q_vector 4006 */ 4007 static irqreturn_t igc_msix_other(int irq, void *data) 4008 { 4009 struct igc_adapter *adapter = data; 4010 struct igc_hw *hw = &adapter->hw; 4011 u32 icr = rd32(IGC_ICR); 4012 4013 /* reading ICR causes bit 31 of EICR to be cleared */ 4014 if (icr & IGC_ICR_DRSTA) 4015 schedule_work(&adapter->reset_task); 4016 4017 if (icr & IGC_ICR_DOUTSYNC) { 4018 /* HW is reporting DMA is out of sync */ 4019 adapter->stats.doosync++; 4020 } 4021 4022 if (icr & IGC_ICR_LSC) { 4023 hw->mac.get_link_status = 1; 4024 /* guard against interrupt when we're going down */ 4025 if (!test_bit(__IGC_DOWN, &adapter->state)) 4026 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4027 } 4028 4029 if (icr & IGC_ICR_TS) 4030 igc_tsync_interrupt(adapter); 4031 4032 wr32(IGC_EIMS, adapter->eims_other); 4033 4034 return IRQ_HANDLED; 4035 } 4036 4037 static void igc_write_itr(struct igc_q_vector *q_vector) 4038 { 4039 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 4040 4041 if (!q_vector->set_itr) 4042 return; 4043 4044 if (!itr_val) 4045 itr_val = IGC_ITR_VAL_MASK; 4046 4047 itr_val |= IGC_EITR_CNT_IGNR; 4048 4049 writel(itr_val, q_vector->itr_register); 4050 q_vector->set_itr = 0; 4051 } 4052 4053 static irqreturn_t igc_msix_ring(int irq, void *data) 4054 { 4055 struct igc_q_vector *q_vector = data; 4056 4057 /* Write the ITR value calculated from the previous interrupt. */ 4058 igc_write_itr(q_vector); 4059 4060 napi_schedule(&q_vector->napi); 4061 4062 return IRQ_HANDLED; 4063 } 4064 4065 /** 4066 * igc_request_msix - Initialize MSI-X interrupts 4067 * @adapter: Pointer to adapter structure 4068 * 4069 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 4070 * kernel. 4071 */ 4072 static int igc_request_msix(struct igc_adapter *adapter) 4073 { 4074 int i = 0, err = 0, vector = 0, free_vector = 0; 4075 struct net_device *netdev = adapter->netdev; 4076 4077 err = request_irq(adapter->msix_entries[vector].vector, 4078 &igc_msix_other, 0, netdev->name, adapter); 4079 if (err) 4080 goto err_out; 4081 4082 for (i = 0; i < adapter->num_q_vectors; i++) { 4083 struct igc_q_vector *q_vector = adapter->q_vector[i]; 4084 4085 vector++; 4086 4087 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 4088 4089 if (q_vector->rx.ring && q_vector->tx.ring) 4090 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 4091 q_vector->rx.ring->queue_index); 4092 else if (q_vector->tx.ring) 4093 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 4094 q_vector->tx.ring->queue_index); 4095 else if (q_vector->rx.ring) 4096 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 4097 q_vector->rx.ring->queue_index); 4098 else 4099 sprintf(q_vector->name, "%s-unused", netdev->name); 4100 4101 err = request_irq(adapter->msix_entries[vector].vector, 4102 igc_msix_ring, 0, q_vector->name, 4103 q_vector); 4104 if (err) 4105 goto err_free; 4106 } 4107 4108 igc_configure_msix(adapter); 4109 return 0; 4110 4111 err_free: 4112 /* free already assigned IRQs */ 4113 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 4114 4115 vector--; 4116 for (i = 0; i < vector; i++) { 4117 free_irq(adapter->msix_entries[free_vector++].vector, 4118 adapter->q_vector[i]); 4119 } 4120 err_out: 4121 return err; 4122 } 4123 4124 /** 4125 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 4126 * @adapter: Pointer to adapter structure 4127 * 4128 * This function resets the device so that it has 0 rx queues, tx queues, and 4129 * MSI-X interrupts allocated. 4130 */ 4131 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 4132 { 4133 igc_free_q_vectors(adapter); 4134 igc_reset_interrupt_capability(adapter); 4135 } 4136 4137 /* Need to wait a few seconds after link up to get diagnostic information from 4138 * the phy 4139 */ 4140 static void igc_update_phy_info(struct timer_list *t) 4141 { 4142 struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); 4143 4144 igc_get_phy_info(&adapter->hw); 4145 } 4146 4147 /** 4148 * igc_has_link - check shared code for link and determine up/down 4149 * @adapter: pointer to driver private info 4150 */ 4151 bool igc_has_link(struct igc_adapter *adapter) 4152 { 4153 struct igc_hw *hw = &adapter->hw; 4154 bool link_active = false; 4155 4156 /* get_link_status is set on LSC (link status) interrupt or 4157 * rx sequence error interrupt. get_link_status will stay 4158 * false until the igc_check_for_link establishes link 4159 * for copper adapters ONLY 4160 */ 4161 switch (hw->phy.media_type) { 4162 case igc_media_type_copper: 4163 if (!hw->mac.get_link_status) 4164 return true; 4165 hw->mac.ops.check_for_link(hw); 4166 link_active = !hw->mac.get_link_status; 4167 break; 4168 default: 4169 case igc_media_type_unknown: 4170 break; 4171 } 4172 4173 if (hw->mac.type == igc_i225 && 4174 hw->phy.id == I225_I_PHY_ID) { 4175 if (!netif_carrier_ok(adapter->netdev)) { 4176 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4177 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 4178 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 4179 adapter->link_check_timeout = jiffies; 4180 } 4181 } 4182 4183 return link_active; 4184 } 4185 4186 /** 4187 * igc_watchdog - Timer Call-back 4188 * @t: timer for the watchdog 4189 */ 4190 static void igc_watchdog(struct timer_list *t) 4191 { 4192 struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); 4193 /* Do the rest outside of interrupt context */ 4194 schedule_work(&adapter->watchdog_task); 4195 } 4196 4197 static void igc_watchdog_task(struct work_struct *work) 4198 { 4199 struct igc_adapter *adapter = container_of(work, 4200 struct igc_adapter, 4201 watchdog_task); 4202 struct net_device *netdev = adapter->netdev; 4203 struct igc_hw *hw = &adapter->hw; 4204 struct igc_phy_info *phy = &hw->phy; 4205 u16 phy_data, retry_count = 20; 4206 u32 link; 4207 int i; 4208 4209 link = igc_has_link(adapter); 4210 4211 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 4212 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 4213 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4214 else 4215 link = false; 4216 } 4217 4218 if (link) { 4219 /* Cancel scheduled suspend requests. */ 4220 pm_runtime_resume(netdev->dev.parent); 4221 4222 if (!netif_carrier_ok(netdev)) { 4223 u32 ctrl; 4224 4225 hw->mac.ops.get_speed_and_duplex(hw, 4226 &adapter->link_speed, 4227 &adapter->link_duplex); 4228 4229 ctrl = rd32(IGC_CTRL); 4230 /* Link status message must follow this format */ 4231 netdev_info(netdev, 4232 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 4233 adapter->link_speed, 4234 adapter->link_duplex == FULL_DUPLEX ? 4235 "Full" : "Half", 4236 (ctrl & IGC_CTRL_TFCE) && 4237 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 4238 (ctrl & IGC_CTRL_RFCE) ? "RX" : 4239 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 4240 4241 /* disable EEE if enabled */ 4242 if ((adapter->flags & IGC_FLAG_EEE) && 4243 adapter->link_duplex == HALF_DUPLEX) { 4244 netdev_info(netdev, 4245 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n"); 4246 adapter->hw.dev_spec._base.eee_enable = false; 4247 adapter->flags &= ~IGC_FLAG_EEE; 4248 } 4249 4250 /* check if SmartSpeed worked */ 4251 igc_check_downshift(hw); 4252 if (phy->speed_downgraded) 4253 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 4254 4255 /* adjust timeout factor according to speed/duplex */ 4256 adapter->tx_timeout_factor = 1; 4257 switch (adapter->link_speed) { 4258 case SPEED_10: 4259 adapter->tx_timeout_factor = 14; 4260 break; 4261 case SPEED_100: 4262 /* maybe add some timeout factor ? */ 4263 break; 4264 } 4265 4266 if (adapter->link_speed != SPEED_1000) 4267 goto no_wait; 4268 4269 /* wait for Remote receiver status OK */ 4270 retry_read_status: 4271 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 4272 &phy_data)) { 4273 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 4274 retry_count) { 4275 msleep(100); 4276 retry_count--; 4277 goto retry_read_status; 4278 } else if (!retry_count) { 4279 netdev_err(netdev, "exceed max 2 second\n"); 4280 } 4281 } else { 4282 netdev_err(netdev, "read 1000Base-T Status Reg\n"); 4283 } 4284 no_wait: 4285 netif_carrier_on(netdev); 4286 4287 /* link state has changed, schedule phy info update */ 4288 if (!test_bit(__IGC_DOWN, &adapter->state)) 4289 mod_timer(&adapter->phy_info_timer, 4290 round_jiffies(jiffies + 2 * HZ)); 4291 } 4292 } else { 4293 if (netif_carrier_ok(netdev)) { 4294 adapter->link_speed = 0; 4295 adapter->link_duplex = 0; 4296 4297 /* Links status message must follow this format */ 4298 netdev_info(netdev, "NIC Link is Down\n"); 4299 netif_carrier_off(netdev); 4300 4301 /* link state has changed, schedule phy info update */ 4302 if (!test_bit(__IGC_DOWN, &adapter->state)) 4303 mod_timer(&adapter->phy_info_timer, 4304 round_jiffies(jiffies + 2 * HZ)); 4305 4306 /* link is down, time to check for alternate media */ 4307 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 4308 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4309 schedule_work(&adapter->reset_task); 4310 /* return immediately */ 4311 return; 4312 } 4313 } 4314 pm_schedule_suspend(netdev->dev.parent, 4315 MSEC_PER_SEC * 5); 4316 4317 /* also check for alternate media here */ 4318 } else if (!netif_carrier_ok(netdev) && 4319 (adapter->flags & IGC_FLAG_MAS_ENABLE)) { 4320 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4321 schedule_work(&adapter->reset_task); 4322 /* return immediately */ 4323 return; 4324 } 4325 } 4326 } 4327 4328 spin_lock(&adapter->stats64_lock); 4329 igc_update_stats(adapter); 4330 spin_unlock(&adapter->stats64_lock); 4331 4332 for (i = 0; i < adapter->num_tx_queues; i++) { 4333 struct igc_ring *tx_ring = adapter->tx_ring[i]; 4334 4335 if (!netif_carrier_ok(netdev)) { 4336 /* We've lost link, so the controller stops DMA, 4337 * but we've got queued Tx work that's never going 4338 * to get done, so reset controller to flush Tx. 4339 * (Do the reset outside of interrupt context). 4340 */ 4341 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 4342 adapter->tx_timeout_count++; 4343 schedule_work(&adapter->reset_task); 4344 /* return immediately since reset is imminent */ 4345 return; 4346 } 4347 } 4348 4349 /* Force detection of hung controller every watchdog period */ 4350 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 4351 } 4352 4353 /* Cause software interrupt to ensure Rx ring is cleaned */ 4354 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4355 u32 eics = 0; 4356 4357 for (i = 0; i < adapter->num_q_vectors; i++) 4358 eics |= adapter->q_vector[i]->eims_value; 4359 wr32(IGC_EICS, eics); 4360 } else { 4361 wr32(IGC_ICS, IGC_ICS_RXDMT0); 4362 } 4363 4364 igc_ptp_tx_hang(adapter); 4365 4366 /* Reset the timer */ 4367 if (!test_bit(__IGC_DOWN, &adapter->state)) { 4368 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 4369 mod_timer(&adapter->watchdog_timer, 4370 round_jiffies(jiffies + HZ)); 4371 else 4372 mod_timer(&adapter->watchdog_timer, 4373 round_jiffies(jiffies + 2 * HZ)); 4374 } 4375 } 4376 4377 /** 4378 * igc_intr_msi - Interrupt Handler 4379 * @irq: interrupt number 4380 * @data: pointer to a network interface device structure 4381 */ 4382 static irqreturn_t igc_intr_msi(int irq, void *data) 4383 { 4384 struct igc_adapter *adapter = data; 4385 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4386 struct igc_hw *hw = &adapter->hw; 4387 /* read ICR disables interrupts using IAM */ 4388 u32 icr = rd32(IGC_ICR); 4389 4390 igc_write_itr(q_vector); 4391 4392 if (icr & IGC_ICR_DRSTA) 4393 schedule_work(&adapter->reset_task); 4394 4395 if (icr & IGC_ICR_DOUTSYNC) { 4396 /* HW is reporting DMA is out of sync */ 4397 adapter->stats.doosync++; 4398 } 4399 4400 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4401 hw->mac.get_link_status = 1; 4402 if (!test_bit(__IGC_DOWN, &adapter->state)) 4403 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4404 } 4405 4406 napi_schedule(&q_vector->napi); 4407 4408 return IRQ_HANDLED; 4409 } 4410 4411 /** 4412 * igc_intr - Legacy Interrupt Handler 4413 * @irq: interrupt number 4414 * @data: pointer to a network interface device structure 4415 */ 4416 static irqreturn_t igc_intr(int irq, void *data) 4417 { 4418 struct igc_adapter *adapter = data; 4419 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4420 struct igc_hw *hw = &adapter->hw; 4421 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 4422 * need for the IMC write 4423 */ 4424 u32 icr = rd32(IGC_ICR); 4425 4426 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 4427 * not set, then the adapter didn't send an interrupt 4428 */ 4429 if (!(icr & IGC_ICR_INT_ASSERTED)) 4430 return IRQ_NONE; 4431 4432 igc_write_itr(q_vector); 4433 4434 if (icr & IGC_ICR_DRSTA) 4435 schedule_work(&adapter->reset_task); 4436 4437 if (icr & IGC_ICR_DOUTSYNC) { 4438 /* HW is reporting DMA is out of sync */ 4439 adapter->stats.doosync++; 4440 } 4441 4442 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4443 hw->mac.get_link_status = 1; 4444 /* guard against interrupt when we're going down */ 4445 if (!test_bit(__IGC_DOWN, &adapter->state)) 4446 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4447 } 4448 4449 napi_schedule(&q_vector->napi); 4450 4451 return IRQ_HANDLED; 4452 } 4453 4454 static void igc_free_irq(struct igc_adapter *adapter) 4455 { 4456 if (adapter->msix_entries) { 4457 int vector = 0, i; 4458 4459 free_irq(adapter->msix_entries[vector++].vector, adapter); 4460 4461 for (i = 0; i < adapter->num_q_vectors; i++) 4462 free_irq(adapter->msix_entries[vector++].vector, 4463 adapter->q_vector[i]); 4464 } else { 4465 free_irq(adapter->pdev->irq, adapter); 4466 } 4467 } 4468 4469 /** 4470 * igc_request_irq - initialize interrupts 4471 * @adapter: Pointer to adapter structure 4472 * 4473 * Attempts to configure interrupts using the best available 4474 * capabilities of the hardware and kernel. 4475 */ 4476 static int igc_request_irq(struct igc_adapter *adapter) 4477 { 4478 struct net_device *netdev = adapter->netdev; 4479 struct pci_dev *pdev = adapter->pdev; 4480 int err = 0; 4481 4482 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4483 err = igc_request_msix(adapter); 4484 if (!err) 4485 goto request_done; 4486 /* fall back to MSI */ 4487 igc_free_all_tx_resources(adapter); 4488 igc_free_all_rx_resources(adapter); 4489 4490 igc_clear_interrupt_scheme(adapter); 4491 err = igc_init_interrupt_scheme(adapter, false); 4492 if (err) 4493 goto request_done; 4494 igc_setup_all_tx_resources(adapter); 4495 igc_setup_all_rx_resources(adapter); 4496 igc_configure(adapter); 4497 } 4498 4499 igc_assign_vector(adapter->q_vector[0], 0); 4500 4501 if (adapter->flags & IGC_FLAG_HAS_MSI) { 4502 err = request_irq(pdev->irq, &igc_intr_msi, 0, 4503 netdev->name, adapter); 4504 if (!err) 4505 goto request_done; 4506 4507 /* fall back to legacy interrupts */ 4508 igc_reset_interrupt_capability(adapter); 4509 adapter->flags &= ~IGC_FLAG_HAS_MSI; 4510 } 4511 4512 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 4513 netdev->name, adapter); 4514 4515 if (err) 4516 netdev_err(netdev, "Error %d getting interrupt\n", err); 4517 4518 request_done: 4519 return err; 4520 } 4521 4522 /** 4523 * __igc_open - Called when a network interface is made active 4524 * @netdev: network interface device structure 4525 * @resuming: boolean indicating if the device is resuming 4526 * 4527 * Returns 0 on success, negative value on failure 4528 * 4529 * The open entry point is called when a network interface is made 4530 * active by the system (IFF_UP). At this point all resources needed 4531 * for transmit and receive operations are allocated, the interrupt 4532 * handler is registered with the OS, the watchdog timer is started, 4533 * and the stack is notified that the interface is ready. 4534 */ 4535 static int __igc_open(struct net_device *netdev, bool resuming) 4536 { 4537 struct igc_adapter *adapter = netdev_priv(netdev); 4538 struct pci_dev *pdev = adapter->pdev; 4539 struct igc_hw *hw = &adapter->hw; 4540 int err = 0; 4541 int i = 0; 4542 4543 /* disallow open during test */ 4544 4545 if (test_bit(__IGC_TESTING, &adapter->state)) { 4546 WARN_ON(resuming); 4547 return -EBUSY; 4548 } 4549 4550 if (!resuming) 4551 pm_runtime_get_sync(&pdev->dev); 4552 4553 netif_carrier_off(netdev); 4554 4555 /* allocate transmit descriptors */ 4556 err = igc_setup_all_tx_resources(adapter); 4557 if (err) 4558 goto err_setup_tx; 4559 4560 /* allocate receive descriptors */ 4561 err = igc_setup_all_rx_resources(adapter); 4562 if (err) 4563 goto err_setup_rx; 4564 4565 igc_power_up_link(adapter); 4566 4567 igc_configure(adapter); 4568 4569 err = igc_request_irq(adapter); 4570 if (err) 4571 goto err_req_irq; 4572 4573 /* Notify the stack of the actual queue counts. */ 4574 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 4575 if (err) 4576 goto err_set_queues; 4577 4578 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 4579 if (err) 4580 goto err_set_queues; 4581 4582 clear_bit(__IGC_DOWN, &adapter->state); 4583 4584 for (i = 0; i < adapter->num_q_vectors; i++) 4585 napi_enable(&adapter->q_vector[i]->napi); 4586 4587 /* Clear any pending interrupts. */ 4588 rd32(IGC_ICR); 4589 igc_irq_enable(adapter); 4590 4591 if (!resuming) 4592 pm_runtime_put(&pdev->dev); 4593 4594 netif_tx_start_all_queues(netdev); 4595 4596 /* start the watchdog. */ 4597 hw->mac.get_link_status = 1; 4598 schedule_work(&adapter->watchdog_task); 4599 4600 return IGC_SUCCESS; 4601 4602 err_set_queues: 4603 igc_free_irq(adapter); 4604 err_req_irq: 4605 igc_release_hw_control(adapter); 4606 igc_power_down_phy_copper_base(&adapter->hw); 4607 igc_free_all_rx_resources(adapter); 4608 err_setup_rx: 4609 igc_free_all_tx_resources(adapter); 4610 err_setup_tx: 4611 igc_reset(adapter); 4612 if (!resuming) 4613 pm_runtime_put(&pdev->dev); 4614 4615 return err; 4616 } 4617 4618 int igc_open(struct net_device *netdev) 4619 { 4620 return __igc_open(netdev, false); 4621 } 4622 4623 /** 4624 * __igc_close - Disables a network interface 4625 * @netdev: network interface device structure 4626 * @suspending: boolean indicating the device is suspending 4627 * 4628 * Returns 0, this is not allowed to fail 4629 * 4630 * The close entry point is called when an interface is de-activated 4631 * by the OS. The hardware is still under the driver's control, but 4632 * needs to be disabled. A global MAC reset is issued to stop the 4633 * hardware, and all transmit and receive resources are freed. 4634 */ 4635 static int __igc_close(struct net_device *netdev, bool suspending) 4636 { 4637 struct igc_adapter *adapter = netdev_priv(netdev); 4638 struct pci_dev *pdev = adapter->pdev; 4639 4640 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 4641 4642 if (!suspending) 4643 pm_runtime_get_sync(&pdev->dev); 4644 4645 igc_down(adapter); 4646 4647 igc_release_hw_control(adapter); 4648 4649 igc_free_irq(adapter); 4650 4651 igc_free_all_tx_resources(adapter); 4652 igc_free_all_rx_resources(adapter); 4653 4654 if (!suspending) 4655 pm_runtime_put_sync(&pdev->dev); 4656 4657 return 0; 4658 } 4659 4660 int igc_close(struct net_device *netdev) 4661 { 4662 if (netif_device_present(netdev) || netdev->dismantle) 4663 return __igc_close(netdev, false); 4664 return 0; 4665 } 4666 4667 /** 4668 * igc_ioctl - Access the hwtstamp interface 4669 * @netdev: network interface device structure 4670 * @ifreq: interface request data 4671 * @cmd: ioctl command 4672 **/ 4673 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 4674 { 4675 switch (cmd) { 4676 case SIOCGHWTSTAMP: 4677 return igc_ptp_get_ts_config(netdev, ifr); 4678 case SIOCSHWTSTAMP: 4679 return igc_ptp_set_ts_config(netdev, ifr); 4680 default: 4681 return -EOPNOTSUPP; 4682 } 4683 } 4684 4685 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, 4686 bool enable) 4687 { 4688 struct igc_ring *ring; 4689 int i; 4690 4691 if (queue < 0 || queue >= adapter->num_tx_queues) 4692 return -EINVAL; 4693 4694 ring = adapter->tx_ring[queue]; 4695 ring->launchtime_enable = enable; 4696 4697 if (adapter->base_time) 4698 return 0; 4699 4700 adapter->cycle_time = NSEC_PER_SEC; 4701 4702 for (i = 0; i < adapter->num_tx_queues; i++) { 4703 ring = adapter->tx_ring[i]; 4704 ring->start_time = 0; 4705 ring->end_time = NSEC_PER_SEC; 4706 } 4707 4708 return 0; 4709 } 4710 4711 static bool validate_schedule(const struct tc_taprio_qopt_offload *qopt) 4712 { 4713 int queue_uses[IGC_MAX_TX_QUEUES] = { }; 4714 size_t n; 4715 4716 if (qopt->cycle_time_extension) 4717 return false; 4718 4719 for (n = 0; n < qopt->num_entries; n++) { 4720 const struct tc_taprio_sched_entry *e; 4721 int i; 4722 4723 e = &qopt->entries[n]; 4724 4725 /* i225 only supports "global" frame preemption 4726 * settings. 4727 */ 4728 if (e->command != TC_TAPRIO_CMD_SET_GATES) 4729 return false; 4730 4731 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4732 if (e->gate_mask & BIT(i)) 4733 queue_uses[i]++; 4734 4735 if (queue_uses[i] > 1) 4736 return false; 4737 } 4738 } 4739 4740 return true; 4741 } 4742 4743 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, 4744 struct tc_etf_qopt_offload *qopt) 4745 { 4746 struct igc_hw *hw = &adapter->hw; 4747 int err; 4748 4749 if (hw->mac.type != igc_i225) 4750 return -EOPNOTSUPP; 4751 4752 err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable); 4753 if (err) 4754 return err; 4755 4756 return igc_tsn_offload_apply(adapter); 4757 } 4758 4759 static int igc_save_qbv_schedule(struct igc_adapter *adapter, 4760 struct tc_taprio_qopt_offload *qopt) 4761 { 4762 u32 start_time = 0, end_time = 0; 4763 size_t n; 4764 4765 if (!qopt->enable) { 4766 adapter->base_time = 0; 4767 return 0; 4768 } 4769 4770 if (adapter->base_time) 4771 return -EALREADY; 4772 4773 if (!validate_schedule(qopt)) 4774 return -EINVAL; 4775 4776 adapter->cycle_time = qopt->cycle_time; 4777 adapter->base_time = qopt->base_time; 4778 4779 /* FIXME: be a little smarter about cases when the gate for a 4780 * queue stays open for more than one entry. 4781 */ 4782 for (n = 0; n < qopt->num_entries; n++) { 4783 struct tc_taprio_sched_entry *e = &qopt->entries[n]; 4784 int i; 4785 4786 end_time += e->interval; 4787 4788 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4789 struct igc_ring *ring = adapter->tx_ring[i]; 4790 4791 if (!(e->gate_mask & BIT(i))) 4792 continue; 4793 4794 ring->start_time = start_time; 4795 ring->end_time = end_time; 4796 } 4797 4798 start_time += e->interval; 4799 } 4800 4801 return 0; 4802 } 4803 4804 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, 4805 struct tc_taprio_qopt_offload *qopt) 4806 { 4807 struct igc_hw *hw = &adapter->hw; 4808 int err; 4809 4810 if (hw->mac.type != igc_i225) 4811 return -EOPNOTSUPP; 4812 4813 err = igc_save_qbv_schedule(adapter, qopt); 4814 if (err) 4815 return err; 4816 4817 return igc_tsn_offload_apply(adapter); 4818 } 4819 4820 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, 4821 void *type_data) 4822 { 4823 struct igc_adapter *adapter = netdev_priv(dev); 4824 4825 switch (type) { 4826 case TC_SETUP_QDISC_TAPRIO: 4827 return igc_tsn_enable_qbv_scheduling(adapter, type_data); 4828 4829 case TC_SETUP_QDISC_ETF: 4830 return igc_tsn_enable_launchtime(adapter, type_data); 4831 4832 default: 4833 return -EOPNOTSUPP; 4834 } 4835 } 4836 4837 static const struct net_device_ops igc_netdev_ops = { 4838 .ndo_open = igc_open, 4839 .ndo_stop = igc_close, 4840 .ndo_start_xmit = igc_xmit_frame, 4841 .ndo_set_rx_mode = igc_set_rx_mode, 4842 .ndo_set_mac_address = igc_set_mac, 4843 .ndo_change_mtu = igc_change_mtu, 4844 .ndo_get_stats = igc_get_stats, 4845 .ndo_fix_features = igc_fix_features, 4846 .ndo_set_features = igc_set_features, 4847 .ndo_features_check = igc_features_check, 4848 .ndo_do_ioctl = igc_ioctl, 4849 .ndo_setup_tc = igc_setup_tc, 4850 }; 4851 4852 /* PCIe configuration access */ 4853 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4854 { 4855 struct igc_adapter *adapter = hw->back; 4856 4857 pci_read_config_word(adapter->pdev, reg, value); 4858 } 4859 4860 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4861 { 4862 struct igc_adapter *adapter = hw->back; 4863 4864 pci_write_config_word(adapter->pdev, reg, *value); 4865 } 4866 4867 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4868 { 4869 struct igc_adapter *adapter = hw->back; 4870 4871 if (!pci_is_pcie(adapter->pdev)) 4872 return -IGC_ERR_CONFIG; 4873 4874 pcie_capability_read_word(adapter->pdev, reg, value); 4875 4876 return IGC_SUCCESS; 4877 } 4878 4879 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4880 { 4881 struct igc_adapter *adapter = hw->back; 4882 4883 if (!pci_is_pcie(adapter->pdev)) 4884 return -IGC_ERR_CONFIG; 4885 4886 pcie_capability_write_word(adapter->pdev, reg, *value); 4887 4888 return IGC_SUCCESS; 4889 } 4890 4891 u32 igc_rd32(struct igc_hw *hw, u32 reg) 4892 { 4893 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 4894 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 4895 u32 value = 0; 4896 4897 value = readl(&hw_addr[reg]); 4898 4899 /* reads should not return all F's */ 4900 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 4901 struct net_device *netdev = igc->netdev; 4902 4903 hw->hw_addr = NULL; 4904 netif_device_detach(netdev); 4905 netdev_err(netdev, "PCIe link lost, device now detached\n"); 4906 WARN(pci_device_is_present(igc->pdev), 4907 "igc: Failed to read reg 0x%x!\n", reg); 4908 } 4909 4910 return value; 4911 } 4912 4913 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) 4914 { 4915 struct igc_mac_info *mac = &adapter->hw.mac; 4916 4917 mac->autoneg = 0; 4918 4919 /* Make sure dplx is at most 1 bit and lsb of speed is not set 4920 * for the switch() below to work 4921 */ 4922 if ((spd & 1) || (dplx & ~1)) 4923 goto err_inval; 4924 4925 switch (spd + dplx) { 4926 case SPEED_10 + DUPLEX_HALF: 4927 mac->forced_speed_duplex = ADVERTISE_10_HALF; 4928 break; 4929 case SPEED_10 + DUPLEX_FULL: 4930 mac->forced_speed_duplex = ADVERTISE_10_FULL; 4931 break; 4932 case SPEED_100 + DUPLEX_HALF: 4933 mac->forced_speed_duplex = ADVERTISE_100_HALF; 4934 break; 4935 case SPEED_100 + DUPLEX_FULL: 4936 mac->forced_speed_duplex = ADVERTISE_100_FULL; 4937 break; 4938 case SPEED_1000 + DUPLEX_FULL: 4939 mac->autoneg = 1; 4940 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 4941 break; 4942 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 4943 goto err_inval; 4944 case SPEED_2500 + DUPLEX_FULL: 4945 mac->autoneg = 1; 4946 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 4947 break; 4948 case SPEED_2500 + DUPLEX_HALF: /* not supported */ 4949 default: 4950 goto err_inval; 4951 } 4952 4953 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 4954 adapter->hw.phy.mdix = AUTO_ALL_MODES; 4955 4956 return 0; 4957 4958 err_inval: 4959 netdev_err(adapter->netdev, "Unsupported Speed/Duplex configuration\n"); 4960 return -EINVAL; 4961 } 4962 4963 /** 4964 * igc_probe - Device Initialization Routine 4965 * @pdev: PCI device information struct 4966 * @ent: entry in igc_pci_tbl 4967 * 4968 * Returns 0 on success, negative on failure 4969 * 4970 * igc_probe initializes an adapter identified by a pci_dev structure. 4971 * The OS initialization, configuring the adapter private structure, 4972 * and a hardware reset occur. 4973 */ 4974 static int igc_probe(struct pci_dev *pdev, 4975 const struct pci_device_id *ent) 4976 { 4977 struct igc_adapter *adapter; 4978 struct net_device *netdev; 4979 struct igc_hw *hw; 4980 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 4981 int err, pci_using_dac; 4982 4983 err = pci_enable_device_mem(pdev); 4984 if (err) 4985 return err; 4986 4987 pci_using_dac = 0; 4988 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4989 if (!err) { 4990 pci_using_dac = 1; 4991 } else { 4992 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 4993 if (err) { 4994 dev_err(&pdev->dev, 4995 "No usable DMA configuration, aborting\n"); 4996 goto err_dma; 4997 } 4998 } 4999 5000 err = pci_request_mem_regions(pdev, igc_driver_name); 5001 if (err) 5002 goto err_pci_reg; 5003 5004 pci_enable_pcie_error_reporting(pdev); 5005 5006 pci_set_master(pdev); 5007 5008 err = -ENOMEM; 5009 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 5010 IGC_MAX_TX_QUEUES); 5011 5012 if (!netdev) 5013 goto err_alloc_etherdev; 5014 5015 SET_NETDEV_DEV(netdev, &pdev->dev); 5016 5017 pci_set_drvdata(pdev, netdev); 5018 adapter = netdev_priv(netdev); 5019 adapter->netdev = netdev; 5020 adapter->pdev = pdev; 5021 hw = &adapter->hw; 5022 hw->back = adapter; 5023 adapter->port_num = hw->bus.func; 5024 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 5025 5026 err = pci_save_state(pdev); 5027 if (err) 5028 goto err_ioremap; 5029 5030 err = -EIO; 5031 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 5032 pci_resource_len(pdev, 0)); 5033 if (!adapter->io_addr) 5034 goto err_ioremap; 5035 5036 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 5037 hw->hw_addr = adapter->io_addr; 5038 5039 netdev->netdev_ops = &igc_netdev_ops; 5040 igc_ethtool_set_ops(netdev); 5041 netdev->watchdog_timeo = 5 * HZ; 5042 5043 netdev->mem_start = pci_resource_start(pdev, 0); 5044 netdev->mem_end = pci_resource_end(pdev, 0); 5045 5046 /* PCI config space info */ 5047 hw->vendor_id = pdev->vendor; 5048 hw->device_id = pdev->device; 5049 hw->revision_id = pdev->revision; 5050 hw->subsystem_vendor_id = pdev->subsystem_vendor; 5051 hw->subsystem_device_id = pdev->subsystem_device; 5052 5053 /* Copy the default MAC and PHY function pointers */ 5054 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 5055 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 5056 5057 /* Initialize skew-specific constants */ 5058 err = ei->get_invariants(hw); 5059 if (err) 5060 goto err_sw_init; 5061 5062 /* Add supported features to the features list*/ 5063 netdev->features |= NETIF_F_SG; 5064 netdev->features |= NETIF_F_TSO; 5065 netdev->features |= NETIF_F_TSO6; 5066 netdev->features |= NETIF_F_TSO_ECN; 5067 netdev->features |= NETIF_F_RXCSUM; 5068 netdev->features |= NETIF_F_HW_CSUM; 5069 netdev->features |= NETIF_F_SCTP_CRC; 5070 netdev->features |= NETIF_F_HW_TC; 5071 5072 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 5073 NETIF_F_GSO_GRE_CSUM | \ 5074 NETIF_F_GSO_IPXIP4 | \ 5075 NETIF_F_GSO_IPXIP6 | \ 5076 NETIF_F_GSO_UDP_TUNNEL | \ 5077 NETIF_F_GSO_UDP_TUNNEL_CSUM) 5078 5079 netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; 5080 netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; 5081 5082 /* setup the private structure */ 5083 err = igc_sw_init(adapter); 5084 if (err) 5085 goto err_sw_init; 5086 5087 /* copy netdev features into list of user selectable features */ 5088 netdev->hw_features |= NETIF_F_NTUPLE; 5089 netdev->hw_features |= netdev->features; 5090 5091 if (pci_using_dac) 5092 netdev->features |= NETIF_F_HIGHDMA; 5093 5094 /* MTU range: 68 - 9216 */ 5095 netdev->min_mtu = ETH_MIN_MTU; 5096 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 5097 5098 /* before reading the NVM, reset the controller to put the device in a 5099 * known good starting state 5100 */ 5101 hw->mac.ops.reset_hw(hw); 5102 5103 if (igc_get_flash_presence_i225(hw)) { 5104 if (hw->nvm.ops.validate(hw) < 0) { 5105 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 5106 err = -EIO; 5107 goto err_eeprom; 5108 } 5109 } 5110 5111 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 5112 /* copy the MAC address out of the NVM */ 5113 if (hw->mac.ops.read_mac_addr(hw)) 5114 dev_err(&pdev->dev, "NVM Read Error\n"); 5115 } 5116 5117 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 5118 5119 if (!is_valid_ether_addr(netdev->dev_addr)) { 5120 dev_err(&pdev->dev, "Invalid MAC Address\n"); 5121 err = -EIO; 5122 goto err_eeprom; 5123 } 5124 5125 /* configure RXPBSIZE and TXPBSIZE */ 5126 wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT); 5127 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); 5128 5129 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 5130 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 5131 5132 INIT_WORK(&adapter->reset_task, igc_reset_task); 5133 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 5134 5135 /* Initialize link properties that are user-changeable */ 5136 adapter->fc_autoneg = true; 5137 hw->mac.autoneg = true; 5138 hw->phy.autoneg_advertised = 0xaf; 5139 5140 hw->fc.requested_mode = igc_fc_default; 5141 hw->fc.current_mode = igc_fc_default; 5142 5143 /* By default, support wake on port A */ 5144 adapter->flags |= IGC_FLAG_WOL_SUPPORTED; 5145 5146 /* initialize the wol settings based on the eeprom settings */ 5147 if (adapter->flags & IGC_FLAG_WOL_SUPPORTED) 5148 adapter->wol |= IGC_WUFC_MAG; 5149 5150 device_set_wakeup_enable(&adapter->pdev->dev, 5151 adapter->flags & IGC_FLAG_WOL_SUPPORTED); 5152 5153 /* reset the hardware with the new settings */ 5154 igc_reset(adapter); 5155 5156 /* let the f/w know that the h/w is now under the control of the 5157 * driver. 5158 */ 5159 igc_get_hw_control(adapter); 5160 5161 strncpy(netdev->name, "eth%d", IFNAMSIZ); 5162 err = register_netdev(netdev); 5163 if (err) 5164 goto err_register; 5165 5166 /* carrier off reporting is important to ethtool even BEFORE open */ 5167 netif_carrier_off(netdev); 5168 5169 /* do hw tstamp init after resetting */ 5170 igc_ptp_init(adapter); 5171 5172 /* Check if Media Autosense is enabled */ 5173 adapter->ei = *ei; 5174 5175 /* print pcie link status and MAC address */ 5176 pcie_print_link_status(pdev); 5177 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 5178 5179 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); 5180 /* Disable EEE for internal PHY devices */ 5181 hw->dev_spec._base.eee_enable = false; 5182 adapter->flags &= ~IGC_FLAG_EEE; 5183 igc_set_eee_i225(hw, false, false, false); 5184 5185 pm_runtime_put_noidle(&pdev->dev); 5186 5187 return 0; 5188 5189 err_register: 5190 igc_release_hw_control(adapter); 5191 err_eeprom: 5192 if (!igc_check_reset_block(hw)) 5193 igc_reset_phy(hw); 5194 err_sw_init: 5195 igc_clear_interrupt_scheme(adapter); 5196 iounmap(adapter->io_addr); 5197 err_ioremap: 5198 free_netdev(netdev); 5199 err_alloc_etherdev: 5200 pci_release_mem_regions(pdev); 5201 err_pci_reg: 5202 err_dma: 5203 pci_disable_device(pdev); 5204 return err; 5205 } 5206 5207 /** 5208 * igc_remove - Device Removal Routine 5209 * @pdev: PCI device information struct 5210 * 5211 * igc_remove is called by the PCI subsystem to alert the driver 5212 * that it should release a PCI device. This could be caused by a 5213 * Hot-Plug event, or because the driver is going to be removed from 5214 * memory. 5215 */ 5216 static void igc_remove(struct pci_dev *pdev) 5217 { 5218 struct net_device *netdev = pci_get_drvdata(pdev); 5219 struct igc_adapter *adapter = netdev_priv(netdev); 5220 5221 pm_runtime_get_noresume(&pdev->dev); 5222 5223 igc_flush_nfc_rules(adapter); 5224 5225 igc_ptp_stop(adapter); 5226 5227 set_bit(__IGC_DOWN, &adapter->state); 5228 5229 del_timer_sync(&adapter->watchdog_timer); 5230 del_timer_sync(&adapter->phy_info_timer); 5231 5232 cancel_work_sync(&adapter->reset_task); 5233 cancel_work_sync(&adapter->watchdog_task); 5234 5235 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5236 * would have already happened in close and is redundant. 5237 */ 5238 igc_release_hw_control(adapter); 5239 unregister_netdev(netdev); 5240 5241 igc_clear_interrupt_scheme(adapter); 5242 pci_iounmap(pdev, adapter->io_addr); 5243 pci_release_mem_regions(pdev); 5244 5245 free_netdev(netdev); 5246 5247 pci_disable_pcie_error_reporting(pdev); 5248 5249 pci_disable_device(pdev); 5250 } 5251 5252 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, 5253 bool runtime) 5254 { 5255 struct net_device *netdev = pci_get_drvdata(pdev); 5256 struct igc_adapter *adapter = netdev_priv(netdev); 5257 u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol; 5258 struct igc_hw *hw = &adapter->hw; 5259 u32 ctrl, rctl, status; 5260 bool wake; 5261 5262 rtnl_lock(); 5263 netif_device_detach(netdev); 5264 5265 if (netif_running(netdev)) 5266 __igc_close(netdev, true); 5267 5268 igc_ptp_suspend(adapter); 5269 5270 igc_clear_interrupt_scheme(adapter); 5271 rtnl_unlock(); 5272 5273 status = rd32(IGC_STATUS); 5274 if (status & IGC_STATUS_LU) 5275 wufc &= ~IGC_WUFC_LNKC; 5276 5277 if (wufc) { 5278 igc_setup_rctl(adapter); 5279 igc_set_rx_mode(netdev); 5280 5281 /* turn on all-multi mode if wake on multicast is enabled */ 5282 if (wufc & IGC_WUFC_MC) { 5283 rctl = rd32(IGC_RCTL); 5284 rctl |= IGC_RCTL_MPE; 5285 wr32(IGC_RCTL, rctl); 5286 } 5287 5288 ctrl = rd32(IGC_CTRL); 5289 ctrl |= IGC_CTRL_ADVD3WUC; 5290 wr32(IGC_CTRL, ctrl); 5291 5292 /* Allow time for pending master requests to run */ 5293 igc_disable_pcie_master(hw); 5294 5295 wr32(IGC_WUC, IGC_WUC_PME_EN); 5296 wr32(IGC_WUFC, wufc); 5297 } else { 5298 wr32(IGC_WUC, 0); 5299 wr32(IGC_WUFC, 0); 5300 } 5301 5302 wake = wufc || adapter->en_mng_pt; 5303 if (!wake) 5304 igc_power_down_phy_copper_base(&adapter->hw); 5305 else 5306 igc_power_up_link(adapter); 5307 5308 if (enable_wake) 5309 *enable_wake = wake; 5310 5311 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5312 * would have already happened in close and is redundant. 5313 */ 5314 igc_release_hw_control(adapter); 5315 5316 pci_disable_device(pdev); 5317 5318 return 0; 5319 } 5320 5321 #ifdef CONFIG_PM 5322 static int __maybe_unused igc_runtime_suspend(struct device *dev) 5323 { 5324 return __igc_shutdown(to_pci_dev(dev), NULL, 1); 5325 } 5326 5327 static void igc_deliver_wake_packet(struct net_device *netdev) 5328 { 5329 struct igc_adapter *adapter = netdev_priv(netdev); 5330 struct igc_hw *hw = &adapter->hw; 5331 struct sk_buff *skb; 5332 u32 wupl; 5333 5334 wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK; 5335 5336 /* WUPM stores only the first 128 bytes of the wake packet. 5337 * Read the packet only if we have the whole thing. 5338 */ 5339 if (wupl == 0 || wupl > IGC_WUPM_BYTES) 5340 return; 5341 5342 skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES); 5343 if (!skb) 5344 return; 5345 5346 skb_put(skb, wupl); 5347 5348 /* Ensure reads are 32-bit aligned */ 5349 wupl = roundup(wupl, 4); 5350 5351 memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl); 5352 5353 skb->protocol = eth_type_trans(skb, netdev); 5354 netif_rx(skb); 5355 } 5356 5357 static int __maybe_unused igc_resume(struct device *dev) 5358 { 5359 struct pci_dev *pdev = to_pci_dev(dev); 5360 struct net_device *netdev = pci_get_drvdata(pdev); 5361 struct igc_adapter *adapter = netdev_priv(netdev); 5362 struct igc_hw *hw = &adapter->hw; 5363 u32 err, val; 5364 5365 pci_set_power_state(pdev, PCI_D0); 5366 pci_restore_state(pdev); 5367 pci_save_state(pdev); 5368 5369 if (!pci_device_is_present(pdev)) 5370 return -ENODEV; 5371 err = pci_enable_device_mem(pdev); 5372 if (err) { 5373 netdev_err(netdev, "Cannot enable PCI device from suspend\n"); 5374 return err; 5375 } 5376 pci_set_master(pdev); 5377 5378 pci_enable_wake(pdev, PCI_D3hot, 0); 5379 pci_enable_wake(pdev, PCI_D3cold, 0); 5380 5381 if (igc_init_interrupt_scheme(adapter, true)) { 5382 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5383 return -ENOMEM; 5384 } 5385 5386 igc_reset(adapter); 5387 5388 /* let the f/w know that the h/w is now under the control of the 5389 * driver. 5390 */ 5391 igc_get_hw_control(adapter); 5392 5393 val = rd32(IGC_WUS); 5394 if (val & WAKE_PKT_WUS) 5395 igc_deliver_wake_packet(netdev); 5396 5397 wr32(IGC_WUS, ~0); 5398 5399 rtnl_lock(); 5400 if (!err && netif_running(netdev)) 5401 err = __igc_open(netdev, true); 5402 5403 if (!err) 5404 netif_device_attach(netdev); 5405 rtnl_unlock(); 5406 5407 return err; 5408 } 5409 5410 static int __maybe_unused igc_runtime_resume(struct device *dev) 5411 { 5412 return igc_resume(dev); 5413 } 5414 5415 static int __maybe_unused igc_suspend(struct device *dev) 5416 { 5417 return __igc_shutdown(to_pci_dev(dev), NULL, 0); 5418 } 5419 5420 static int __maybe_unused igc_runtime_idle(struct device *dev) 5421 { 5422 struct net_device *netdev = dev_get_drvdata(dev); 5423 struct igc_adapter *adapter = netdev_priv(netdev); 5424 5425 if (!igc_has_link(adapter)) 5426 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 5427 5428 return -EBUSY; 5429 } 5430 #endif /* CONFIG_PM */ 5431 5432 static void igc_shutdown(struct pci_dev *pdev) 5433 { 5434 bool wake; 5435 5436 __igc_shutdown(pdev, &wake, 0); 5437 5438 if (system_state == SYSTEM_POWER_OFF) { 5439 pci_wake_from_d3(pdev, wake); 5440 pci_set_power_state(pdev, PCI_D3hot); 5441 } 5442 } 5443 5444 /** 5445 * igc_io_error_detected - called when PCI error is detected 5446 * @pdev: Pointer to PCI device 5447 * @state: The current PCI connection state 5448 * 5449 * This function is called after a PCI bus error affecting 5450 * this device has been detected. 5451 **/ 5452 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, 5453 pci_channel_state_t state) 5454 { 5455 struct net_device *netdev = pci_get_drvdata(pdev); 5456 struct igc_adapter *adapter = netdev_priv(netdev); 5457 5458 netif_device_detach(netdev); 5459 5460 if (state == pci_channel_io_perm_failure) 5461 return PCI_ERS_RESULT_DISCONNECT; 5462 5463 if (netif_running(netdev)) 5464 igc_down(adapter); 5465 pci_disable_device(pdev); 5466 5467 /* Request a slot reset. */ 5468 return PCI_ERS_RESULT_NEED_RESET; 5469 } 5470 5471 /** 5472 * igc_io_slot_reset - called after the PCI bus has been reset. 5473 * @pdev: Pointer to PCI device 5474 * 5475 * Restart the card from scratch, as if from a cold-boot. Implementation 5476 * resembles the first-half of the igc_resume routine. 5477 **/ 5478 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) 5479 { 5480 struct net_device *netdev = pci_get_drvdata(pdev); 5481 struct igc_adapter *adapter = netdev_priv(netdev); 5482 struct igc_hw *hw = &adapter->hw; 5483 pci_ers_result_t result; 5484 5485 if (pci_enable_device_mem(pdev)) { 5486 netdev_err(netdev, "Could not re-enable PCI device after reset\n"); 5487 result = PCI_ERS_RESULT_DISCONNECT; 5488 } else { 5489 pci_set_master(pdev); 5490 pci_restore_state(pdev); 5491 pci_save_state(pdev); 5492 5493 pci_enable_wake(pdev, PCI_D3hot, 0); 5494 pci_enable_wake(pdev, PCI_D3cold, 0); 5495 5496 /* In case of PCI error, adapter loses its HW address 5497 * so we should re-assign it here. 5498 */ 5499 hw->hw_addr = adapter->io_addr; 5500 5501 igc_reset(adapter); 5502 wr32(IGC_WUS, ~0); 5503 result = PCI_ERS_RESULT_RECOVERED; 5504 } 5505 5506 return result; 5507 } 5508 5509 /** 5510 * igc_io_resume - called when traffic can start to flow again. 5511 * @pdev: Pointer to PCI device 5512 * 5513 * This callback is called when the error recovery driver tells us that 5514 * its OK to resume normal operation. Implementation resembles the 5515 * second-half of the igc_resume routine. 5516 */ 5517 static void igc_io_resume(struct pci_dev *pdev) 5518 { 5519 struct net_device *netdev = pci_get_drvdata(pdev); 5520 struct igc_adapter *adapter = netdev_priv(netdev); 5521 5522 rtnl_lock(); 5523 if (netif_running(netdev)) { 5524 if (igc_open(netdev)) { 5525 netdev_err(netdev, "igc_open failed after reset\n"); 5526 return; 5527 } 5528 } 5529 5530 netif_device_attach(netdev); 5531 5532 /* let the f/w know that the h/w is now under the control of the 5533 * driver. 5534 */ 5535 igc_get_hw_control(adapter); 5536 rtnl_unlock(); 5537 } 5538 5539 static const struct pci_error_handlers igc_err_handler = { 5540 .error_detected = igc_io_error_detected, 5541 .slot_reset = igc_io_slot_reset, 5542 .resume = igc_io_resume, 5543 }; 5544 5545 #ifdef CONFIG_PM 5546 static const struct dev_pm_ops igc_pm_ops = { 5547 SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume) 5548 SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume, 5549 igc_runtime_idle) 5550 }; 5551 #endif 5552 5553 static struct pci_driver igc_driver = { 5554 .name = igc_driver_name, 5555 .id_table = igc_pci_tbl, 5556 .probe = igc_probe, 5557 .remove = igc_remove, 5558 #ifdef CONFIG_PM 5559 .driver.pm = &igc_pm_ops, 5560 #endif 5561 .shutdown = igc_shutdown, 5562 .err_handler = &igc_err_handler, 5563 }; 5564 5565 /** 5566 * igc_reinit_queues - return error 5567 * @adapter: pointer to adapter structure 5568 */ 5569 int igc_reinit_queues(struct igc_adapter *adapter) 5570 { 5571 struct net_device *netdev = adapter->netdev; 5572 int err = 0; 5573 5574 if (netif_running(netdev)) 5575 igc_close(netdev); 5576 5577 igc_reset_interrupt_capability(adapter); 5578 5579 if (igc_init_interrupt_scheme(adapter, true)) { 5580 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5581 return -ENOMEM; 5582 } 5583 5584 if (netif_running(netdev)) 5585 err = igc_open(netdev); 5586 5587 return err; 5588 } 5589 5590 /** 5591 * igc_get_hw_dev - return device 5592 * @hw: pointer to hardware structure 5593 * 5594 * used by hardware layer to print debugging information 5595 */ 5596 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 5597 { 5598 struct igc_adapter *adapter = hw->back; 5599 5600 return adapter->netdev; 5601 } 5602 5603 /** 5604 * igc_init_module - Driver Registration Routine 5605 * 5606 * igc_init_module is the first routine called when the driver is 5607 * loaded. All it does is register with the PCI subsystem. 5608 */ 5609 static int __init igc_init_module(void) 5610 { 5611 int ret; 5612 5613 pr_info("%s\n", igc_driver_string); 5614 pr_info("%s\n", igc_copyright); 5615 5616 ret = pci_register_driver(&igc_driver); 5617 return ret; 5618 } 5619 5620 module_init(igc_init_module); 5621 5622 /** 5623 * igc_exit_module - Driver Exit Cleanup Routine 5624 * 5625 * igc_exit_module is called just before the driver is removed 5626 * from memory. 5627 */ 5628 static void __exit igc_exit_module(void) 5629 { 5630 pci_unregister_driver(&igc_driver); 5631 } 5632 5633 module_exit(igc_exit_module); 5634 /* igc_main.c */ 5635