1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 1999 - 2018 Intel Corporation. */ 3 4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 5 6 #include <linux/module.h> 7 #include <linux/types.h> 8 #include <linux/init.h> 9 #include <linux/pci.h> 10 #include <linux/vmalloc.h> 11 #include <linux/pagemap.h> 12 #include <linux/delay.h> 13 #include <linux/netdevice.h> 14 #include <linux/interrupt.h> 15 #include <linux/tcp.h> 16 #include <linux/ipv6.h> 17 #include <linux/slab.h> 18 #include <net/checksum.h> 19 #include <net/ip6_checksum.h> 20 #include <linux/ethtool.h> 21 #include <linux/if_vlan.h> 22 #include <linux/cpu.h> 23 #include <linux/smp.h> 24 #include <linux/pm_qos.h> 25 #include <linux/pm_runtime.h> 26 #include <linux/aer.h> 27 #include <linux/prefetch.h> 28 #include <linux/suspend.h> 29 30 #include "e1000.h" 31 32 char e1000e_driver_name[] = "e1000e"; 33 34 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 35 static int debug = -1; 36 module_param(debug, int, 0); 37 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 38 39 static const struct e1000_info *e1000_info_tbl[] = { 40 [board_82571] = &e1000_82571_info, 41 [board_82572] = &e1000_82572_info, 42 [board_82573] = &e1000_82573_info, 43 [board_82574] = &e1000_82574_info, 44 [board_82583] = &e1000_82583_info, 45 [board_80003es2lan] = &e1000_es2_info, 46 [board_ich8lan] = &e1000_ich8_info, 47 [board_ich9lan] = &e1000_ich9_info, 48 [board_ich10lan] = &e1000_ich10_info, 49 [board_pchlan] = &e1000_pch_info, 50 [board_pch2lan] = &e1000_pch2_info, 51 [board_pch_lpt] = &e1000_pch_lpt_info, 52 [board_pch_spt] = &e1000_pch_spt_info, 53 [board_pch_cnp] = &e1000_pch_cnp_info, 54 }; 55 56 struct e1000_reg_info { 57 u32 ofs; 58 char *name; 59 }; 60 61 static const struct e1000_reg_info e1000_reg_info_tbl[] = { 62 /* General Registers */ 63 {E1000_CTRL, "CTRL"}, 64 {E1000_STATUS, "STATUS"}, 65 {E1000_CTRL_EXT, "CTRL_EXT"}, 66 67 /* Interrupt Registers */ 68 {E1000_ICR, "ICR"}, 69 70 /* Rx Registers */ 71 {E1000_RCTL, "RCTL"}, 72 {E1000_RDLEN(0), "RDLEN"}, 73 {E1000_RDH(0), "RDH"}, 74 {E1000_RDT(0), "RDT"}, 75 {E1000_RDTR, "RDTR"}, 76 {E1000_RXDCTL(0), "RXDCTL"}, 77 {E1000_ERT, "ERT"}, 78 {E1000_RDBAL(0), "RDBAL"}, 79 {E1000_RDBAH(0), "RDBAH"}, 80 {E1000_RDFH, "RDFH"}, 81 {E1000_RDFT, "RDFT"}, 82 {E1000_RDFHS, "RDFHS"}, 83 {E1000_RDFTS, "RDFTS"}, 84 {E1000_RDFPC, "RDFPC"}, 85 86 /* Tx Registers */ 87 {E1000_TCTL, "TCTL"}, 88 {E1000_TDBAL(0), "TDBAL"}, 89 {E1000_TDBAH(0), "TDBAH"}, 90 {E1000_TDLEN(0), "TDLEN"}, 91 {E1000_TDH(0), "TDH"}, 92 {E1000_TDT(0), "TDT"}, 93 {E1000_TIDV, "TIDV"}, 94 {E1000_TXDCTL(0), "TXDCTL"}, 95 {E1000_TADV, "TADV"}, 96 {E1000_TARC(0), "TARC"}, 97 {E1000_TDFH, "TDFH"}, 98 {E1000_TDFT, "TDFT"}, 99 {E1000_TDFHS, "TDFHS"}, 100 {E1000_TDFTS, "TDFTS"}, 101 {E1000_TDFPC, "TDFPC"}, 102 103 /* List Terminator */ 104 {0, NULL} 105 }; 106 107 /** 108 * __ew32_prepare - prepare to write to MAC CSR register on certain parts 109 * @hw: pointer to the HW structure 110 * 111 * When updating the MAC CSR registers, the Manageability Engine (ME) could 112 * be accessing the registers at the same time. Normally, this is handled in 113 * h/w by an arbiter but on some parts there is a bug that acknowledges Host 114 * accesses later than it should which could result in the register to have 115 * an incorrect value. Workaround this by checking the FWSM register which 116 * has bit 24 set while ME is accessing MAC CSR registers, wait if it is set 117 * and try again a number of times. 118 **/ 119 static void __ew32_prepare(struct e1000_hw *hw) 120 { 121 s32 i = E1000_ICH_FWSM_PCIM2PCI_COUNT; 122 123 while ((er32(FWSM) & E1000_ICH_FWSM_PCIM2PCI) && --i) 124 udelay(50); 125 } 126 127 void __ew32(struct e1000_hw *hw, unsigned long reg, u32 val) 128 { 129 if (hw->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 130 __ew32_prepare(hw); 131 132 writel(val, hw->hw_addr + reg); 133 } 134 135 /** 136 * e1000_regdump - register printout routine 137 * @hw: pointer to the HW structure 138 * @reginfo: pointer to the register info table 139 **/ 140 static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo) 141 { 142 int n = 0; 143 char rname[16]; 144 u32 regs[8]; 145 146 switch (reginfo->ofs) { 147 case E1000_RXDCTL(0): 148 for (n = 0; n < 2; n++) 149 regs[n] = __er32(hw, E1000_RXDCTL(n)); 150 break; 151 case E1000_TXDCTL(0): 152 for (n = 0; n < 2; n++) 153 regs[n] = __er32(hw, E1000_TXDCTL(n)); 154 break; 155 case E1000_TARC(0): 156 for (n = 0; n < 2; n++) 157 regs[n] = __er32(hw, E1000_TARC(n)); 158 break; 159 default: 160 pr_info("%-15s %08x\n", 161 reginfo->name, __er32(hw, reginfo->ofs)); 162 return; 163 } 164 165 snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]"); 166 pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]); 167 } 168 169 static void e1000e_dump_ps_pages(struct e1000_adapter *adapter, 170 struct e1000_buffer *bi) 171 { 172 int i; 173 struct e1000_ps_page *ps_page; 174 175 for (i = 0; i < adapter->rx_ps_pages; i++) { 176 ps_page = &bi->ps_pages[i]; 177 178 if (ps_page->page) { 179 pr_info("packet dump for ps_page %d:\n", i); 180 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 181 16, 1, page_address(ps_page->page), 182 PAGE_SIZE, true); 183 } 184 } 185 } 186 187 /** 188 * e1000e_dump - Print registers, Tx-ring and Rx-ring 189 * @adapter: board private structure 190 **/ 191 static void e1000e_dump(struct e1000_adapter *adapter) 192 { 193 struct net_device *netdev = adapter->netdev; 194 struct e1000_hw *hw = &adapter->hw; 195 struct e1000_reg_info *reginfo; 196 struct e1000_ring *tx_ring = adapter->tx_ring; 197 struct e1000_tx_desc *tx_desc; 198 struct my_u0 { 199 __le64 a; 200 __le64 b; 201 } *u0; 202 struct e1000_buffer *buffer_info; 203 struct e1000_ring *rx_ring = adapter->rx_ring; 204 union e1000_rx_desc_packet_split *rx_desc_ps; 205 union e1000_rx_desc_extended *rx_desc; 206 struct my_u1 { 207 __le64 a; 208 __le64 b; 209 __le64 c; 210 __le64 d; 211 } *u1; 212 u32 staterr; 213 int i = 0; 214 215 if (!netif_msg_hw(adapter)) 216 return; 217 218 /* Print netdevice Info */ 219 if (netdev) { 220 dev_info(&adapter->pdev->dev, "Net device Info\n"); 221 pr_info("Device Name state trans_start\n"); 222 pr_info("%-15s %016lX %016lX\n", netdev->name, 223 netdev->state, dev_trans_start(netdev)); 224 } 225 226 /* Print Registers */ 227 dev_info(&adapter->pdev->dev, "Register Dump\n"); 228 pr_info(" Register Name Value\n"); 229 for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl; 230 reginfo->name; reginfo++) { 231 e1000_regdump(hw, reginfo); 232 } 233 234 /* Print Tx Ring Summary */ 235 if (!netdev || !netif_running(netdev)) 236 return; 237 238 dev_info(&adapter->pdev->dev, "Tx Ring Summary\n"); 239 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); 240 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean]; 241 pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n", 242 0, tx_ring->next_to_use, tx_ring->next_to_clean, 243 (unsigned long long)buffer_info->dma, 244 buffer_info->length, 245 buffer_info->next_to_watch, 246 (unsigned long long)buffer_info->time_stamp); 247 248 /* Print Tx Ring */ 249 if (!netif_msg_tx_done(adapter)) 250 goto rx_ring_summary; 251 252 dev_info(&adapter->pdev->dev, "Tx Ring Dump\n"); 253 254 /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended) 255 * 256 * Legacy Transmit Descriptor 257 * +--------------------------------------------------------------+ 258 * 0 | Buffer Address [63:0] (Reserved on Write Back) | 259 * +--------------------------------------------------------------+ 260 * 8 | Special | CSS | Status | CMD | CSO | Length | 261 * +--------------------------------------------------------------+ 262 * 63 48 47 36 35 32 31 24 23 16 15 0 263 * 264 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload 265 * 63 48 47 40 39 32 31 16 15 8 7 0 266 * +----------------------------------------------------------------+ 267 * 0 | TUCSE | TUCS0 | TUCSS | IPCSE | IPCS0 | IPCSS | 268 * +----------------------------------------------------------------+ 269 * 8 | MSS | HDRLEN | RSV | STA | TUCMD | DTYP | PAYLEN | 270 * +----------------------------------------------------------------+ 271 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0 272 * 273 * Extended Data Descriptor (DTYP=0x1) 274 * +----------------------------------------------------------------+ 275 * 0 | Buffer Address [63:0] | 276 * +----------------------------------------------------------------+ 277 * 8 | VLAN tag | POPTS | Rsvd | Status | Command | DTYP | DTALEN | 278 * +----------------------------------------------------------------+ 279 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0 280 */ 281 pr_info("Tl[desc] [address 63:0 ] [SpeCssSCmCsLen] [bi->dma ] leng ntw timestamp bi->skb <-- Legacy format\n"); 282 pr_info("Tc[desc] [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Context format\n"); 283 pr_info("Td[desc] [address 63:0 ] [VlaPoRSCm1Dlen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Data format\n"); 284 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { 285 const char *next_desc; 286 tx_desc = E1000_TX_DESC(*tx_ring, i); 287 buffer_info = &tx_ring->buffer_info[i]; 288 u0 = (struct my_u0 *)tx_desc; 289 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean) 290 next_desc = " NTC/U"; 291 else if (i == tx_ring->next_to_use) 292 next_desc = " NTU"; 293 else if (i == tx_ring->next_to_clean) 294 next_desc = " NTC"; 295 else 296 next_desc = ""; 297 pr_info("T%c[0x%03X] %016llX %016llX %016llX %04X %3X %016llX %p%s\n", 298 (!(le64_to_cpu(u0->b) & BIT(29)) ? 'l' : 299 ((le64_to_cpu(u0->b) & BIT(20)) ? 'd' : 'c')), 300 i, 301 (unsigned long long)le64_to_cpu(u0->a), 302 (unsigned long long)le64_to_cpu(u0->b), 303 (unsigned long long)buffer_info->dma, 304 buffer_info->length, buffer_info->next_to_watch, 305 (unsigned long long)buffer_info->time_stamp, 306 buffer_info->skb, next_desc); 307 308 if (netif_msg_pktdata(adapter) && buffer_info->skb) 309 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 310 16, 1, buffer_info->skb->data, 311 buffer_info->skb->len, true); 312 } 313 314 /* Print Rx Ring Summary */ 315 rx_ring_summary: 316 dev_info(&adapter->pdev->dev, "Rx Ring Summary\n"); 317 pr_info("Queue [NTU] [NTC]\n"); 318 pr_info(" %5d %5X %5X\n", 319 0, rx_ring->next_to_use, rx_ring->next_to_clean); 320 321 /* Print Rx Ring */ 322 if (!netif_msg_rx_status(adapter)) 323 return; 324 325 dev_info(&adapter->pdev->dev, "Rx Ring Dump\n"); 326 switch (adapter->rx_ps_pages) { 327 case 1: 328 case 2: 329 case 3: 330 /* [Extended] Packet Split Receive Descriptor Format 331 * 332 * +-----------------------------------------------------+ 333 * 0 | Buffer Address 0 [63:0] | 334 * +-----------------------------------------------------+ 335 * 8 | Buffer Address 1 [63:0] | 336 * +-----------------------------------------------------+ 337 * 16 | Buffer Address 2 [63:0] | 338 * +-----------------------------------------------------+ 339 * 24 | Buffer Address 3 [63:0] | 340 * +-----------------------------------------------------+ 341 */ 342 pr_info("R [desc] [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma ] [bi->skb] <-- Ext Pkt Split format\n"); 343 /* [Extended] Receive Descriptor (Write-Back) Format 344 * 345 * 63 48 47 32 31 13 12 8 7 4 3 0 346 * +------------------------------------------------------+ 347 * 0 | Packet | IP | Rsvd | MRQ | Rsvd | MRQ RSS | 348 * | Checksum | Ident | | Queue | | Type | 349 * +------------------------------------------------------+ 350 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 351 * +------------------------------------------------------+ 352 * 63 48 47 32 31 20 19 0 353 */ 354 pr_info("RWB[desc] [ck ipid mrqhsh] [vl l0 ee es] [ l3 l2 l1 hs] [reserved ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n"); 355 for (i = 0; i < rx_ring->count; i++) { 356 const char *next_desc; 357 buffer_info = &rx_ring->buffer_info[i]; 358 rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i); 359 u1 = (struct my_u1 *)rx_desc_ps; 360 staterr = 361 le32_to_cpu(rx_desc_ps->wb.middle.status_error); 362 363 if (i == rx_ring->next_to_use) 364 next_desc = " NTU"; 365 else if (i == rx_ring->next_to_clean) 366 next_desc = " NTC"; 367 else 368 next_desc = ""; 369 370 if (staterr & E1000_RXD_STAT_DD) { 371 /* Descriptor Done */ 372 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX ---------------- %p%s\n", 373 "RWB", i, 374 (unsigned long long)le64_to_cpu(u1->a), 375 (unsigned long long)le64_to_cpu(u1->b), 376 (unsigned long long)le64_to_cpu(u1->c), 377 (unsigned long long)le64_to_cpu(u1->d), 378 buffer_info->skb, next_desc); 379 } else { 380 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX %016llX %p%s\n", 381 "R ", i, 382 (unsigned long long)le64_to_cpu(u1->a), 383 (unsigned long long)le64_to_cpu(u1->b), 384 (unsigned long long)le64_to_cpu(u1->c), 385 (unsigned long long)le64_to_cpu(u1->d), 386 (unsigned long long)buffer_info->dma, 387 buffer_info->skb, next_desc); 388 389 if (netif_msg_pktdata(adapter)) 390 e1000e_dump_ps_pages(adapter, 391 buffer_info); 392 } 393 } 394 break; 395 default: 396 case 0: 397 /* Extended Receive Descriptor (Read) Format 398 * 399 * +-----------------------------------------------------+ 400 * 0 | Buffer Address [63:0] | 401 * +-----------------------------------------------------+ 402 * 8 | Reserved | 403 * +-----------------------------------------------------+ 404 */ 405 pr_info("R [desc] [buf addr 63:0 ] [reserved 63:0 ] [bi->dma ] [bi->skb] <-- Ext (Read) format\n"); 406 /* Extended Receive Descriptor (Write-Back) Format 407 * 408 * 63 48 47 32 31 24 23 4 3 0 409 * +------------------------------------------------------+ 410 * | RSS Hash | | | | 411 * 0 +-------------------+ Rsvd | Reserved | MRQ RSS | 412 * | Packet | IP | | | Type | 413 * | Checksum | Ident | | | | 414 * +------------------------------------------------------+ 415 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 416 * +------------------------------------------------------+ 417 * 63 48 47 32 31 20 19 0 418 */ 419 pr_info("RWB[desc] [cs ipid mrq] [vt ln xe xs] [bi->skb] <-- Ext (Write-Back) format\n"); 420 421 for (i = 0; i < rx_ring->count; i++) { 422 const char *next_desc; 423 424 buffer_info = &rx_ring->buffer_info[i]; 425 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 426 u1 = (struct my_u1 *)rx_desc; 427 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 428 429 if (i == rx_ring->next_to_use) 430 next_desc = " NTU"; 431 else if (i == rx_ring->next_to_clean) 432 next_desc = " NTC"; 433 else 434 next_desc = ""; 435 436 if (staterr & E1000_RXD_STAT_DD) { 437 /* Descriptor Done */ 438 pr_info("%s[0x%03X] %016llX %016llX ---------------- %p%s\n", 439 "RWB", i, 440 (unsigned long long)le64_to_cpu(u1->a), 441 (unsigned long long)le64_to_cpu(u1->b), 442 buffer_info->skb, next_desc); 443 } else { 444 pr_info("%s[0x%03X] %016llX %016llX %016llX %p%s\n", 445 "R ", i, 446 (unsigned long long)le64_to_cpu(u1->a), 447 (unsigned long long)le64_to_cpu(u1->b), 448 (unsigned long long)buffer_info->dma, 449 buffer_info->skb, next_desc); 450 451 if (netif_msg_pktdata(adapter) && 452 buffer_info->skb) 453 print_hex_dump(KERN_INFO, "", 454 DUMP_PREFIX_ADDRESS, 16, 455 1, 456 buffer_info->skb->data, 457 adapter->rx_buffer_len, 458 true); 459 } 460 } 461 } 462 } 463 464 /** 465 * e1000_desc_unused - calculate if we have unused descriptors 466 * @ring: pointer to ring struct to perform calculation on 467 **/ 468 static int e1000_desc_unused(struct e1000_ring *ring) 469 { 470 if (ring->next_to_clean > ring->next_to_use) 471 return ring->next_to_clean - ring->next_to_use - 1; 472 473 return ring->count + ring->next_to_clean - ring->next_to_use - 1; 474 } 475 476 /** 477 * e1000e_systim_to_hwtstamp - convert system time value to hw time stamp 478 * @adapter: board private structure 479 * @hwtstamps: time stamp structure to update 480 * @systim: unsigned 64bit system time value. 481 * 482 * Convert the system time value stored in the RX/TXSTMP registers into a 483 * hwtstamp which can be used by the upper level time stamping functions. 484 * 485 * The 'systim_lock' spinlock is used to protect the consistency of the 486 * system time value. This is needed because reading the 64 bit time 487 * value involves reading two 32 bit registers. The first read latches the 488 * value. 489 **/ 490 static void e1000e_systim_to_hwtstamp(struct e1000_adapter *adapter, 491 struct skb_shared_hwtstamps *hwtstamps, 492 u64 systim) 493 { 494 u64 ns; 495 unsigned long flags; 496 497 spin_lock_irqsave(&adapter->systim_lock, flags); 498 ns = timecounter_cyc2time(&adapter->tc, systim); 499 spin_unlock_irqrestore(&adapter->systim_lock, flags); 500 501 memset(hwtstamps, 0, sizeof(*hwtstamps)); 502 hwtstamps->hwtstamp = ns_to_ktime(ns); 503 } 504 505 /** 506 * e1000e_rx_hwtstamp - utility function which checks for Rx time stamp 507 * @adapter: board private structure 508 * @status: descriptor extended error and status field 509 * @skb: particular skb to include time stamp 510 * 511 * If the time stamp is valid, convert it into the timecounter ns value 512 * and store that result into the shhwtstamps structure which is passed 513 * up the network stack. 514 **/ 515 static void e1000e_rx_hwtstamp(struct e1000_adapter *adapter, u32 status, 516 struct sk_buff *skb) 517 { 518 struct e1000_hw *hw = &adapter->hw; 519 u64 rxstmp; 520 521 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP) || 522 !(status & E1000_RXDEXT_STATERR_TST) || 523 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) 524 return; 525 526 /* The Rx time stamp registers contain the time stamp. No other 527 * received packet will be time stamped until the Rx time stamp 528 * registers are read. Because only one packet can be time stamped 529 * at a time, the register values must belong to this packet and 530 * therefore none of the other additional attributes need to be 531 * compared. 532 */ 533 rxstmp = (u64)er32(RXSTMPL); 534 rxstmp |= (u64)er32(RXSTMPH) << 32; 535 e1000e_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), rxstmp); 536 537 adapter->flags2 &= ~FLAG2_CHECK_RX_HWTSTAMP; 538 } 539 540 /** 541 * e1000_receive_skb - helper function to handle Rx indications 542 * @adapter: board private structure 543 * @netdev: pointer to netdev struct 544 * @staterr: descriptor extended error and status field as written by hardware 545 * @vlan: descriptor vlan field as written by hardware (no le/be conversion) 546 * @skb: pointer to sk_buff to be indicated to stack 547 **/ 548 static void e1000_receive_skb(struct e1000_adapter *adapter, 549 struct net_device *netdev, struct sk_buff *skb, 550 u32 staterr, __le16 vlan) 551 { 552 u16 tag = le16_to_cpu(vlan); 553 554 e1000e_rx_hwtstamp(adapter, staterr, skb); 555 556 skb->protocol = eth_type_trans(skb, netdev); 557 558 if (staterr & E1000_RXD_STAT_VP) 559 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag); 560 561 napi_gro_receive(&adapter->napi, skb); 562 } 563 564 /** 565 * e1000_rx_checksum - Receive Checksum Offload 566 * @adapter: board private structure 567 * @status_err: receive descriptor status and error fields 568 * @skb: socket buffer with received data 569 **/ 570 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err, 571 struct sk_buff *skb) 572 { 573 u16 status = (u16)status_err; 574 u8 errors = (u8)(status_err >> 24); 575 576 skb_checksum_none_assert(skb); 577 578 /* Rx checksum disabled */ 579 if (!(adapter->netdev->features & NETIF_F_RXCSUM)) 580 return; 581 582 /* Ignore Checksum bit is set */ 583 if (status & E1000_RXD_STAT_IXSM) 584 return; 585 586 /* TCP/UDP checksum error bit or IP checksum error bit is set */ 587 if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) { 588 /* let the stack verify checksum errors */ 589 adapter->hw_csum_err++; 590 return; 591 } 592 593 /* TCP/UDP Checksum has not been calculated */ 594 if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) 595 return; 596 597 /* It must be a TCP or UDP packet with a valid checksum */ 598 skb->ip_summed = CHECKSUM_UNNECESSARY; 599 adapter->hw_csum_good++; 600 } 601 602 static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i) 603 { 604 struct e1000_adapter *adapter = rx_ring->adapter; 605 struct e1000_hw *hw = &adapter->hw; 606 607 __ew32_prepare(hw); 608 writel(i, rx_ring->tail); 609 610 if (unlikely(i != readl(rx_ring->tail))) { 611 u32 rctl = er32(RCTL); 612 613 ew32(RCTL, rctl & ~E1000_RCTL_EN); 614 e_err("ME firmware caused invalid RDT - resetting\n"); 615 schedule_work(&adapter->reset_task); 616 } 617 } 618 619 static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i) 620 { 621 struct e1000_adapter *adapter = tx_ring->adapter; 622 struct e1000_hw *hw = &adapter->hw; 623 624 __ew32_prepare(hw); 625 writel(i, tx_ring->tail); 626 627 if (unlikely(i != readl(tx_ring->tail))) { 628 u32 tctl = er32(TCTL); 629 630 ew32(TCTL, tctl & ~E1000_TCTL_EN); 631 e_err("ME firmware caused invalid TDT - resetting\n"); 632 schedule_work(&adapter->reset_task); 633 } 634 } 635 636 /** 637 * e1000_alloc_rx_buffers - Replace used receive buffers 638 * @rx_ring: Rx descriptor ring 639 * @cleaned_count: number to reallocate 640 * @gfp: flags for allocation 641 **/ 642 static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring, 643 int cleaned_count, gfp_t gfp) 644 { 645 struct e1000_adapter *adapter = rx_ring->adapter; 646 struct net_device *netdev = adapter->netdev; 647 struct pci_dev *pdev = adapter->pdev; 648 union e1000_rx_desc_extended *rx_desc; 649 struct e1000_buffer *buffer_info; 650 struct sk_buff *skb; 651 unsigned int i; 652 unsigned int bufsz = adapter->rx_buffer_len; 653 654 i = rx_ring->next_to_use; 655 buffer_info = &rx_ring->buffer_info[i]; 656 657 while (cleaned_count--) { 658 skb = buffer_info->skb; 659 if (skb) { 660 skb_trim(skb, 0); 661 goto map_skb; 662 } 663 664 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp); 665 if (!skb) { 666 /* Better luck next round */ 667 adapter->alloc_rx_buff_failed++; 668 break; 669 } 670 671 buffer_info->skb = skb; 672 map_skb: 673 buffer_info->dma = dma_map_single(&pdev->dev, skb->data, 674 adapter->rx_buffer_len, 675 DMA_FROM_DEVICE); 676 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 677 dev_err(&pdev->dev, "Rx DMA map failed\n"); 678 adapter->rx_dma_failed++; 679 break; 680 } 681 682 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 683 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma); 684 685 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) { 686 /* Force memory writes to complete before letting h/w 687 * know there are new descriptors to fetch. (Only 688 * applicable for weak-ordered memory model archs, 689 * such as IA-64). 690 */ 691 wmb(); 692 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 693 e1000e_update_rdt_wa(rx_ring, i); 694 else 695 writel(i, rx_ring->tail); 696 } 697 i++; 698 if (i == rx_ring->count) 699 i = 0; 700 buffer_info = &rx_ring->buffer_info[i]; 701 } 702 703 rx_ring->next_to_use = i; 704 } 705 706 /** 707 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split 708 * @rx_ring: Rx descriptor ring 709 * @cleaned_count: number to reallocate 710 * @gfp: flags for allocation 711 **/ 712 static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring, 713 int cleaned_count, gfp_t gfp) 714 { 715 struct e1000_adapter *adapter = rx_ring->adapter; 716 struct net_device *netdev = adapter->netdev; 717 struct pci_dev *pdev = adapter->pdev; 718 union e1000_rx_desc_packet_split *rx_desc; 719 struct e1000_buffer *buffer_info; 720 struct e1000_ps_page *ps_page; 721 struct sk_buff *skb; 722 unsigned int i, j; 723 724 i = rx_ring->next_to_use; 725 buffer_info = &rx_ring->buffer_info[i]; 726 727 while (cleaned_count--) { 728 rx_desc = E1000_RX_DESC_PS(*rx_ring, i); 729 730 for (j = 0; j < PS_PAGE_BUFFERS; j++) { 731 ps_page = &buffer_info->ps_pages[j]; 732 if (j >= adapter->rx_ps_pages) { 733 /* all unused desc entries get hw null ptr */ 734 rx_desc->read.buffer_addr[j + 1] = 735 ~cpu_to_le64(0); 736 continue; 737 } 738 if (!ps_page->page) { 739 ps_page->page = alloc_page(gfp); 740 if (!ps_page->page) { 741 adapter->alloc_rx_buff_failed++; 742 goto no_buffers; 743 } 744 ps_page->dma = dma_map_page(&pdev->dev, 745 ps_page->page, 746 0, PAGE_SIZE, 747 DMA_FROM_DEVICE); 748 if (dma_mapping_error(&pdev->dev, 749 ps_page->dma)) { 750 dev_err(&adapter->pdev->dev, 751 "Rx DMA page map failed\n"); 752 adapter->rx_dma_failed++; 753 goto no_buffers; 754 } 755 } 756 /* Refresh the desc even if buffer_addrs 757 * didn't change because each write-back 758 * erases this info. 759 */ 760 rx_desc->read.buffer_addr[j + 1] = 761 cpu_to_le64(ps_page->dma); 762 } 763 764 skb = __netdev_alloc_skb_ip_align(netdev, adapter->rx_ps_bsize0, 765 gfp); 766 767 if (!skb) { 768 adapter->alloc_rx_buff_failed++; 769 break; 770 } 771 772 buffer_info->skb = skb; 773 buffer_info->dma = dma_map_single(&pdev->dev, skb->data, 774 adapter->rx_ps_bsize0, 775 DMA_FROM_DEVICE); 776 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 777 dev_err(&pdev->dev, "Rx DMA map failed\n"); 778 adapter->rx_dma_failed++; 779 /* cleanup skb */ 780 dev_kfree_skb_any(skb); 781 buffer_info->skb = NULL; 782 break; 783 } 784 785 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma); 786 787 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) { 788 /* Force memory writes to complete before letting h/w 789 * know there are new descriptors to fetch. (Only 790 * applicable for weak-ordered memory model archs, 791 * such as IA-64). 792 */ 793 wmb(); 794 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 795 e1000e_update_rdt_wa(rx_ring, i << 1); 796 else 797 writel(i << 1, rx_ring->tail); 798 } 799 800 i++; 801 if (i == rx_ring->count) 802 i = 0; 803 buffer_info = &rx_ring->buffer_info[i]; 804 } 805 806 no_buffers: 807 rx_ring->next_to_use = i; 808 } 809 810 /** 811 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers 812 * @rx_ring: Rx descriptor ring 813 * @cleaned_count: number of buffers to allocate this pass 814 * @gfp: flags for allocation 815 **/ 816 817 static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring, 818 int cleaned_count, gfp_t gfp) 819 { 820 struct e1000_adapter *adapter = rx_ring->adapter; 821 struct net_device *netdev = adapter->netdev; 822 struct pci_dev *pdev = adapter->pdev; 823 union e1000_rx_desc_extended *rx_desc; 824 struct e1000_buffer *buffer_info; 825 struct sk_buff *skb; 826 unsigned int i; 827 unsigned int bufsz = 256 - 16; /* for skb_reserve */ 828 829 i = rx_ring->next_to_use; 830 buffer_info = &rx_ring->buffer_info[i]; 831 832 while (cleaned_count--) { 833 skb = buffer_info->skb; 834 if (skb) { 835 skb_trim(skb, 0); 836 goto check_page; 837 } 838 839 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp); 840 if (unlikely(!skb)) { 841 /* Better luck next round */ 842 adapter->alloc_rx_buff_failed++; 843 break; 844 } 845 846 buffer_info->skb = skb; 847 check_page: 848 /* allocate a new page if necessary */ 849 if (!buffer_info->page) { 850 buffer_info->page = alloc_page(gfp); 851 if (unlikely(!buffer_info->page)) { 852 adapter->alloc_rx_buff_failed++; 853 break; 854 } 855 } 856 857 if (!buffer_info->dma) { 858 buffer_info->dma = dma_map_page(&pdev->dev, 859 buffer_info->page, 0, 860 PAGE_SIZE, 861 DMA_FROM_DEVICE); 862 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 863 adapter->alloc_rx_buff_failed++; 864 break; 865 } 866 } 867 868 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 869 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma); 870 871 if (unlikely(++i == rx_ring->count)) 872 i = 0; 873 buffer_info = &rx_ring->buffer_info[i]; 874 } 875 876 if (likely(rx_ring->next_to_use != i)) { 877 rx_ring->next_to_use = i; 878 if (unlikely(i-- == 0)) 879 i = (rx_ring->count - 1); 880 881 /* Force memory writes to complete before letting h/w 882 * know there are new descriptors to fetch. (Only 883 * applicable for weak-ordered memory model archs, 884 * such as IA-64). 885 */ 886 wmb(); 887 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 888 e1000e_update_rdt_wa(rx_ring, i); 889 else 890 writel(i, rx_ring->tail); 891 } 892 } 893 894 static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss, 895 struct sk_buff *skb) 896 { 897 if (netdev->features & NETIF_F_RXHASH) 898 skb_set_hash(skb, le32_to_cpu(rss), PKT_HASH_TYPE_L3); 899 } 900 901 /** 902 * e1000_clean_rx_irq - Send received data up the network stack 903 * @rx_ring: Rx descriptor ring 904 * @work_done: output parameter for indicating completed work 905 * @work_to_do: how many packets we can clean 906 * 907 * the return value indicates whether actual cleaning was done, there 908 * is no guarantee that everything was cleaned 909 **/ 910 static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done, 911 int work_to_do) 912 { 913 struct e1000_adapter *adapter = rx_ring->adapter; 914 struct net_device *netdev = adapter->netdev; 915 struct pci_dev *pdev = adapter->pdev; 916 struct e1000_hw *hw = &adapter->hw; 917 union e1000_rx_desc_extended *rx_desc, *next_rxd; 918 struct e1000_buffer *buffer_info, *next_buffer; 919 u32 length, staterr; 920 unsigned int i; 921 int cleaned_count = 0; 922 bool cleaned = false; 923 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 924 925 i = rx_ring->next_to_clean; 926 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 927 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 928 buffer_info = &rx_ring->buffer_info[i]; 929 930 while (staterr & E1000_RXD_STAT_DD) { 931 struct sk_buff *skb; 932 933 if (*work_done >= work_to_do) 934 break; 935 (*work_done)++; 936 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ 937 938 skb = buffer_info->skb; 939 buffer_info->skb = NULL; 940 941 prefetch(skb->data - NET_IP_ALIGN); 942 943 i++; 944 if (i == rx_ring->count) 945 i = 0; 946 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i); 947 prefetch(next_rxd); 948 949 next_buffer = &rx_ring->buffer_info[i]; 950 951 cleaned = true; 952 cleaned_count++; 953 dma_unmap_single(&pdev->dev, buffer_info->dma, 954 adapter->rx_buffer_len, DMA_FROM_DEVICE); 955 buffer_info->dma = 0; 956 957 length = le16_to_cpu(rx_desc->wb.upper.length); 958 959 /* !EOP means multiple descriptors were used to store a single 960 * packet, if that's the case we need to toss it. In fact, we 961 * need to toss every packet with the EOP bit clear and the 962 * next frame that _does_ have the EOP bit set, as it is by 963 * definition only a frame fragment 964 */ 965 if (unlikely(!(staterr & E1000_RXD_STAT_EOP))) 966 adapter->flags2 |= FLAG2_IS_DISCARDING; 967 968 if (adapter->flags2 & FLAG2_IS_DISCARDING) { 969 /* All receives must fit into a single buffer */ 970 e_dbg("Receive packet consumed multiple buffers\n"); 971 /* recycle */ 972 buffer_info->skb = skb; 973 if (staterr & E1000_RXD_STAT_EOP) 974 adapter->flags2 &= ~FLAG2_IS_DISCARDING; 975 goto next_desc; 976 } 977 978 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) && 979 !(netdev->features & NETIF_F_RXALL))) { 980 /* recycle */ 981 buffer_info->skb = skb; 982 goto next_desc; 983 } 984 985 /* adjust length to remove Ethernet CRC */ 986 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) { 987 /* If configured to store CRC, don't subtract FCS, 988 * but keep the FCS bytes out of the total_rx_bytes 989 * counter 990 */ 991 if (netdev->features & NETIF_F_RXFCS) 992 total_rx_bytes -= 4; 993 else 994 length -= 4; 995 } 996 997 total_rx_bytes += length; 998 total_rx_packets++; 999 1000 /* code added for copybreak, this should improve 1001 * performance for small packets with large amounts 1002 * of reassembly being done in the stack 1003 */ 1004 if (length < copybreak) { 1005 struct sk_buff *new_skb = 1006 napi_alloc_skb(&adapter->napi, length); 1007 if (new_skb) { 1008 skb_copy_to_linear_data_offset(new_skb, 1009 -NET_IP_ALIGN, 1010 (skb->data - 1011 NET_IP_ALIGN), 1012 (length + 1013 NET_IP_ALIGN)); 1014 /* save the skb in buffer_info as good */ 1015 buffer_info->skb = skb; 1016 skb = new_skb; 1017 } 1018 /* else just continue with the old one */ 1019 } 1020 /* end copybreak code */ 1021 skb_put(skb, length); 1022 1023 /* Receive Checksum Offload */ 1024 e1000_rx_checksum(adapter, staterr, skb); 1025 1026 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb); 1027 1028 e1000_receive_skb(adapter, netdev, skb, staterr, 1029 rx_desc->wb.upper.vlan); 1030 1031 next_desc: 1032 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF); 1033 1034 /* return some buffers to hardware, one at a time is too slow */ 1035 if (cleaned_count >= E1000_RX_BUFFER_WRITE) { 1036 adapter->alloc_rx_buf(rx_ring, cleaned_count, 1037 GFP_ATOMIC); 1038 cleaned_count = 0; 1039 } 1040 1041 /* use prefetched values */ 1042 rx_desc = next_rxd; 1043 buffer_info = next_buffer; 1044 1045 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 1046 } 1047 rx_ring->next_to_clean = i; 1048 1049 cleaned_count = e1000_desc_unused(rx_ring); 1050 if (cleaned_count) 1051 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC); 1052 1053 adapter->total_rx_bytes += total_rx_bytes; 1054 adapter->total_rx_packets += total_rx_packets; 1055 return cleaned; 1056 } 1057 1058 static void e1000_put_txbuf(struct e1000_ring *tx_ring, 1059 struct e1000_buffer *buffer_info, 1060 bool drop) 1061 { 1062 struct e1000_adapter *adapter = tx_ring->adapter; 1063 1064 if (buffer_info->dma) { 1065 if (buffer_info->mapped_as_page) 1066 dma_unmap_page(&adapter->pdev->dev, buffer_info->dma, 1067 buffer_info->length, DMA_TO_DEVICE); 1068 else 1069 dma_unmap_single(&adapter->pdev->dev, buffer_info->dma, 1070 buffer_info->length, DMA_TO_DEVICE); 1071 buffer_info->dma = 0; 1072 } 1073 if (buffer_info->skb) { 1074 if (drop) 1075 dev_kfree_skb_any(buffer_info->skb); 1076 else 1077 dev_consume_skb_any(buffer_info->skb); 1078 buffer_info->skb = NULL; 1079 } 1080 buffer_info->time_stamp = 0; 1081 } 1082 1083 static void e1000_print_hw_hang(struct work_struct *work) 1084 { 1085 struct e1000_adapter *adapter = container_of(work, 1086 struct e1000_adapter, 1087 print_hang_task); 1088 struct net_device *netdev = adapter->netdev; 1089 struct e1000_ring *tx_ring = adapter->tx_ring; 1090 unsigned int i = tx_ring->next_to_clean; 1091 unsigned int eop = tx_ring->buffer_info[i].next_to_watch; 1092 struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop); 1093 struct e1000_hw *hw = &adapter->hw; 1094 u16 phy_status, phy_1000t_status, phy_ext_status; 1095 u16 pci_status; 1096 1097 if (test_bit(__E1000_DOWN, &adapter->state)) 1098 return; 1099 1100 if (!adapter->tx_hang_recheck && (adapter->flags2 & FLAG2_DMA_BURST)) { 1101 /* May be block on write-back, flush and detect again 1102 * flush pending descriptor writebacks to memory 1103 */ 1104 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 1105 /* execute the writes immediately */ 1106 e1e_flush(); 1107 /* Due to rare timing issues, write to TIDV again to ensure 1108 * the write is successful 1109 */ 1110 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 1111 /* execute the writes immediately */ 1112 e1e_flush(); 1113 adapter->tx_hang_recheck = true; 1114 return; 1115 } 1116 adapter->tx_hang_recheck = false; 1117 1118 if (er32(TDH(0)) == er32(TDT(0))) { 1119 e_dbg("false hang detected, ignoring\n"); 1120 return; 1121 } 1122 1123 /* Real hang detected */ 1124 netif_stop_queue(netdev); 1125 1126 e1e_rphy(hw, MII_BMSR, &phy_status); 1127 e1e_rphy(hw, MII_STAT1000, &phy_1000t_status); 1128 e1e_rphy(hw, MII_ESTATUS, &phy_ext_status); 1129 1130 pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status); 1131 1132 /* detected Hardware unit hang */ 1133 e_err("Detected Hardware Unit Hang:\n" 1134 " TDH <%x>\n" 1135 " TDT <%x>\n" 1136 " next_to_use <%x>\n" 1137 " next_to_clean <%x>\n" 1138 "buffer_info[next_to_clean]:\n" 1139 " time_stamp <%lx>\n" 1140 " next_to_watch <%x>\n" 1141 " jiffies <%lx>\n" 1142 " next_to_watch.status <%x>\n" 1143 "MAC Status <%x>\n" 1144 "PHY Status <%x>\n" 1145 "PHY 1000BASE-T Status <%x>\n" 1146 "PHY Extended Status <%x>\n" 1147 "PCI Status <%x>\n", 1148 readl(tx_ring->head), readl(tx_ring->tail), tx_ring->next_to_use, 1149 tx_ring->next_to_clean, tx_ring->buffer_info[eop].time_stamp, 1150 eop, jiffies, eop_desc->upper.fields.status, er32(STATUS), 1151 phy_status, phy_1000t_status, phy_ext_status, pci_status); 1152 1153 e1000e_dump(adapter); 1154 1155 /* Suggest workaround for known h/w issue */ 1156 if ((hw->mac.type == e1000_pchlan) && (er32(CTRL) & E1000_CTRL_TFCE)) 1157 e_err("Try turning off Tx pause (flow control) via ethtool\n"); 1158 } 1159 1160 /** 1161 * e1000e_tx_hwtstamp_work - check for Tx time stamp 1162 * @work: pointer to work struct 1163 * 1164 * This work function polls the TSYNCTXCTL valid bit to determine when a 1165 * timestamp has been taken for the current stored skb. The timestamp must 1166 * be for this skb because only one such packet is allowed in the queue. 1167 */ 1168 static void e1000e_tx_hwtstamp_work(struct work_struct *work) 1169 { 1170 struct e1000_adapter *adapter = container_of(work, struct e1000_adapter, 1171 tx_hwtstamp_work); 1172 struct e1000_hw *hw = &adapter->hw; 1173 1174 if (er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID) { 1175 struct sk_buff *skb = adapter->tx_hwtstamp_skb; 1176 struct skb_shared_hwtstamps shhwtstamps; 1177 u64 txstmp; 1178 1179 txstmp = er32(TXSTMPL); 1180 txstmp |= (u64)er32(TXSTMPH) << 32; 1181 1182 e1000e_systim_to_hwtstamp(adapter, &shhwtstamps, txstmp); 1183 1184 /* Clear the global tx_hwtstamp_skb pointer and force writes 1185 * prior to notifying the stack of a Tx timestamp. 1186 */ 1187 adapter->tx_hwtstamp_skb = NULL; 1188 wmb(); /* force write prior to skb_tstamp_tx */ 1189 1190 skb_tstamp_tx(skb, &shhwtstamps); 1191 dev_consume_skb_any(skb); 1192 } else if (time_after(jiffies, adapter->tx_hwtstamp_start 1193 + adapter->tx_timeout_factor * HZ)) { 1194 dev_kfree_skb_any(adapter->tx_hwtstamp_skb); 1195 adapter->tx_hwtstamp_skb = NULL; 1196 adapter->tx_hwtstamp_timeouts++; 1197 e_warn("clearing Tx timestamp hang\n"); 1198 } else { 1199 /* reschedule to check later */ 1200 schedule_work(&adapter->tx_hwtstamp_work); 1201 } 1202 } 1203 1204 /** 1205 * e1000_clean_tx_irq - Reclaim resources after transmit completes 1206 * @tx_ring: Tx descriptor ring 1207 * 1208 * the return value indicates whether actual cleaning was done, there 1209 * is no guarantee that everything was cleaned 1210 **/ 1211 static bool e1000_clean_tx_irq(struct e1000_ring *tx_ring) 1212 { 1213 struct e1000_adapter *adapter = tx_ring->adapter; 1214 struct net_device *netdev = adapter->netdev; 1215 struct e1000_hw *hw = &adapter->hw; 1216 struct e1000_tx_desc *tx_desc, *eop_desc; 1217 struct e1000_buffer *buffer_info; 1218 unsigned int i, eop; 1219 unsigned int count = 0; 1220 unsigned int total_tx_bytes = 0, total_tx_packets = 0; 1221 unsigned int bytes_compl = 0, pkts_compl = 0; 1222 1223 i = tx_ring->next_to_clean; 1224 eop = tx_ring->buffer_info[i].next_to_watch; 1225 eop_desc = E1000_TX_DESC(*tx_ring, eop); 1226 1227 while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) && 1228 (count < tx_ring->count)) { 1229 bool cleaned = false; 1230 1231 dma_rmb(); /* read buffer_info after eop_desc */ 1232 for (; !cleaned; count++) { 1233 tx_desc = E1000_TX_DESC(*tx_ring, i); 1234 buffer_info = &tx_ring->buffer_info[i]; 1235 cleaned = (i == eop); 1236 1237 if (cleaned) { 1238 total_tx_packets += buffer_info->segs; 1239 total_tx_bytes += buffer_info->bytecount; 1240 if (buffer_info->skb) { 1241 bytes_compl += buffer_info->skb->len; 1242 pkts_compl++; 1243 } 1244 } 1245 1246 e1000_put_txbuf(tx_ring, buffer_info, false); 1247 tx_desc->upper.data = 0; 1248 1249 i++; 1250 if (i == tx_ring->count) 1251 i = 0; 1252 } 1253 1254 if (i == tx_ring->next_to_use) 1255 break; 1256 eop = tx_ring->buffer_info[i].next_to_watch; 1257 eop_desc = E1000_TX_DESC(*tx_ring, eop); 1258 } 1259 1260 tx_ring->next_to_clean = i; 1261 1262 netdev_completed_queue(netdev, pkts_compl, bytes_compl); 1263 1264 #define TX_WAKE_THRESHOLD 32 1265 if (count && netif_carrier_ok(netdev) && 1266 e1000_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD) { 1267 /* Make sure that anybody stopping the queue after this 1268 * sees the new next_to_clean. 1269 */ 1270 smp_mb(); 1271 1272 if (netif_queue_stopped(netdev) && 1273 !(test_bit(__E1000_DOWN, &adapter->state))) { 1274 netif_wake_queue(netdev); 1275 ++adapter->restart_queue; 1276 } 1277 } 1278 1279 if (adapter->detect_tx_hung) { 1280 /* Detect a transmit hang in hardware, this serializes the 1281 * check with the clearing of time_stamp and movement of i 1282 */ 1283 adapter->detect_tx_hung = false; 1284 if (tx_ring->buffer_info[i].time_stamp && 1285 time_after(jiffies, tx_ring->buffer_info[i].time_stamp 1286 + (adapter->tx_timeout_factor * HZ)) && 1287 !(er32(STATUS) & E1000_STATUS_TXOFF)) 1288 schedule_work(&adapter->print_hang_task); 1289 else 1290 adapter->tx_hang_recheck = false; 1291 } 1292 adapter->total_tx_bytes += total_tx_bytes; 1293 adapter->total_tx_packets += total_tx_packets; 1294 return count < tx_ring->count; 1295 } 1296 1297 /** 1298 * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split 1299 * @rx_ring: Rx descriptor ring 1300 * @work_done: output parameter for indicating completed work 1301 * @work_to_do: how many packets we can clean 1302 * 1303 * the return value indicates whether actual cleaning was done, there 1304 * is no guarantee that everything was cleaned 1305 **/ 1306 static bool e1000_clean_rx_irq_ps(struct e1000_ring *rx_ring, int *work_done, 1307 int work_to_do) 1308 { 1309 struct e1000_adapter *adapter = rx_ring->adapter; 1310 struct e1000_hw *hw = &adapter->hw; 1311 union e1000_rx_desc_packet_split *rx_desc, *next_rxd; 1312 struct net_device *netdev = adapter->netdev; 1313 struct pci_dev *pdev = adapter->pdev; 1314 struct e1000_buffer *buffer_info, *next_buffer; 1315 struct e1000_ps_page *ps_page; 1316 struct sk_buff *skb; 1317 unsigned int i, j; 1318 u32 length, staterr; 1319 int cleaned_count = 0; 1320 bool cleaned = false; 1321 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1322 1323 i = rx_ring->next_to_clean; 1324 rx_desc = E1000_RX_DESC_PS(*rx_ring, i); 1325 staterr = le32_to_cpu(rx_desc->wb.middle.status_error); 1326 buffer_info = &rx_ring->buffer_info[i]; 1327 1328 while (staterr & E1000_RXD_STAT_DD) { 1329 if (*work_done >= work_to_do) 1330 break; 1331 (*work_done)++; 1332 skb = buffer_info->skb; 1333 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ 1334 1335 /* in the packet split case this is header only */ 1336 prefetch(skb->data - NET_IP_ALIGN); 1337 1338 i++; 1339 if (i == rx_ring->count) 1340 i = 0; 1341 next_rxd = E1000_RX_DESC_PS(*rx_ring, i); 1342 prefetch(next_rxd); 1343 1344 next_buffer = &rx_ring->buffer_info[i]; 1345 1346 cleaned = true; 1347 cleaned_count++; 1348 dma_unmap_single(&pdev->dev, buffer_info->dma, 1349 adapter->rx_ps_bsize0, DMA_FROM_DEVICE); 1350 buffer_info->dma = 0; 1351 1352 /* see !EOP comment in other Rx routine */ 1353 if (!(staterr & E1000_RXD_STAT_EOP)) 1354 adapter->flags2 |= FLAG2_IS_DISCARDING; 1355 1356 if (adapter->flags2 & FLAG2_IS_DISCARDING) { 1357 e_dbg("Packet Split buffers didn't pick up the full packet\n"); 1358 dev_kfree_skb_irq(skb); 1359 if (staterr & E1000_RXD_STAT_EOP) 1360 adapter->flags2 &= ~FLAG2_IS_DISCARDING; 1361 goto next_desc; 1362 } 1363 1364 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) && 1365 !(netdev->features & NETIF_F_RXALL))) { 1366 dev_kfree_skb_irq(skb); 1367 goto next_desc; 1368 } 1369 1370 length = le16_to_cpu(rx_desc->wb.middle.length0); 1371 1372 if (!length) { 1373 e_dbg("Last part of the packet spanning multiple descriptors\n"); 1374 dev_kfree_skb_irq(skb); 1375 goto next_desc; 1376 } 1377 1378 /* Good Receive */ 1379 skb_put(skb, length); 1380 1381 { 1382 /* this looks ugly, but it seems compiler issues make 1383 * it more efficient than reusing j 1384 */ 1385 int l1 = le16_to_cpu(rx_desc->wb.upper.length[0]); 1386 1387 /* page alloc/put takes too long and effects small 1388 * packet throughput, so unsplit small packets and 1389 * save the alloc/put only valid in softirq (napi) 1390 * context to call kmap_* 1391 */ 1392 if (l1 && (l1 <= copybreak) && 1393 ((length + l1) <= adapter->rx_ps_bsize0)) { 1394 u8 *vaddr; 1395 1396 ps_page = &buffer_info->ps_pages[0]; 1397 1398 /* there is no documentation about how to call 1399 * kmap_atomic, so we can't hold the mapping 1400 * very long 1401 */ 1402 dma_sync_single_for_cpu(&pdev->dev, 1403 ps_page->dma, 1404 PAGE_SIZE, 1405 DMA_FROM_DEVICE); 1406 vaddr = kmap_atomic(ps_page->page); 1407 memcpy(skb_tail_pointer(skb), vaddr, l1); 1408 kunmap_atomic(vaddr); 1409 dma_sync_single_for_device(&pdev->dev, 1410 ps_page->dma, 1411 PAGE_SIZE, 1412 DMA_FROM_DEVICE); 1413 1414 /* remove the CRC */ 1415 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) { 1416 if (!(netdev->features & NETIF_F_RXFCS)) 1417 l1 -= 4; 1418 } 1419 1420 skb_put(skb, l1); 1421 goto copydone; 1422 } /* if */ 1423 } 1424 1425 for (j = 0; j < PS_PAGE_BUFFERS; j++) { 1426 length = le16_to_cpu(rx_desc->wb.upper.length[j]); 1427 if (!length) 1428 break; 1429 1430 ps_page = &buffer_info->ps_pages[j]; 1431 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE, 1432 DMA_FROM_DEVICE); 1433 ps_page->dma = 0; 1434 skb_fill_page_desc(skb, j, ps_page->page, 0, length); 1435 ps_page->page = NULL; 1436 skb->len += length; 1437 skb->data_len += length; 1438 skb->truesize += PAGE_SIZE; 1439 } 1440 1441 /* strip the ethernet crc, problem is we're using pages now so 1442 * this whole operation can get a little cpu intensive 1443 */ 1444 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) { 1445 if (!(netdev->features & NETIF_F_RXFCS)) 1446 pskb_trim(skb, skb->len - 4); 1447 } 1448 1449 copydone: 1450 total_rx_bytes += skb->len; 1451 total_rx_packets++; 1452 1453 e1000_rx_checksum(adapter, staterr, skb); 1454 1455 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb); 1456 1457 if (rx_desc->wb.upper.header_status & 1458 cpu_to_le16(E1000_RXDPS_HDRSTAT_HDRSP)) 1459 adapter->rx_hdr_split++; 1460 1461 e1000_receive_skb(adapter, netdev, skb, staterr, 1462 rx_desc->wb.middle.vlan); 1463 1464 next_desc: 1465 rx_desc->wb.middle.status_error &= cpu_to_le32(~0xFF); 1466 buffer_info->skb = NULL; 1467 1468 /* return some buffers to hardware, one at a time is too slow */ 1469 if (cleaned_count >= E1000_RX_BUFFER_WRITE) { 1470 adapter->alloc_rx_buf(rx_ring, cleaned_count, 1471 GFP_ATOMIC); 1472 cleaned_count = 0; 1473 } 1474 1475 /* use prefetched values */ 1476 rx_desc = next_rxd; 1477 buffer_info = next_buffer; 1478 1479 staterr = le32_to_cpu(rx_desc->wb.middle.status_error); 1480 } 1481 rx_ring->next_to_clean = i; 1482 1483 cleaned_count = e1000_desc_unused(rx_ring); 1484 if (cleaned_count) 1485 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC); 1486 1487 adapter->total_rx_bytes += total_rx_bytes; 1488 adapter->total_rx_packets += total_rx_packets; 1489 return cleaned; 1490 } 1491 1492 static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb, 1493 u16 length) 1494 { 1495 bi->page = NULL; 1496 skb->len += length; 1497 skb->data_len += length; 1498 skb->truesize += PAGE_SIZE; 1499 } 1500 1501 /** 1502 * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy 1503 * @rx_ring: Rx descriptor ring 1504 * @work_done: output parameter for indicating completed work 1505 * @work_to_do: how many packets we can clean 1506 * 1507 * the return value indicates whether actual cleaning was done, there 1508 * is no guarantee that everything was cleaned 1509 **/ 1510 static bool e1000_clean_jumbo_rx_irq(struct e1000_ring *rx_ring, int *work_done, 1511 int work_to_do) 1512 { 1513 struct e1000_adapter *adapter = rx_ring->adapter; 1514 struct net_device *netdev = adapter->netdev; 1515 struct pci_dev *pdev = adapter->pdev; 1516 union e1000_rx_desc_extended *rx_desc, *next_rxd; 1517 struct e1000_buffer *buffer_info, *next_buffer; 1518 u32 length, staterr; 1519 unsigned int i; 1520 int cleaned_count = 0; 1521 bool cleaned = false; 1522 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1523 struct skb_shared_info *shinfo; 1524 1525 i = rx_ring->next_to_clean; 1526 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 1527 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 1528 buffer_info = &rx_ring->buffer_info[i]; 1529 1530 while (staterr & E1000_RXD_STAT_DD) { 1531 struct sk_buff *skb; 1532 1533 if (*work_done >= work_to_do) 1534 break; 1535 (*work_done)++; 1536 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ 1537 1538 skb = buffer_info->skb; 1539 buffer_info->skb = NULL; 1540 1541 ++i; 1542 if (i == rx_ring->count) 1543 i = 0; 1544 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i); 1545 prefetch(next_rxd); 1546 1547 next_buffer = &rx_ring->buffer_info[i]; 1548 1549 cleaned = true; 1550 cleaned_count++; 1551 dma_unmap_page(&pdev->dev, buffer_info->dma, PAGE_SIZE, 1552 DMA_FROM_DEVICE); 1553 buffer_info->dma = 0; 1554 1555 length = le16_to_cpu(rx_desc->wb.upper.length); 1556 1557 /* errors is only valid for DD + EOP descriptors */ 1558 if (unlikely((staterr & E1000_RXD_STAT_EOP) && 1559 ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) && 1560 !(netdev->features & NETIF_F_RXALL)))) { 1561 /* recycle both page and skb */ 1562 buffer_info->skb = skb; 1563 /* an error means any chain goes out the window too */ 1564 if (rx_ring->rx_skb_top) 1565 dev_kfree_skb_irq(rx_ring->rx_skb_top); 1566 rx_ring->rx_skb_top = NULL; 1567 goto next_desc; 1568 } 1569 #define rxtop (rx_ring->rx_skb_top) 1570 if (!(staterr & E1000_RXD_STAT_EOP)) { 1571 /* this descriptor is only the beginning (or middle) */ 1572 if (!rxtop) { 1573 /* this is the beginning of a chain */ 1574 rxtop = skb; 1575 skb_fill_page_desc(rxtop, 0, buffer_info->page, 1576 0, length); 1577 } else { 1578 /* this is the middle of a chain */ 1579 shinfo = skb_shinfo(rxtop); 1580 skb_fill_page_desc(rxtop, shinfo->nr_frags, 1581 buffer_info->page, 0, 1582 length); 1583 /* re-use the skb, only consumed the page */ 1584 buffer_info->skb = skb; 1585 } 1586 e1000_consume_page(buffer_info, rxtop, length); 1587 goto next_desc; 1588 } else { 1589 if (rxtop) { 1590 /* end of the chain */ 1591 shinfo = skb_shinfo(rxtop); 1592 skb_fill_page_desc(rxtop, shinfo->nr_frags, 1593 buffer_info->page, 0, 1594 length); 1595 /* re-use the current skb, we only consumed the 1596 * page 1597 */ 1598 buffer_info->skb = skb; 1599 skb = rxtop; 1600 rxtop = NULL; 1601 e1000_consume_page(buffer_info, skb, length); 1602 } else { 1603 /* no chain, got EOP, this buf is the packet 1604 * copybreak to save the put_page/alloc_page 1605 */ 1606 if (length <= copybreak && 1607 skb_tailroom(skb) >= length) { 1608 u8 *vaddr; 1609 vaddr = kmap_atomic(buffer_info->page); 1610 memcpy(skb_tail_pointer(skb), vaddr, 1611 length); 1612 kunmap_atomic(vaddr); 1613 /* re-use the page, so don't erase 1614 * buffer_info->page 1615 */ 1616 skb_put(skb, length); 1617 } else { 1618 skb_fill_page_desc(skb, 0, 1619 buffer_info->page, 0, 1620 length); 1621 e1000_consume_page(buffer_info, skb, 1622 length); 1623 } 1624 } 1625 } 1626 1627 /* Receive Checksum Offload */ 1628 e1000_rx_checksum(adapter, staterr, skb); 1629 1630 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb); 1631 1632 /* probably a little skewed due to removing CRC */ 1633 total_rx_bytes += skb->len; 1634 total_rx_packets++; 1635 1636 /* eth type trans needs skb->data to point to something */ 1637 if (!pskb_may_pull(skb, ETH_HLEN)) { 1638 e_err("pskb_may_pull failed.\n"); 1639 dev_kfree_skb_irq(skb); 1640 goto next_desc; 1641 } 1642 1643 e1000_receive_skb(adapter, netdev, skb, staterr, 1644 rx_desc->wb.upper.vlan); 1645 1646 next_desc: 1647 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF); 1648 1649 /* return some buffers to hardware, one at a time is too slow */ 1650 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) { 1651 adapter->alloc_rx_buf(rx_ring, cleaned_count, 1652 GFP_ATOMIC); 1653 cleaned_count = 0; 1654 } 1655 1656 /* use prefetched values */ 1657 rx_desc = next_rxd; 1658 buffer_info = next_buffer; 1659 1660 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 1661 } 1662 rx_ring->next_to_clean = i; 1663 1664 cleaned_count = e1000_desc_unused(rx_ring); 1665 if (cleaned_count) 1666 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC); 1667 1668 adapter->total_rx_bytes += total_rx_bytes; 1669 adapter->total_rx_packets += total_rx_packets; 1670 return cleaned; 1671 } 1672 1673 /** 1674 * e1000_clean_rx_ring - Free Rx Buffers per Queue 1675 * @rx_ring: Rx descriptor ring 1676 **/ 1677 static void e1000_clean_rx_ring(struct e1000_ring *rx_ring) 1678 { 1679 struct e1000_adapter *adapter = rx_ring->adapter; 1680 struct e1000_buffer *buffer_info; 1681 struct e1000_ps_page *ps_page; 1682 struct pci_dev *pdev = adapter->pdev; 1683 unsigned int i, j; 1684 1685 /* Free all the Rx ring sk_buffs */ 1686 for (i = 0; i < rx_ring->count; i++) { 1687 buffer_info = &rx_ring->buffer_info[i]; 1688 if (buffer_info->dma) { 1689 if (adapter->clean_rx == e1000_clean_rx_irq) 1690 dma_unmap_single(&pdev->dev, buffer_info->dma, 1691 adapter->rx_buffer_len, 1692 DMA_FROM_DEVICE); 1693 else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) 1694 dma_unmap_page(&pdev->dev, buffer_info->dma, 1695 PAGE_SIZE, DMA_FROM_DEVICE); 1696 else if (adapter->clean_rx == e1000_clean_rx_irq_ps) 1697 dma_unmap_single(&pdev->dev, buffer_info->dma, 1698 adapter->rx_ps_bsize0, 1699 DMA_FROM_DEVICE); 1700 buffer_info->dma = 0; 1701 } 1702 1703 if (buffer_info->page) { 1704 put_page(buffer_info->page); 1705 buffer_info->page = NULL; 1706 } 1707 1708 if (buffer_info->skb) { 1709 dev_kfree_skb(buffer_info->skb); 1710 buffer_info->skb = NULL; 1711 } 1712 1713 for (j = 0; j < PS_PAGE_BUFFERS; j++) { 1714 ps_page = &buffer_info->ps_pages[j]; 1715 if (!ps_page->page) 1716 break; 1717 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE, 1718 DMA_FROM_DEVICE); 1719 ps_page->dma = 0; 1720 put_page(ps_page->page); 1721 ps_page->page = NULL; 1722 } 1723 } 1724 1725 /* there also may be some cached data from a chained receive */ 1726 if (rx_ring->rx_skb_top) { 1727 dev_kfree_skb(rx_ring->rx_skb_top); 1728 rx_ring->rx_skb_top = NULL; 1729 } 1730 1731 /* Zero out the descriptor ring */ 1732 memset(rx_ring->desc, 0, rx_ring->size); 1733 1734 rx_ring->next_to_clean = 0; 1735 rx_ring->next_to_use = 0; 1736 adapter->flags2 &= ~FLAG2_IS_DISCARDING; 1737 } 1738 1739 static void e1000e_downshift_workaround(struct work_struct *work) 1740 { 1741 struct e1000_adapter *adapter = container_of(work, 1742 struct e1000_adapter, 1743 downshift_task); 1744 1745 if (test_bit(__E1000_DOWN, &adapter->state)) 1746 return; 1747 1748 e1000e_gig_downshift_workaround_ich8lan(&adapter->hw); 1749 } 1750 1751 /** 1752 * e1000_intr_msi - Interrupt Handler 1753 * @irq: interrupt number 1754 * @data: pointer to a network interface device structure 1755 **/ 1756 static irqreturn_t e1000_intr_msi(int __always_unused irq, void *data) 1757 { 1758 struct net_device *netdev = data; 1759 struct e1000_adapter *adapter = netdev_priv(netdev); 1760 struct e1000_hw *hw = &adapter->hw; 1761 u32 icr = er32(ICR); 1762 1763 /* read ICR disables interrupts using IAM */ 1764 if (icr & E1000_ICR_LSC) { 1765 hw->mac.get_link_status = true; 1766 /* ICH8 workaround-- Call gig speed drop workaround on cable 1767 * disconnect (LSC) before accessing any PHY registers 1768 */ 1769 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) && 1770 (!(er32(STATUS) & E1000_STATUS_LU))) 1771 schedule_work(&adapter->downshift_task); 1772 1773 /* 80003ES2LAN workaround-- For packet buffer work-around on 1774 * link down event; disable receives here in the ISR and reset 1775 * adapter in watchdog 1776 */ 1777 if (netif_carrier_ok(netdev) && 1778 adapter->flags & FLAG_RX_NEEDS_RESTART) { 1779 /* disable receives */ 1780 u32 rctl = er32(RCTL); 1781 1782 ew32(RCTL, rctl & ~E1000_RCTL_EN); 1783 adapter->flags |= FLAG_RESTART_NOW; 1784 } 1785 /* guard against interrupt when we're going down */ 1786 if (!test_bit(__E1000_DOWN, &adapter->state)) 1787 mod_timer(&adapter->watchdog_timer, jiffies + 1); 1788 } 1789 1790 /* Reset on uncorrectable ECC error */ 1791 if ((icr & E1000_ICR_ECCER) && (hw->mac.type >= e1000_pch_lpt)) { 1792 u32 pbeccsts = er32(PBECCSTS); 1793 1794 adapter->corr_errors += 1795 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 1796 adapter->uncorr_errors += 1797 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >> 1798 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT; 1799 1800 /* Do the reset outside of interrupt context */ 1801 schedule_work(&adapter->reset_task); 1802 1803 /* return immediately since reset is imminent */ 1804 return IRQ_HANDLED; 1805 } 1806 1807 if (napi_schedule_prep(&adapter->napi)) { 1808 adapter->total_tx_bytes = 0; 1809 adapter->total_tx_packets = 0; 1810 adapter->total_rx_bytes = 0; 1811 adapter->total_rx_packets = 0; 1812 __napi_schedule(&adapter->napi); 1813 } 1814 1815 return IRQ_HANDLED; 1816 } 1817 1818 /** 1819 * e1000_intr - Interrupt Handler 1820 * @irq: interrupt number 1821 * @data: pointer to a network interface device structure 1822 **/ 1823 static irqreturn_t e1000_intr(int __always_unused irq, void *data) 1824 { 1825 struct net_device *netdev = data; 1826 struct e1000_adapter *adapter = netdev_priv(netdev); 1827 struct e1000_hw *hw = &adapter->hw; 1828 u32 rctl, icr = er32(ICR); 1829 1830 if (!icr || test_bit(__E1000_DOWN, &adapter->state)) 1831 return IRQ_NONE; /* Not our interrupt */ 1832 1833 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 1834 * not set, then the adapter didn't send an interrupt 1835 */ 1836 if (!(icr & E1000_ICR_INT_ASSERTED)) 1837 return IRQ_NONE; 1838 1839 /* Interrupt Auto-Mask...upon reading ICR, 1840 * interrupts are masked. No need for the 1841 * IMC write 1842 */ 1843 1844 if (icr & E1000_ICR_LSC) { 1845 hw->mac.get_link_status = true; 1846 /* ICH8 workaround-- Call gig speed drop workaround on cable 1847 * disconnect (LSC) before accessing any PHY registers 1848 */ 1849 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) && 1850 (!(er32(STATUS) & E1000_STATUS_LU))) 1851 schedule_work(&adapter->downshift_task); 1852 1853 /* 80003ES2LAN workaround-- 1854 * For packet buffer work-around on link down event; 1855 * disable receives here in the ISR and 1856 * reset adapter in watchdog 1857 */ 1858 if (netif_carrier_ok(netdev) && 1859 (adapter->flags & FLAG_RX_NEEDS_RESTART)) { 1860 /* disable receives */ 1861 rctl = er32(RCTL); 1862 ew32(RCTL, rctl & ~E1000_RCTL_EN); 1863 adapter->flags |= FLAG_RESTART_NOW; 1864 } 1865 /* guard against interrupt when we're going down */ 1866 if (!test_bit(__E1000_DOWN, &adapter->state)) 1867 mod_timer(&adapter->watchdog_timer, jiffies + 1); 1868 } 1869 1870 /* Reset on uncorrectable ECC error */ 1871 if ((icr & E1000_ICR_ECCER) && (hw->mac.type >= e1000_pch_lpt)) { 1872 u32 pbeccsts = er32(PBECCSTS); 1873 1874 adapter->corr_errors += 1875 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 1876 adapter->uncorr_errors += 1877 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >> 1878 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT; 1879 1880 /* Do the reset outside of interrupt context */ 1881 schedule_work(&adapter->reset_task); 1882 1883 /* return immediately since reset is imminent */ 1884 return IRQ_HANDLED; 1885 } 1886 1887 if (napi_schedule_prep(&adapter->napi)) { 1888 adapter->total_tx_bytes = 0; 1889 adapter->total_tx_packets = 0; 1890 adapter->total_rx_bytes = 0; 1891 adapter->total_rx_packets = 0; 1892 __napi_schedule(&adapter->napi); 1893 } 1894 1895 return IRQ_HANDLED; 1896 } 1897 1898 static irqreturn_t e1000_msix_other(int __always_unused irq, void *data) 1899 { 1900 struct net_device *netdev = data; 1901 struct e1000_adapter *adapter = netdev_priv(netdev); 1902 struct e1000_hw *hw = &adapter->hw; 1903 u32 icr = er32(ICR); 1904 1905 if (icr & adapter->eiac_mask) 1906 ew32(ICS, (icr & adapter->eiac_mask)); 1907 1908 if (icr & E1000_ICR_LSC) { 1909 hw->mac.get_link_status = true; 1910 /* guard against interrupt when we're going down */ 1911 if (!test_bit(__E1000_DOWN, &adapter->state)) 1912 mod_timer(&adapter->watchdog_timer, jiffies + 1); 1913 } 1914 1915 if (!test_bit(__E1000_DOWN, &adapter->state)) 1916 ew32(IMS, E1000_IMS_OTHER | IMS_OTHER_MASK); 1917 1918 return IRQ_HANDLED; 1919 } 1920 1921 static irqreturn_t e1000_intr_msix_tx(int __always_unused irq, void *data) 1922 { 1923 struct net_device *netdev = data; 1924 struct e1000_adapter *adapter = netdev_priv(netdev); 1925 struct e1000_hw *hw = &adapter->hw; 1926 struct e1000_ring *tx_ring = adapter->tx_ring; 1927 1928 adapter->total_tx_bytes = 0; 1929 adapter->total_tx_packets = 0; 1930 1931 if (!e1000_clean_tx_irq(tx_ring)) 1932 /* Ring was not completely cleaned, so fire another interrupt */ 1933 ew32(ICS, tx_ring->ims_val); 1934 1935 if (!test_bit(__E1000_DOWN, &adapter->state)) 1936 ew32(IMS, adapter->tx_ring->ims_val); 1937 1938 return IRQ_HANDLED; 1939 } 1940 1941 static irqreturn_t e1000_intr_msix_rx(int __always_unused irq, void *data) 1942 { 1943 struct net_device *netdev = data; 1944 struct e1000_adapter *adapter = netdev_priv(netdev); 1945 struct e1000_ring *rx_ring = adapter->rx_ring; 1946 1947 /* Write the ITR value calculated at the end of the 1948 * previous interrupt. 1949 */ 1950 if (rx_ring->set_itr) { 1951 u32 itr = rx_ring->itr_val ? 1952 1000000000 / (rx_ring->itr_val * 256) : 0; 1953 1954 writel(itr, rx_ring->itr_register); 1955 rx_ring->set_itr = 0; 1956 } 1957 1958 if (napi_schedule_prep(&adapter->napi)) { 1959 adapter->total_rx_bytes = 0; 1960 adapter->total_rx_packets = 0; 1961 __napi_schedule(&adapter->napi); 1962 } 1963 return IRQ_HANDLED; 1964 } 1965 1966 /** 1967 * e1000_configure_msix - Configure MSI-X hardware 1968 * @adapter: board private structure 1969 * 1970 * e1000_configure_msix sets up the hardware to properly 1971 * generate MSI-X interrupts. 1972 **/ 1973 static void e1000_configure_msix(struct e1000_adapter *adapter) 1974 { 1975 struct e1000_hw *hw = &adapter->hw; 1976 struct e1000_ring *rx_ring = adapter->rx_ring; 1977 struct e1000_ring *tx_ring = adapter->tx_ring; 1978 int vector = 0; 1979 u32 ctrl_ext, ivar = 0; 1980 1981 adapter->eiac_mask = 0; 1982 1983 /* Workaround issue with spurious interrupts on 82574 in MSI-X mode */ 1984 if (hw->mac.type == e1000_82574) { 1985 u32 rfctl = er32(RFCTL); 1986 1987 rfctl |= E1000_RFCTL_ACK_DIS; 1988 ew32(RFCTL, rfctl); 1989 } 1990 1991 /* Configure Rx vector */ 1992 rx_ring->ims_val = E1000_IMS_RXQ0; 1993 adapter->eiac_mask |= rx_ring->ims_val; 1994 if (rx_ring->itr_val) 1995 writel(1000000000 / (rx_ring->itr_val * 256), 1996 rx_ring->itr_register); 1997 else 1998 writel(1, rx_ring->itr_register); 1999 ivar = E1000_IVAR_INT_ALLOC_VALID | vector; 2000 2001 /* Configure Tx vector */ 2002 tx_ring->ims_val = E1000_IMS_TXQ0; 2003 vector++; 2004 if (tx_ring->itr_val) 2005 writel(1000000000 / (tx_ring->itr_val * 256), 2006 tx_ring->itr_register); 2007 else 2008 writel(1, tx_ring->itr_register); 2009 adapter->eiac_mask |= tx_ring->ims_val; 2010 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 8); 2011 2012 /* set vector for Other Causes, e.g. link changes */ 2013 vector++; 2014 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 16); 2015 if (rx_ring->itr_val) 2016 writel(1000000000 / (rx_ring->itr_val * 256), 2017 hw->hw_addr + E1000_EITR_82574(vector)); 2018 else 2019 writel(1, hw->hw_addr + E1000_EITR_82574(vector)); 2020 2021 /* Cause Tx interrupts on every write back */ 2022 ivar |= BIT(31); 2023 2024 ew32(IVAR, ivar); 2025 2026 /* enable MSI-X PBA support */ 2027 ctrl_ext = er32(CTRL_EXT) & ~E1000_CTRL_EXT_IAME; 2028 ctrl_ext |= E1000_CTRL_EXT_PBA_CLR | E1000_CTRL_EXT_EIAME; 2029 ew32(CTRL_EXT, ctrl_ext); 2030 e1e_flush(); 2031 } 2032 2033 void e1000e_reset_interrupt_capability(struct e1000_adapter *adapter) 2034 { 2035 if (adapter->msix_entries) { 2036 pci_disable_msix(adapter->pdev); 2037 kfree(adapter->msix_entries); 2038 adapter->msix_entries = NULL; 2039 } else if (adapter->flags & FLAG_MSI_ENABLED) { 2040 pci_disable_msi(adapter->pdev); 2041 adapter->flags &= ~FLAG_MSI_ENABLED; 2042 } 2043 } 2044 2045 /** 2046 * e1000e_set_interrupt_capability - set MSI or MSI-X if supported 2047 * @adapter: board private structure 2048 * 2049 * Attempt to configure interrupts using the best available 2050 * capabilities of the hardware and kernel. 2051 **/ 2052 void e1000e_set_interrupt_capability(struct e1000_adapter *adapter) 2053 { 2054 int err; 2055 int i; 2056 2057 switch (adapter->int_mode) { 2058 case E1000E_INT_MODE_MSIX: 2059 if (adapter->flags & FLAG_HAS_MSIX) { 2060 adapter->num_vectors = 3; /* RxQ0, TxQ0 and other */ 2061 adapter->msix_entries = kcalloc(adapter->num_vectors, 2062 sizeof(struct 2063 msix_entry), 2064 GFP_KERNEL); 2065 if (adapter->msix_entries) { 2066 struct e1000_adapter *a = adapter; 2067 2068 for (i = 0; i < adapter->num_vectors; i++) 2069 adapter->msix_entries[i].entry = i; 2070 2071 err = pci_enable_msix_range(a->pdev, 2072 a->msix_entries, 2073 a->num_vectors, 2074 a->num_vectors); 2075 if (err > 0) 2076 return; 2077 } 2078 /* MSI-X failed, so fall through and try MSI */ 2079 e_err("Failed to initialize MSI-X interrupts. Falling back to MSI interrupts.\n"); 2080 e1000e_reset_interrupt_capability(adapter); 2081 } 2082 adapter->int_mode = E1000E_INT_MODE_MSI; 2083 fallthrough; 2084 case E1000E_INT_MODE_MSI: 2085 if (!pci_enable_msi(adapter->pdev)) { 2086 adapter->flags |= FLAG_MSI_ENABLED; 2087 } else { 2088 adapter->int_mode = E1000E_INT_MODE_LEGACY; 2089 e_err("Failed to initialize MSI interrupts. Falling back to legacy interrupts.\n"); 2090 } 2091 fallthrough; 2092 case E1000E_INT_MODE_LEGACY: 2093 /* Don't do anything; this is the system default */ 2094 break; 2095 } 2096 2097 /* store the number of vectors being used */ 2098 adapter->num_vectors = 1; 2099 } 2100 2101 /** 2102 * e1000_request_msix - Initialize MSI-X interrupts 2103 * @adapter: board private structure 2104 * 2105 * e1000_request_msix allocates MSI-X vectors and requests interrupts from the 2106 * kernel. 2107 **/ 2108 static int e1000_request_msix(struct e1000_adapter *adapter) 2109 { 2110 struct net_device *netdev = adapter->netdev; 2111 int err = 0, vector = 0; 2112 2113 if (strlen(netdev->name) < (IFNAMSIZ - 5)) 2114 snprintf(adapter->rx_ring->name, 2115 sizeof(adapter->rx_ring->name) - 1, 2116 "%.14s-rx-0", netdev->name); 2117 else 2118 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ); 2119 err = request_irq(adapter->msix_entries[vector].vector, 2120 e1000_intr_msix_rx, 0, adapter->rx_ring->name, 2121 netdev); 2122 if (err) 2123 return err; 2124 adapter->rx_ring->itr_register = adapter->hw.hw_addr + 2125 E1000_EITR_82574(vector); 2126 adapter->rx_ring->itr_val = adapter->itr; 2127 vector++; 2128 2129 if (strlen(netdev->name) < (IFNAMSIZ - 5)) 2130 snprintf(adapter->tx_ring->name, 2131 sizeof(adapter->tx_ring->name) - 1, 2132 "%.14s-tx-0", netdev->name); 2133 else 2134 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ); 2135 err = request_irq(adapter->msix_entries[vector].vector, 2136 e1000_intr_msix_tx, 0, adapter->tx_ring->name, 2137 netdev); 2138 if (err) 2139 return err; 2140 adapter->tx_ring->itr_register = adapter->hw.hw_addr + 2141 E1000_EITR_82574(vector); 2142 adapter->tx_ring->itr_val = adapter->itr; 2143 vector++; 2144 2145 err = request_irq(adapter->msix_entries[vector].vector, 2146 e1000_msix_other, 0, netdev->name, netdev); 2147 if (err) 2148 return err; 2149 2150 e1000_configure_msix(adapter); 2151 2152 return 0; 2153 } 2154 2155 /** 2156 * e1000_request_irq - initialize interrupts 2157 * @adapter: board private structure 2158 * 2159 * Attempts to configure interrupts using the best available 2160 * capabilities of the hardware and kernel. 2161 **/ 2162 static int e1000_request_irq(struct e1000_adapter *adapter) 2163 { 2164 struct net_device *netdev = adapter->netdev; 2165 int err; 2166 2167 if (adapter->msix_entries) { 2168 err = e1000_request_msix(adapter); 2169 if (!err) 2170 return err; 2171 /* fall back to MSI */ 2172 e1000e_reset_interrupt_capability(adapter); 2173 adapter->int_mode = E1000E_INT_MODE_MSI; 2174 e1000e_set_interrupt_capability(adapter); 2175 } 2176 if (adapter->flags & FLAG_MSI_ENABLED) { 2177 err = request_irq(adapter->pdev->irq, e1000_intr_msi, 0, 2178 netdev->name, netdev); 2179 if (!err) 2180 return err; 2181 2182 /* fall back to legacy interrupt */ 2183 e1000e_reset_interrupt_capability(adapter); 2184 adapter->int_mode = E1000E_INT_MODE_LEGACY; 2185 } 2186 2187 err = request_irq(adapter->pdev->irq, e1000_intr, IRQF_SHARED, 2188 netdev->name, netdev); 2189 if (err) 2190 e_err("Unable to allocate interrupt, Error: %d\n", err); 2191 2192 return err; 2193 } 2194 2195 static void e1000_free_irq(struct e1000_adapter *adapter) 2196 { 2197 struct net_device *netdev = adapter->netdev; 2198 2199 if (adapter->msix_entries) { 2200 int vector = 0; 2201 2202 free_irq(adapter->msix_entries[vector].vector, netdev); 2203 vector++; 2204 2205 free_irq(adapter->msix_entries[vector].vector, netdev); 2206 vector++; 2207 2208 /* Other Causes interrupt vector */ 2209 free_irq(adapter->msix_entries[vector].vector, netdev); 2210 return; 2211 } 2212 2213 free_irq(adapter->pdev->irq, netdev); 2214 } 2215 2216 /** 2217 * e1000_irq_disable - Mask off interrupt generation on the NIC 2218 * @adapter: board private structure 2219 **/ 2220 static void e1000_irq_disable(struct e1000_adapter *adapter) 2221 { 2222 struct e1000_hw *hw = &adapter->hw; 2223 2224 ew32(IMC, ~0); 2225 if (adapter->msix_entries) 2226 ew32(EIAC_82574, 0); 2227 e1e_flush(); 2228 2229 if (adapter->msix_entries) { 2230 int i; 2231 2232 for (i = 0; i < adapter->num_vectors; i++) 2233 synchronize_irq(adapter->msix_entries[i].vector); 2234 } else { 2235 synchronize_irq(adapter->pdev->irq); 2236 } 2237 } 2238 2239 /** 2240 * e1000_irq_enable - Enable default interrupt generation settings 2241 * @adapter: board private structure 2242 **/ 2243 static void e1000_irq_enable(struct e1000_adapter *adapter) 2244 { 2245 struct e1000_hw *hw = &adapter->hw; 2246 2247 if (adapter->msix_entries) { 2248 ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574); 2249 ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER | 2250 IMS_OTHER_MASK); 2251 } else if (hw->mac.type >= e1000_pch_lpt) { 2252 ew32(IMS, IMS_ENABLE_MASK | E1000_IMS_ECCER); 2253 } else { 2254 ew32(IMS, IMS_ENABLE_MASK); 2255 } 2256 e1e_flush(); 2257 } 2258 2259 /** 2260 * e1000e_get_hw_control - get control of the h/w from f/w 2261 * @adapter: address of board private structure 2262 * 2263 * e1000e_get_hw_control sets {CTRL_EXT|SWSM}:DRV_LOAD bit. 2264 * For ASF and Pass Through versions of f/w this means that 2265 * the driver is loaded. For AMT version (only with 82573) 2266 * of the f/w this means that the network i/f is open. 2267 **/ 2268 void e1000e_get_hw_control(struct e1000_adapter *adapter) 2269 { 2270 struct e1000_hw *hw = &adapter->hw; 2271 u32 ctrl_ext; 2272 u32 swsm; 2273 2274 /* Let firmware know the driver has taken over */ 2275 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) { 2276 swsm = er32(SWSM); 2277 ew32(SWSM, swsm | E1000_SWSM_DRV_LOAD); 2278 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) { 2279 ctrl_ext = er32(CTRL_EXT); 2280 ew32(CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2281 } 2282 } 2283 2284 /** 2285 * e1000e_release_hw_control - release control of the h/w to f/w 2286 * @adapter: address of board private structure 2287 * 2288 * e1000e_release_hw_control resets {CTRL_EXT|SWSM}:DRV_LOAD bit. 2289 * For ASF and Pass Through versions of f/w this means that the 2290 * driver is no longer loaded. For AMT version (only with 82573) i 2291 * of the f/w this means that the network i/f is closed. 2292 * 2293 **/ 2294 void e1000e_release_hw_control(struct e1000_adapter *adapter) 2295 { 2296 struct e1000_hw *hw = &adapter->hw; 2297 u32 ctrl_ext; 2298 u32 swsm; 2299 2300 /* Let firmware taken over control of h/w */ 2301 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) { 2302 swsm = er32(SWSM); 2303 ew32(SWSM, swsm & ~E1000_SWSM_DRV_LOAD); 2304 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) { 2305 ctrl_ext = er32(CTRL_EXT); 2306 ew32(CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2307 } 2308 } 2309 2310 /** 2311 * e1000_alloc_ring_dma - allocate memory for a ring structure 2312 * @adapter: board private structure 2313 * @ring: ring struct for which to allocate dma 2314 **/ 2315 static int e1000_alloc_ring_dma(struct e1000_adapter *adapter, 2316 struct e1000_ring *ring) 2317 { 2318 struct pci_dev *pdev = adapter->pdev; 2319 2320 ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma, 2321 GFP_KERNEL); 2322 if (!ring->desc) 2323 return -ENOMEM; 2324 2325 return 0; 2326 } 2327 2328 /** 2329 * e1000e_setup_tx_resources - allocate Tx resources (Descriptors) 2330 * @tx_ring: Tx descriptor ring 2331 * 2332 * Return 0 on success, negative on failure 2333 **/ 2334 int e1000e_setup_tx_resources(struct e1000_ring *tx_ring) 2335 { 2336 struct e1000_adapter *adapter = tx_ring->adapter; 2337 int err = -ENOMEM, size; 2338 2339 size = sizeof(struct e1000_buffer) * tx_ring->count; 2340 tx_ring->buffer_info = vzalloc(size); 2341 if (!tx_ring->buffer_info) 2342 goto err; 2343 2344 /* round up to nearest 4K */ 2345 tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc); 2346 tx_ring->size = ALIGN(tx_ring->size, 4096); 2347 2348 err = e1000_alloc_ring_dma(adapter, tx_ring); 2349 if (err) 2350 goto err; 2351 2352 tx_ring->next_to_use = 0; 2353 tx_ring->next_to_clean = 0; 2354 2355 return 0; 2356 err: 2357 vfree(tx_ring->buffer_info); 2358 e_err("Unable to allocate memory for the transmit descriptor ring\n"); 2359 return err; 2360 } 2361 2362 /** 2363 * e1000e_setup_rx_resources - allocate Rx resources (Descriptors) 2364 * @rx_ring: Rx descriptor ring 2365 * 2366 * Returns 0 on success, negative on failure 2367 **/ 2368 int e1000e_setup_rx_resources(struct e1000_ring *rx_ring) 2369 { 2370 struct e1000_adapter *adapter = rx_ring->adapter; 2371 struct e1000_buffer *buffer_info; 2372 int i, size, desc_len, err = -ENOMEM; 2373 2374 size = sizeof(struct e1000_buffer) * rx_ring->count; 2375 rx_ring->buffer_info = vzalloc(size); 2376 if (!rx_ring->buffer_info) 2377 goto err; 2378 2379 for (i = 0; i < rx_ring->count; i++) { 2380 buffer_info = &rx_ring->buffer_info[i]; 2381 buffer_info->ps_pages = kcalloc(PS_PAGE_BUFFERS, 2382 sizeof(struct e1000_ps_page), 2383 GFP_KERNEL); 2384 if (!buffer_info->ps_pages) 2385 goto err_pages; 2386 } 2387 2388 desc_len = sizeof(union e1000_rx_desc_packet_split); 2389 2390 /* Round up to nearest 4K */ 2391 rx_ring->size = rx_ring->count * desc_len; 2392 rx_ring->size = ALIGN(rx_ring->size, 4096); 2393 2394 err = e1000_alloc_ring_dma(adapter, rx_ring); 2395 if (err) 2396 goto err_pages; 2397 2398 rx_ring->next_to_clean = 0; 2399 rx_ring->next_to_use = 0; 2400 rx_ring->rx_skb_top = NULL; 2401 2402 return 0; 2403 2404 err_pages: 2405 for (i = 0; i < rx_ring->count; i++) { 2406 buffer_info = &rx_ring->buffer_info[i]; 2407 kfree(buffer_info->ps_pages); 2408 } 2409 err: 2410 vfree(rx_ring->buffer_info); 2411 e_err("Unable to allocate memory for the receive descriptor ring\n"); 2412 return err; 2413 } 2414 2415 /** 2416 * e1000_clean_tx_ring - Free Tx Buffers 2417 * @tx_ring: Tx descriptor ring 2418 **/ 2419 static void e1000_clean_tx_ring(struct e1000_ring *tx_ring) 2420 { 2421 struct e1000_adapter *adapter = tx_ring->adapter; 2422 struct e1000_buffer *buffer_info; 2423 unsigned long size; 2424 unsigned int i; 2425 2426 for (i = 0; i < tx_ring->count; i++) { 2427 buffer_info = &tx_ring->buffer_info[i]; 2428 e1000_put_txbuf(tx_ring, buffer_info, false); 2429 } 2430 2431 netdev_reset_queue(adapter->netdev); 2432 size = sizeof(struct e1000_buffer) * tx_ring->count; 2433 memset(tx_ring->buffer_info, 0, size); 2434 2435 memset(tx_ring->desc, 0, tx_ring->size); 2436 2437 tx_ring->next_to_use = 0; 2438 tx_ring->next_to_clean = 0; 2439 } 2440 2441 /** 2442 * e1000e_free_tx_resources - Free Tx Resources per Queue 2443 * @tx_ring: Tx descriptor ring 2444 * 2445 * Free all transmit software resources 2446 **/ 2447 void e1000e_free_tx_resources(struct e1000_ring *tx_ring) 2448 { 2449 struct e1000_adapter *adapter = tx_ring->adapter; 2450 struct pci_dev *pdev = adapter->pdev; 2451 2452 e1000_clean_tx_ring(tx_ring); 2453 2454 vfree(tx_ring->buffer_info); 2455 tx_ring->buffer_info = NULL; 2456 2457 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc, 2458 tx_ring->dma); 2459 tx_ring->desc = NULL; 2460 } 2461 2462 /** 2463 * e1000e_free_rx_resources - Free Rx Resources 2464 * @rx_ring: Rx descriptor ring 2465 * 2466 * Free all receive software resources 2467 **/ 2468 void e1000e_free_rx_resources(struct e1000_ring *rx_ring) 2469 { 2470 struct e1000_adapter *adapter = rx_ring->adapter; 2471 struct pci_dev *pdev = adapter->pdev; 2472 int i; 2473 2474 e1000_clean_rx_ring(rx_ring); 2475 2476 for (i = 0; i < rx_ring->count; i++) 2477 kfree(rx_ring->buffer_info[i].ps_pages); 2478 2479 vfree(rx_ring->buffer_info); 2480 rx_ring->buffer_info = NULL; 2481 2482 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, 2483 rx_ring->dma); 2484 rx_ring->desc = NULL; 2485 } 2486 2487 /** 2488 * e1000_update_itr - update the dynamic ITR value based on statistics 2489 * @itr_setting: current adapter->itr 2490 * @packets: the number of packets during this measurement interval 2491 * @bytes: the number of bytes during this measurement interval 2492 * 2493 * Stores a new ITR value based on packets and byte 2494 * counts during the last interrupt. The advantage of per interrupt 2495 * computation is faster updates and more accurate ITR for the current 2496 * traffic pattern. Constants in this function were computed 2497 * based on theoretical maximum wire speed and thresholds were set based 2498 * on testing data as well as attempting to minimize response time 2499 * while increasing bulk throughput. This functionality is controlled 2500 * by the InterruptThrottleRate module parameter. 2501 **/ 2502 static unsigned int e1000_update_itr(u16 itr_setting, int packets, int bytes) 2503 { 2504 unsigned int retval = itr_setting; 2505 2506 if (packets == 0) 2507 return itr_setting; 2508 2509 switch (itr_setting) { 2510 case lowest_latency: 2511 /* handle TSO and jumbo frames */ 2512 if (bytes / packets > 8000) 2513 retval = bulk_latency; 2514 else if ((packets < 5) && (bytes > 512)) 2515 retval = low_latency; 2516 break; 2517 case low_latency: /* 50 usec aka 20000 ints/s */ 2518 if (bytes > 10000) { 2519 /* this if handles the TSO accounting */ 2520 if (bytes / packets > 8000) 2521 retval = bulk_latency; 2522 else if ((packets < 10) || ((bytes / packets) > 1200)) 2523 retval = bulk_latency; 2524 else if ((packets > 35)) 2525 retval = lowest_latency; 2526 } else if (bytes / packets > 2000) { 2527 retval = bulk_latency; 2528 } else if (packets <= 2 && bytes < 512) { 2529 retval = lowest_latency; 2530 } 2531 break; 2532 case bulk_latency: /* 250 usec aka 4000 ints/s */ 2533 if (bytes > 25000) { 2534 if (packets > 35) 2535 retval = low_latency; 2536 } else if (bytes < 6000) { 2537 retval = low_latency; 2538 } 2539 break; 2540 } 2541 2542 return retval; 2543 } 2544 2545 static void e1000_set_itr(struct e1000_adapter *adapter) 2546 { 2547 u16 current_itr; 2548 u32 new_itr = adapter->itr; 2549 2550 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 2551 if (adapter->link_speed != SPEED_1000) { 2552 current_itr = 0; 2553 new_itr = 4000; 2554 goto set_itr_now; 2555 } 2556 2557 if (adapter->flags2 & FLAG2_DISABLE_AIM) { 2558 new_itr = 0; 2559 goto set_itr_now; 2560 } 2561 2562 adapter->tx_itr = e1000_update_itr(adapter->tx_itr, 2563 adapter->total_tx_packets, 2564 adapter->total_tx_bytes); 2565 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 2566 if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency) 2567 adapter->tx_itr = low_latency; 2568 2569 adapter->rx_itr = e1000_update_itr(adapter->rx_itr, 2570 adapter->total_rx_packets, 2571 adapter->total_rx_bytes); 2572 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 2573 if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency) 2574 adapter->rx_itr = low_latency; 2575 2576 current_itr = max(adapter->rx_itr, adapter->tx_itr); 2577 2578 /* counts and packets in update_itr are dependent on these numbers */ 2579 switch (current_itr) { 2580 case lowest_latency: 2581 new_itr = 70000; 2582 break; 2583 case low_latency: 2584 new_itr = 20000; /* aka hwitr = ~200 */ 2585 break; 2586 case bulk_latency: 2587 new_itr = 4000; 2588 break; 2589 default: 2590 break; 2591 } 2592 2593 set_itr_now: 2594 if (new_itr != adapter->itr) { 2595 /* this attempts to bias the interrupt rate towards Bulk 2596 * by adding intermediate steps when interrupt rate is 2597 * increasing 2598 */ 2599 new_itr = new_itr > adapter->itr ? 2600 min(adapter->itr + (new_itr >> 2), new_itr) : new_itr; 2601 adapter->itr = new_itr; 2602 adapter->rx_ring->itr_val = new_itr; 2603 if (adapter->msix_entries) 2604 adapter->rx_ring->set_itr = 1; 2605 else 2606 e1000e_write_itr(adapter, new_itr); 2607 } 2608 } 2609 2610 /** 2611 * e1000e_write_itr - write the ITR value to the appropriate registers 2612 * @adapter: address of board private structure 2613 * @itr: new ITR value to program 2614 * 2615 * e1000e_write_itr determines if the adapter is in MSI-X mode 2616 * and, if so, writes the EITR registers with the ITR value. 2617 * Otherwise, it writes the ITR value into the ITR register. 2618 **/ 2619 void e1000e_write_itr(struct e1000_adapter *adapter, u32 itr) 2620 { 2621 struct e1000_hw *hw = &adapter->hw; 2622 u32 new_itr = itr ? 1000000000 / (itr * 256) : 0; 2623 2624 if (adapter->msix_entries) { 2625 int vector; 2626 2627 for (vector = 0; vector < adapter->num_vectors; vector++) 2628 writel(new_itr, hw->hw_addr + E1000_EITR_82574(vector)); 2629 } else { 2630 ew32(ITR, new_itr); 2631 } 2632 } 2633 2634 /** 2635 * e1000_alloc_queues - Allocate memory for all rings 2636 * @adapter: board private structure to initialize 2637 **/ 2638 static int e1000_alloc_queues(struct e1000_adapter *adapter) 2639 { 2640 int size = sizeof(struct e1000_ring); 2641 2642 adapter->tx_ring = kzalloc(size, GFP_KERNEL); 2643 if (!adapter->tx_ring) 2644 goto err; 2645 adapter->tx_ring->count = adapter->tx_ring_count; 2646 adapter->tx_ring->adapter = adapter; 2647 2648 adapter->rx_ring = kzalloc(size, GFP_KERNEL); 2649 if (!adapter->rx_ring) 2650 goto err; 2651 adapter->rx_ring->count = adapter->rx_ring_count; 2652 adapter->rx_ring->adapter = adapter; 2653 2654 return 0; 2655 err: 2656 e_err("Unable to allocate memory for queues\n"); 2657 kfree(adapter->rx_ring); 2658 kfree(adapter->tx_ring); 2659 return -ENOMEM; 2660 } 2661 2662 /** 2663 * e1000e_poll - NAPI Rx polling callback 2664 * @napi: struct associated with this polling callback 2665 * @budget: number of packets driver is allowed to process this poll 2666 **/ 2667 static int e1000e_poll(struct napi_struct *napi, int budget) 2668 { 2669 struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter, 2670 napi); 2671 struct e1000_hw *hw = &adapter->hw; 2672 struct net_device *poll_dev = adapter->netdev; 2673 int tx_cleaned = 1, work_done = 0; 2674 2675 adapter = netdev_priv(poll_dev); 2676 2677 if (!adapter->msix_entries || 2678 (adapter->rx_ring->ims_val & adapter->tx_ring->ims_val)) 2679 tx_cleaned = e1000_clean_tx_irq(adapter->tx_ring); 2680 2681 adapter->clean_rx(adapter->rx_ring, &work_done, budget); 2682 2683 if (!tx_cleaned || work_done == budget) 2684 return budget; 2685 2686 /* Exit the polling mode, but don't re-enable interrupts if stack might 2687 * poll us due to busy-polling 2688 */ 2689 if (likely(napi_complete_done(napi, work_done))) { 2690 if (adapter->itr_setting & 3) 2691 e1000_set_itr(adapter); 2692 if (!test_bit(__E1000_DOWN, &adapter->state)) { 2693 if (adapter->msix_entries) 2694 ew32(IMS, adapter->rx_ring->ims_val); 2695 else 2696 e1000_irq_enable(adapter); 2697 } 2698 } 2699 2700 return work_done; 2701 } 2702 2703 static int e1000_vlan_rx_add_vid(struct net_device *netdev, 2704 __always_unused __be16 proto, u16 vid) 2705 { 2706 struct e1000_adapter *adapter = netdev_priv(netdev); 2707 struct e1000_hw *hw = &adapter->hw; 2708 u32 vfta, index; 2709 2710 /* don't update vlan cookie if already programmed */ 2711 if ((adapter->hw.mng_cookie.status & 2712 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) && 2713 (vid == adapter->mng_vlan_id)) 2714 return 0; 2715 2716 /* add VID to filter table */ 2717 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2718 index = (vid >> 5) & 0x7F; 2719 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index); 2720 vfta |= BIT((vid & 0x1F)); 2721 hw->mac.ops.write_vfta(hw, index, vfta); 2722 } 2723 2724 set_bit(vid, adapter->active_vlans); 2725 2726 return 0; 2727 } 2728 2729 static int e1000_vlan_rx_kill_vid(struct net_device *netdev, 2730 __always_unused __be16 proto, u16 vid) 2731 { 2732 struct e1000_adapter *adapter = netdev_priv(netdev); 2733 struct e1000_hw *hw = &adapter->hw; 2734 u32 vfta, index; 2735 2736 if ((adapter->hw.mng_cookie.status & 2737 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) && 2738 (vid == adapter->mng_vlan_id)) { 2739 /* release control to f/w */ 2740 e1000e_release_hw_control(adapter); 2741 return 0; 2742 } 2743 2744 /* remove VID from filter table */ 2745 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2746 index = (vid >> 5) & 0x7F; 2747 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index); 2748 vfta &= ~BIT((vid & 0x1F)); 2749 hw->mac.ops.write_vfta(hw, index, vfta); 2750 } 2751 2752 clear_bit(vid, adapter->active_vlans); 2753 2754 return 0; 2755 } 2756 2757 /** 2758 * e1000e_vlan_filter_disable - helper to disable hw VLAN filtering 2759 * @adapter: board private structure to initialize 2760 **/ 2761 static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter) 2762 { 2763 struct net_device *netdev = adapter->netdev; 2764 struct e1000_hw *hw = &adapter->hw; 2765 u32 rctl; 2766 2767 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2768 /* disable VLAN receive filtering */ 2769 rctl = er32(RCTL); 2770 rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN); 2771 ew32(RCTL, rctl); 2772 2773 if (adapter->mng_vlan_id != (u16)E1000_MNG_VLAN_NONE) { 2774 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), 2775 adapter->mng_vlan_id); 2776 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE; 2777 } 2778 } 2779 } 2780 2781 /** 2782 * e1000e_vlan_filter_enable - helper to enable HW VLAN filtering 2783 * @adapter: board private structure to initialize 2784 **/ 2785 static void e1000e_vlan_filter_enable(struct e1000_adapter *adapter) 2786 { 2787 struct e1000_hw *hw = &adapter->hw; 2788 u32 rctl; 2789 2790 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2791 /* enable VLAN receive filtering */ 2792 rctl = er32(RCTL); 2793 rctl |= E1000_RCTL_VFE; 2794 rctl &= ~E1000_RCTL_CFIEN; 2795 ew32(RCTL, rctl); 2796 } 2797 } 2798 2799 /** 2800 * e1000e_vlan_strip_disable - helper to disable HW VLAN stripping 2801 * @adapter: board private structure to initialize 2802 **/ 2803 static void e1000e_vlan_strip_disable(struct e1000_adapter *adapter) 2804 { 2805 struct e1000_hw *hw = &adapter->hw; 2806 u32 ctrl; 2807 2808 /* disable VLAN tag insert/strip */ 2809 ctrl = er32(CTRL); 2810 ctrl &= ~E1000_CTRL_VME; 2811 ew32(CTRL, ctrl); 2812 } 2813 2814 /** 2815 * e1000e_vlan_strip_enable - helper to enable HW VLAN stripping 2816 * @adapter: board private structure to initialize 2817 **/ 2818 static void e1000e_vlan_strip_enable(struct e1000_adapter *adapter) 2819 { 2820 struct e1000_hw *hw = &adapter->hw; 2821 u32 ctrl; 2822 2823 /* enable VLAN tag insert/strip */ 2824 ctrl = er32(CTRL); 2825 ctrl |= E1000_CTRL_VME; 2826 ew32(CTRL, ctrl); 2827 } 2828 2829 static void e1000_update_mng_vlan(struct e1000_adapter *adapter) 2830 { 2831 struct net_device *netdev = adapter->netdev; 2832 u16 vid = adapter->hw.mng_cookie.vlan_id; 2833 u16 old_vid = adapter->mng_vlan_id; 2834 2835 if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) { 2836 e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid); 2837 adapter->mng_vlan_id = vid; 2838 } 2839 2840 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && (vid != old_vid)) 2841 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), old_vid); 2842 } 2843 2844 static void e1000_restore_vlan(struct e1000_adapter *adapter) 2845 { 2846 u16 vid; 2847 2848 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0); 2849 2850 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2851 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid); 2852 } 2853 2854 static void e1000_init_manageability_pt(struct e1000_adapter *adapter) 2855 { 2856 struct e1000_hw *hw = &adapter->hw; 2857 u32 manc, manc2h, mdef, i, j; 2858 2859 if (!(adapter->flags & FLAG_MNG_PT_ENABLED)) 2860 return; 2861 2862 manc = er32(MANC); 2863 2864 /* enable receiving management packets to the host. this will probably 2865 * generate destination unreachable messages from the host OS, but 2866 * the packets will be handled on SMBUS 2867 */ 2868 manc |= E1000_MANC_EN_MNG2HOST; 2869 manc2h = er32(MANC2H); 2870 2871 switch (hw->mac.type) { 2872 default: 2873 manc2h |= (E1000_MANC2H_PORT_623 | E1000_MANC2H_PORT_664); 2874 break; 2875 case e1000_82574: 2876 case e1000_82583: 2877 /* Check if IPMI pass-through decision filter already exists; 2878 * if so, enable it. 2879 */ 2880 for (i = 0, j = 0; i < 8; i++) { 2881 mdef = er32(MDEF(i)); 2882 2883 /* Ignore filters with anything other than IPMI ports */ 2884 if (mdef & ~(E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664)) 2885 continue; 2886 2887 /* Enable this decision filter in MANC2H */ 2888 if (mdef) 2889 manc2h |= BIT(i); 2890 2891 j |= mdef; 2892 } 2893 2894 if (j == (E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664)) 2895 break; 2896 2897 /* Create new decision filter in an empty filter */ 2898 for (i = 0, j = 0; i < 8; i++) 2899 if (er32(MDEF(i)) == 0) { 2900 ew32(MDEF(i), (E1000_MDEF_PORT_623 | 2901 E1000_MDEF_PORT_664)); 2902 manc2h |= BIT(1); 2903 j++; 2904 break; 2905 } 2906 2907 if (!j) 2908 e_warn("Unable to create IPMI pass-through filter\n"); 2909 break; 2910 } 2911 2912 ew32(MANC2H, manc2h); 2913 ew32(MANC, manc); 2914 } 2915 2916 /** 2917 * e1000_configure_tx - Configure Transmit Unit after Reset 2918 * @adapter: board private structure 2919 * 2920 * Configure the Tx unit of the MAC after a reset. 2921 **/ 2922 static void e1000_configure_tx(struct e1000_adapter *adapter) 2923 { 2924 struct e1000_hw *hw = &adapter->hw; 2925 struct e1000_ring *tx_ring = adapter->tx_ring; 2926 u64 tdba; 2927 u32 tdlen, tctl, tarc; 2928 2929 /* Setup the HW Tx Head and Tail descriptor pointers */ 2930 tdba = tx_ring->dma; 2931 tdlen = tx_ring->count * sizeof(struct e1000_tx_desc); 2932 ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32))); 2933 ew32(TDBAH(0), (tdba >> 32)); 2934 ew32(TDLEN(0), tdlen); 2935 ew32(TDH(0), 0); 2936 ew32(TDT(0), 0); 2937 tx_ring->head = adapter->hw.hw_addr + E1000_TDH(0); 2938 tx_ring->tail = adapter->hw.hw_addr + E1000_TDT(0); 2939 2940 writel(0, tx_ring->head); 2941 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 2942 e1000e_update_tdt_wa(tx_ring, 0); 2943 else 2944 writel(0, tx_ring->tail); 2945 2946 /* Set the Tx Interrupt Delay register */ 2947 ew32(TIDV, adapter->tx_int_delay); 2948 /* Tx irq moderation */ 2949 ew32(TADV, adapter->tx_abs_int_delay); 2950 2951 if (adapter->flags2 & FLAG2_DMA_BURST) { 2952 u32 txdctl = er32(TXDCTL(0)); 2953 2954 txdctl &= ~(E1000_TXDCTL_PTHRESH | E1000_TXDCTL_HTHRESH | 2955 E1000_TXDCTL_WTHRESH); 2956 /* set up some performance related parameters to encourage the 2957 * hardware to use the bus more efficiently in bursts, depends 2958 * on the tx_int_delay to be enabled, 2959 * wthresh = 1 ==> burst write is disabled to avoid Tx stalls 2960 * hthresh = 1 ==> prefetch when one or more available 2961 * pthresh = 0x1f ==> prefetch if internal cache 31 or less 2962 * BEWARE: this seems to work but should be considered first if 2963 * there are Tx hangs or other Tx related bugs 2964 */ 2965 txdctl |= E1000_TXDCTL_DMA_BURST_ENABLE; 2966 ew32(TXDCTL(0), txdctl); 2967 } 2968 /* erratum work around: set txdctl the same for both queues */ 2969 ew32(TXDCTL(1), er32(TXDCTL(0))); 2970 2971 /* Program the Transmit Control Register */ 2972 tctl = er32(TCTL); 2973 tctl &= ~E1000_TCTL_CT; 2974 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | 2975 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2976 2977 if (adapter->flags & FLAG_TARC_SPEED_MODE_BIT) { 2978 tarc = er32(TARC(0)); 2979 /* set the speed mode bit, we'll clear it if we're not at 2980 * gigabit link later 2981 */ 2982 #define SPEED_MODE_BIT BIT(21) 2983 tarc |= SPEED_MODE_BIT; 2984 ew32(TARC(0), tarc); 2985 } 2986 2987 /* errata: program both queues to unweighted RR */ 2988 if (adapter->flags & FLAG_TARC_SET_BIT_ZERO) { 2989 tarc = er32(TARC(0)); 2990 tarc |= 1; 2991 ew32(TARC(0), tarc); 2992 tarc = er32(TARC(1)); 2993 tarc |= 1; 2994 ew32(TARC(1), tarc); 2995 } 2996 2997 /* Setup Transmit Descriptor Settings for eop descriptor */ 2998 adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS; 2999 3000 /* only set IDE if we are delaying interrupts using the timers */ 3001 if (adapter->tx_int_delay) 3002 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3003 3004 /* enable Report Status bit */ 3005 adapter->txd_cmd |= E1000_TXD_CMD_RS; 3006 3007 ew32(TCTL, tctl); 3008 3009 hw->mac.ops.config_collision_dist(hw); 3010 3011 /* SPT and KBL Si errata workaround to avoid data corruption */ 3012 if (hw->mac.type == e1000_pch_spt) { 3013 u32 reg_val; 3014 3015 reg_val = er32(IOSFPC); 3016 reg_val |= E1000_RCTL_RDMTS_HEX; 3017 ew32(IOSFPC, reg_val); 3018 3019 reg_val = er32(TARC(0)); 3020 /* SPT and KBL Si errata workaround to avoid Tx hang. 3021 * Dropping the number of outstanding requests from 3022 * 3 to 2 in order to avoid a buffer overrun. 3023 */ 3024 reg_val &= ~E1000_TARC0_CB_MULTIQ_3_REQ; 3025 reg_val |= E1000_TARC0_CB_MULTIQ_2_REQ; 3026 ew32(TARC(0), reg_val); 3027 } 3028 } 3029 3030 #define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \ 3031 (((S) & (PAGE_SIZE - 1)) ? 1 : 0)) 3032 3033 /** 3034 * e1000_setup_rctl - configure the receive control registers 3035 * @adapter: Board private structure 3036 **/ 3037 static void e1000_setup_rctl(struct e1000_adapter *adapter) 3038 { 3039 struct e1000_hw *hw = &adapter->hw; 3040 u32 rctl, rfctl; 3041 u32 pages = 0; 3042 3043 /* Workaround Si errata on PCHx - configure jumbo frame flow. 3044 * If jumbo frames not set, program related MAC/PHY registers 3045 * to h/w defaults 3046 */ 3047 if (hw->mac.type >= e1000_pch2lan) { 3048 s32 ret_val; 3049 3050 if (adapter->netdev->mtu > ETH_DATA_LEN) 3051 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, true); 3052 else 3053 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, false); 3054 3055 if (ret_val) 3056 e_dbg("failed to enable|disable jumbo frame workaround mode\n"); 3057 } 3058 3059 /* Program MC offset vector base */ 3060 rctl = er32(RCTL); 3061 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 3062 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 3063 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | 3064 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 3065 3066 /* Do not Store bad packets */ 3067 rctl &= ~E1000_RCTL_SBP; 3068 3069 /* Enable Long Packet receive */ 3070 if (adapter->netdev->mtu <= ETH_DATA_LEN) 3071 rctl &= ~E1000_RCTL_LPE; 3072 else 3073 rctl |= E1000_RCTL_LPE; 3074 3075 /* Some systems expect that the CRC is included in SMBUS traffic. The 3076 * hardware strips the CRC before sending to both SMBUS (BMC) and to 3077 * host memory when this is enabled 3078 */ 3079 if (adapter->flags2 & FLAG2_CRC_STRIPPING) 3080 rctl |= E1000_RCTL_SECRC; 3081 3082 /* Workaround Si errata on 82577 PHY - configure IPG for jumbos */ 3083 if ((hw->phy.type == e1000_phy_82577) && (rctl & E1000_RCTL_LPE)) { 3084 u16 phy_data; 3085 3086 e1e_rphy(hw, PHY_REG(770, 26), &phy_data); 3087 phy_data &= 0xfff8; 3088 phy_data |= BIT(2); 3089 e1e_wphy(hw, PHY_REG(770, 26), phy_data); 3090 3091 e1e_rphy(hw, 22, &phy_data); 3092 phy_data &= 0x0fff; 3093 phy_data |= BIT(14); 3094 e1e_wphy(hw, 0x10, 0x2823); 3095 e1e_wphy(hw, 0x11, 0x0003); 3096 e1e_wphy(hw, 22, phy_data); 3097 } 3098 3099 /* Setup buffer sizes */ 3100 rctl &= ~E1000_RCTL_SZ_4096; 3101 rctl |= E1000_RCTL_BSEX; 3102 switch (adapter->rx_buffer_len) { 3103 case 2048: 3104 default: 3105 rctl |= E1000_RCTL_SZ_2048; 3106 rctl &= ~E1000_RCTL_BSEX; 3107 break; 3108 case 4096: 3109 rctl |= E1000_RCTL_SZ_4096; 3110 break; 3111 case 8192: 3112 rctl |= E1000_RCTL_SZ_8192; 3113 break; 3114 case 16384: 3115 rctl |= E1000_RCTL_SZ_16384; 3116 break; 3117 } 3118 3119 /* Enable Extended Status in all Receive Descriptors */ 3120 rfctl = er32(RFCTL); 3121 rfctl |= E1000_RFCTL_EXTEN; 3122 ew32(RFCTL, rfctl); 3123 3124 /* 82571 and greater support packet-split where the protocol 3125 * header is placed in skb->data and the packet data is 3126 * placed in pages hanging off of skb_shinfo(skb)->nr_frags. 3127 * In the case of a non-split, skb->data is linearly filled, 3128 * followed by the page buffers. Therefore, skb->data is 3129 * sized to hold the largest protocol header. 3130 * 3131 * allocations using alloc_page take too long for regular MTU 3132 * so only enable packet split for jumbo frames 3133 * 3134 * Using pages when the page size is greater than 16k wastes 3135 * a lot of memory, since we allocate 3 pages at all times 3136 * per packet. 3137 */ 3138 pages = PAGE_USE_COUNT(adapter->netdev->mtu); 3139 if ((pages <= 3) && (PAGE_SIZE <= 16384) && (rctl & E1000_RCTL_LPE)) 3140 adapter->rx_ps_pages = pages; 3141 else 3142 adapter->rx_ps_pages = 0; 3143 3144 if (adapter->rx_ps_pages) { 3145 u32 psrctl = 0; 3146 3147 /* Enable Packet split descriptors */ 3148 rctl |= E1000_RCTL_DTYP_PS; 3149 3150 psrctl |= adapter->rx_ps_bsize0 >> E1000_PSRCTL_BSIZE0_SHIFT; 3151 3152 switch (adapter->rx_ps_pages) { 3153 case 3: 3154 psrctl |= PAGE_SIZE << E1000_PSRCTL_BSIZE3_SHIFT; 3155 fallthrough; 3156 case 2: 3157 psrctl |= PAGE_SIZE << E1000_PSRCTL_BSIZE2_SHIFT; 3158 fallthrough; 3159 case 1: 3160 psrctl |= PAGE_SIZE >> E1000_PSRCTL_BSIZE1_SHIFT; 3161 break; 3162 } 3163 3164 ew32(PSRCTL, psrctl); 3165 } 3166 3167 /* This is useful for sniffing bad packets. */ 3168 if (adapter->netdev->features & NETIF_F_RXALL) { 3169 /* UPE and MPE will be handled by normal PROMISC logic 3170 * in e1000e_set_rx_mode 3171 */ 3172 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */ 3173 E1000_RCTL_BAM | /* RX All Bcast Pkts */ 3174 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 3175 3176 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */ 3177 E1000_RCTL_DPF | /* Allow filtered pause */ 3178 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */ 3179 /* Do not mess with E1000_CTRL_VME, it affects transmit as well, 3180 * and that breaks VLANs. 3181 */ 3182 } 3183 3184 ew32(RCTL, rctl); 3185 /* just started the receive unit, no need to restart */ 3186 adapter->flags &= ~FLAG_RESTART_NOW; 3187 } 3188 3189 /** 3190 * e1000_configure_rx - Configure Receive Unit after Reset 3191 * @adapter: board private structure 3192 * 3193 * Configure the Rx unit of the MAC after a reset. 3194 **/ 3195 static void e1000_configure_rx(struct e1000_adapter *adapter) 3196 { 3197 struct e1000_hw *hw = &adapter->hw; 3198 struct e1000_ring *rx_ring = adapter->rx_ring; 3199 u64 rdba; 3200 u32 rdlen, rctl, rxcsum, ctrl_ext; 3201 3202 if (adapter->rx_ps_pages) { 3203 /* this is a 32 byte descriptor */ 3204 rdlen = rx_ring->count * 3205 sizeof(union e1000_rx_desc_packet_split); 3206 adapter->clean_rx = e1000_clean_rx_irq_ps; 3207 adapter->alloc_rx_buf = e1000_alloc_rx_buffers_ps; 3208 } else if (adapter->netdev->mtu > ETH_FRAME_LEN + ETH_FCS_LEN) { 3209 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended); 3210 adapter->clean_rx = e1000_clean_jumbo_rx_irq; 3211 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers; 3212 } else { 3213 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended); 3214 adapter->clean_rx = e1000_clean_rx_irq; 3215 adapter->alloc_rx_buf = e1000_alloc_rx_buffers; 3216 } 3217 3218 /* disable receives while setting up the descriptors */ 3219 rctl = er32(RCTL); 3220 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX)) 3221 ew32(RCTL, rctl & ~E1000_RCTL_EN); 3222 e1e_flush(); 3223 usleep_range(10000, 11000); 3224 3225 if (adapter->flags2 & FLAG2_DMA_BURST) { 3226 /* set the writeback threshold (only takes effect if the RDTR 3227 * is set). set GRAN=1 and write back up to 0x4 worth, and 3228 * enable prefetching of 0x20 Rx descriptors 3229 * granularity = 01 3230 * wthresh = 04, 3231 * hthresh = 04, 3232 * pthresh = 0x20 3233 */ 3234 ew32(RXDCTL(0), E1000_RXDCTL_DMA_BURST_ENABLE); 3235 ew32(RXDCTL(1), E1000_RXDCTL_DMA_BURST_ENABLE); 3236 } 3237 3238 /* set the Receive Delay Timer Register */ 3239 ew32(RDTR, adapter->rx_int_delay); 3240 3241 /* irq moderation */ 3242 ew32(RADV, adapter->rx_abs_int_delay); 3243 if ((adapter->itr_setting != 0) && (adapter->itr != 0)) 3244 e1000e_write_itr(adapter, adapter->itr); 3245 3246 ctrl_ext = er32(CTRL_EXT); 3247 /* Auto-Mask interrupts upon ICR access */ 3248 ctrl_ext |= E1000_CTRL_EXT_IAME; 3249 ew32(IAM, 0xffffffff); 3250 ew32(CTRL_EXT, ctrl_ext); 3251 e1e_flush(); 3252 3253 /* Setup the HW Rx Head and Tail Descriptor Pointers and 3254 * the Base and Length of the Rx Descriptor Ring 3255 */ 3256 rdba = rx_ring->dma; 3257 ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32))); 3258 ew32(RDBAH(0), (rdba >> 32)); 3259 ew32(RDLEN(0), rdlen); 3260 ew32(RDH(0), 0); 3261 ew32(RDT(0), 0); 3262 rx_ring->head = adapter->hw.hw_addr + E1000_RDH(0); 3263 rx_ring->tail = adapter->hw.hw_addr + E1000_RDT(0); 3264 3265 writel(0, rx_ring->head); 3266 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 3267 e1000e_update_rdt_wa(rx_ring, 0); 3268 else 3269 writel(0, rx_ring->tail); 3270 3271 /* Enable Receive Checksum Offload for TCP and UDP */ 3272 rxcsum = er32(RXCSUM); 3273 if (adapter->netdev->features & NETIF_F_RXCSUM) 3274 rxcsum |= E1000_RXCSUM_TUOFL; 3275 else 3276 rxcsum &= ~E1000_RXCSUM_TUOFL; 3277 ew32(RXCSUM, rxcsum); 3278 3279 /* With jumbo frames, excessive C-state transition latencies result 3280 * in dropped transactions. 3281 */ 3282 if (adapter->netdev->mtu > ETH_DATA_LEN) { 3283 u32 lat = 3284 ((er32(PBA) & E1000_PBA_RXA_MASK) * 1024 - 3285 adapter->max_frame_size) * 8 / 1000; 3286 3287 if (adapter->flags & FLAG_IS_ICH) { 3288 u32 rxdctl = er32(RXDCTL(0)); 3289 3290 ew32(RXDCTL(0), rxdctl | 0x3 | BIT(8)); 3291 } 3292 3293 dev_info(&adapter->pdev->dev, 3294 "Some CPU C-states have been disabled in order to enable jumbo frames\n"); 3295 cpu_latency_qos_update_request(&adapter->pm_qos_req, lat); 3296 } else { 3297 cpu_latency_qos_update_request(&adapter->pm_qos_req, 3298 PM_QOS_DEFAULT_VALUE); 3299 } 3300 3301 /* Enable Receives */ 3302 ew32(RCTL, rctl); 3303 } 3304 3305 /** 3306 * e1000e_write_mc_addr_list - write multicast addresses to MTA 3307 * @netdev: network interface device structure 3308 * 3309 * Writes multicast address list to the MTA hash table. 3310 * Returns: -ENOMEM on failure 3311 * 0 on no addresses written 3312 * X on writing X addresses to MTA 3313 */ 3314 static int e1000e_write_mc_addr_list(struct net_device *netdev) 3315 { 3316 struct e1000_adapter *adapter = netdev_priv(netdev); 3317 struct e1000_hw *hw = &adapter->hw; 3318 struct netdev_hw_addr *ha; 3319 u8 *mta_list; 3320 int i; 3321 3322 if (netdev_mc_empty(netdev)) { 3323 /* nothing to program, so clear mc list */ 3324 hw->mac.ops.update_mc_addr_list(hw, NULL, 0); 3325 return 0; 3326 } 3327 3328 mta_list = kcalloc(netdev_mc_count(netdev), ETH_ALEN, GFP_ATOMIC); 3329 if (!mta_list) 3330 return -ENOMEM; 3331 3332 /* update_mc_addr_list expects a packed array of only addresses. */ 3333 i = 0; 3334 netdev_for_each_mc_addr(ha, netdev) 3335 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 3336 3337 hw->mac.ops.update_mc_addr_list(hw, mta_list, i); 3338 kfree(mta_list); 3339 3340 return netdev_mc_count(netdev); 3341 } 3342 3343 /** 3344 * e1000e_write_uc_addr_list - write unicast addresses to RAR table 3345 * @netdev: network interface device structure 3346 * 3347 * Writes unicast address list to the RAR table. 3348 * Returns: -ENOMEM on failure/insufficient address space 3349 * 0 on no addresses written 3350 * X on writing X addresses to the RAR table 3351 **/ 3352 static int e1000e_write_uc_addr_list(struct net_device *netdev) 3353 { 3354 struct e1000_adapter *adapter = netdev_priv(netdev); 3355 struct e1000_hw *hw = &adapter->hw; 3356 unsigned int rar_entries; 3357 int count = 0; 3358 3359 rar_entries = hw->mac.ops.rar_get_count(hw); 3360 3361 /* save a rar entry for our hardware address */ 3362 rar_entries--; 3363 3364 /* save a rar entry for the LAA workaround */ 3365 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) 3366 rar_entries--; 3367 3368 /* return ENOMEM indicating insufficient memory for addresses */ 3369 if (netdev_uc_count(netdev) > rar_entries) 3370 return -ENOMEM; 3371 3372 if (!netdev_uc_empty(netdev) && rar_entries) { 3373 struct netdev_hw_addr *ha; 3374 3375 /* write the addresses in reverse order to avoid write 3376 * combining 3377 */ 3378 netdev_for_each_uc_addr(ha, netdev) { 3379 int ret_val; 3380 3381 if (!rar_entries) 3382 break; 3383 ret_val = hw->mac.ops.rar_set(hw, ha->addr, rar_entries--); 3384 if (ret_val < 0) 3385 return -ENOMEM; 3386 count++; 3387 } 3388 } 3389 3390 /* zero out the remaining RAR entries not used above */ 3391 for (; rar_entries > 0; rar_entries--) { 3392 ew32(RAH(rar_entries), 0); 3393 ew32(RAL(rar_entries), 0); 3394 } 3395 e1e_flush(); 3396 3397 return count; 3398 } 3399 3400 /** 3401 * e1000e_set_rx_mode - secondary unicast, Multicast and Promiscuous mode set 3402 * @netdev: network interface device structure 3403 * 3404 * The ndo_set_rx_mode entry point is called whenever the unicast or multicast 3405 * address list or the network interface flags are updated. This routine is 3406 * responsible for configuring the hardware for proper unicast, multicast, 3407 * promiscuous mode, and all-multi behavior. 3408 **/ 3409 static void e1000e_set_rx_mode(struct net_device *netdev) 3410 { 3411 struct e1000_adapter *adapter = netdev_priv(netdev); 3412 struct e1000_hw *hw = &adapter->hw; 3413 u32 rctl; 3414 3415 if (pm_runtime_suspended(netdev->dev.parent)) 3416 return; 3417 3418 /* Check for Promiscuous and All Multicast modes */ 3419 rctl = er32(RCTL); 3420 3421 /* clear the affected bits */ 3422 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE); 3423 3424 if (netdev->flags & IFF_PROMISC) { 3425 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 3426 /* Do not hardware filter VLANs in promisc mode */ 3427 e1000e_vlan_filter_disable(adapter); 3428 } else { 3429 int count; 3430 3431 if (netdev->flags & IFF_ALLMULTI) { 3432 rctl |= E1000_RCTL_MPE; 3433 } else { 3434 /* Write addresses to the MTA, if the attempt fails 3435 * then we should just turn on promiscuous mode so 3436 * that we can at least receive multicast traffic 3437 */ 3438 count = e1000e_write_mc_addr_list(netdev); 3439 if (count < 0) 3440 rctl |= E1000_RCTL_MPE; 3441 } 3442 e1000e_vlan_filter_enable(adapter); 3443 /* Write addresses to available RAR registers, if there is not 3444 * sufficient space to store all the addresses then enable 3445 * unicast promiscuous mode 3446 */ 3447 count = e1000e_write_uc_addr_list(netdev); 3448 if (count < 0) 3449 rctl |= E1000_RCTL_UPE; 3450 } 3451 3452 ew32(RCTL, rctl); 3453 3454 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 3455 e1000e_vlan_strip_enable(adapter); 3456 else 3457 e1000e_vlan_strip_disable(adapter); 3458 } 3459 3460 static void e1000e_setup_rss_hash(struct e1000_adapter *adapter) 3461 { 3462 struct e1000_hw *hw = &adapter->hw; 3463 u32 mrqc, rxcsum; 3464 u32 rss_key[10]; 3465 int i; 3466 3467 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 3468 for (i = 0; i < 10; i++) 3469 ew32(RSSRK(i), rss_key[i]); 3470 3471 /* Direct all traffic to queue 0 */ 3472 for (i = 0; i < 32; i++) 3473 ew32(RETA(i), 0); 3474 3475 /* Disable raw packet checksumming so that RSS hash is placed in 3476 * descriptor on writeback. 3477 */ 3478 rxcsum = er32(RXCSUM); 3479 rxcsum |= E1000_RXCSUM_PCSD; 3480 3481 ew32(RXCSUM, rxcsum); 3482 3483 mrqc = (E1000_MRQC_RSS_FIELD_IPV4 | 3484 E1000_MRQC_RSS_FIELD_IPV4_TCP | 3485 E1000_MRQC_RSS_FIELD_IPV6 | 3486 E1000_MRQC_RSS_FIELD_IPV6_TCP | 3487 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 3488 3489 ew32(MRQC, mrqc); 3490 } 3491 3492 /** 3493 * e1000e_get_base_timinca - get default SYSTIM time increment attributes 3494 * @adapter: board private structure 3495 * @timinca: pointer to returned time increment attributes 3496 * 3497 * Get attributes for incrementing the System Time Register SYSTIML/H at 3498 * the default base frequency, and set the cyclecounter shift value. 3499 **/ 3500 s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca) 3501 { 3502 struct e1000_hw *hw = &adapter->hw; 3503 u32 incvalue, incperiod, shift; 3504 3505 /* Make sure clock is enabled on I217/I218/I219 before checking 3506 * the frequency 3507 */ 3508 if ((hw->mac.type >= e1000_pch_lpt) && 3509 !(er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) && 3510 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_ENABLED)) { 3511 u32 fextnvm7 = er32(FEXTNVM7); 3512 3513 if (!(fextnvm7 & BIT(0))) { 3514 ew32(FEXTNVM7, fextnvm7 | BIT(0)); 3515 e1e_flush(); 3516 } 3517 } 3518 3519 switch (hw->mac.type) { 3520 case e1000_pch2lan: 3521 /* Stable 96MHz frequency */ 3522 incperiod = INCPERIOD_96MHZ; 3523 incvalue = INCVALUE_96MHZ; 3524 shift = INCVALUE_SHIFT_96MHZ; 3525 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHZ; 3526 break; 3527 case e1000_pch_lpt: 3528 if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) { 3529 /* Stable 96MHz frequency */ 3530 incperiod = INCPERIOD_96MHZ; 3531 incvalue = INCVALUE_96MHZ; 3532 shift = INCVALUE_SHIFT_96MHZ; 3533 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHZ; 3534 } else { 3535 /* Stable 25MHz frequency */ 3536 incperiod = INCPERIOD_25MHZ; 3537 incvalue = INCVALUE_25MHZ; 3538 shift = INCVALUE_SHIFT_25MHZ; 3539 adapter->cc.shift = shift; 3540 } 3541 break; 3542 case e1000_pch_spt: 3543 /* Stable 24MHz frequency */ 3544 incperiod = INCPERIOD_24MHZ; 3545 incvalue = INCVALUE_24MHZ; 3546 shift = INCVALUE_SHIFT_24MHZ; 3547 adapter->cc.shift = shift; 3548 break; 3549 case e1000_pch_cnp: 3550 case e1000_pch_tgp: 3551 case e1000_pch_adp: 3552 case e1000_pch_mtp: 3553 if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) { 3554 /* Stable 24MHz frequency */ 3555 incperiod = INCPERIOD_24MHZ; 3556 incvalue = INCVALUE_24MHZ; 3557 shift = INCVALUE_SHIFT_24MHZ; 3558 adapter->cc.shift = shift; 3559 } else { 3560 /* Stable 38400KHz frequency */ 3561 incperiod = INCPERIOD_38400KHZ; 3562 incvalue = INCVALUE_38400KHZ; 3563 shift = INCVALUE_SHIFT_38400KHZ; 3564 adapter->cc.shift = shift; 3565 } 3566 break; 3567 case e1000_82574: 3568 case e1000_82583: 3569 /* Stable 25MHz frequency */ 3570 incperiod = INCPERIOD_25MHZ; 3571 incvalue = INCVALUE_25MHZ; 3572 shift = INCVALUE_SHIFT_25MHZ; 3573 adapter->cc.shift = shift; 3574 break; 3575 default: 3576 return -EINVAL; 3577 } 3578 3579 *timinca = ((incperiod << E1000_TIMINCA_INCPERIOD_SHIFT) | 3580 ((incvalue << shift) & E1000_TIMINCA_INCVALUE_MASK)); 3581 3582 return 0; 3583 } 3584 3585 /** 3586 * e1000e_config_hwtstamp - configure the hwtstamp registers and enable/disable 3587 * @adapter: board private structure 3588 * @config: timestamp configuration 3589 * 3590 * Outgoing time stamping can be enabled and disabled. Play nice and 3591 * disable it when requested, although it shouldn't cause any overhead 3592 * when no packet needs it. At most one packet in the queue may be 3593 * marked for time stamping, otherwise it would be impossible to tell 3594 * for sure to which packet the hardware time stamp belongs. 3595 * 3596 * Incoming time stamping has to be configured via the hardware filters. 3597 * Not all combinations are supported, in particular event type has to be 3598 * specified. Matching the kind of event packet is not supported, with the 3599 * exception of "all V2 events regardless of level 2 or 4". 3600 **/ 3601 static int e1000e_config_hwtstamp(struct e1000_adapter *adapter, 3602 struct hwtstamp_config *config) 3603 { 3604 struct e1000_hw *hw = &adapter->hw; 3605 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED; 3606 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED; 3607 u32 rxmtrl = 0; 3608 u16 rxudp = 0; 3609 bool is_l4 = false; 3610 bool is_l2 = false; 3611 u32 regval; 3612 3613 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP)) 3614 return -EINVAL; 3615 3616 /* flags reserved for future extensions - must be zero */ 3617 if (config->flags) 3618 return -EINVAL; 3619 3620 switch (config->tx_type) { 3621 case HWTSTAMP_TX_OFF: 3622 tsync_tx_ctl = 0; 3623 break; 3624 case HWTSTAMP_TX_ON: 3625 break; 3626 default: 3627 return -ERANGE; 3628 } 3629 3630 switch (config->rx_filter) { 3631 case HWTSTAMP_FILTER_NONE: 3632 tsync_rx_ctl = 0; 3633 break; 3634 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 3635 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1; 3636 rxmtrl = E1000_RXMTRL_PTP_V1_SYNC_MESSAGE; 3637 is_l4 = true; 3638 break; 3639 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 3640 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1; 3641 rxmtrl = E1000_RXMTRL_PTP_V1_DELAY_REQ_MESSAGE; 3642 is_l4 = true; 3643 break; 3644 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 3645 /* Also time stamps V2 L2 Path Delay Request/Response */ 3646 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2; 3647 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE; 3648 is_l2 = true; 3649 break; 3650 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 3651 /* Also time stamps V2 L2 Path Delay Request/Response. */ 3652 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2; 3653 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE; 3654 is_l2 = true; 3655 break; 3656 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 3657 /* Hardware cannot filter just V2 L4 Sync messages */ 3658 fallthrough; 3659 case HWTSTAMP_FILTER_PTP_V2_SYNC: 3660 /* Also time stamps V2 Path Delay Request/Response. */ 3661 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2; 3662 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE; 3663 is_l2 = true; 3664 is_l4 = true; 3665 break; 3666 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 3667 /* Hardware cannot filter just V2 L4 Delay Request messages */ 3668 fallthrough; 3669 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 3670 /* Also time stamps V2 Path Delay Request/Response. */ 3671 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2; 3672 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE; 3673 is_l2 = true; 3674 is_l4 = true; 3675 break; 3676 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 3677 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 3678 /* Hardware cannot filter just V2 L4 or L2 Event messages */ 3679 fallthrough; 3680 case HWTSTAMP_FILTER_PTP_V2_EVENT: 3681 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2; 3682 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 3683 is_l2 = true; 3684 is_l4 = true; 3685 break; 3686 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 3687 /* For V1, the hardware can only filter Sync messages or 3688 * Delay Request messages but not both so fall-through to 3689 * time stamp all packets. 3690 */ 3691 fallthrough; 3692 case HWTSTAMP_FILTER_NTP_ALL: 3693 case HWTSTAMP_FILTER_ALL: 3694 is_l2 = true; 3695 is_l4 = true; 3696 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL; 3697 config->rx_filter = HWTSTAMP_FILTER_ALL; 3698 break; 3699 default: 3700 return -ERANGE; 3701 } 3702 3703 adapter->hwtstamp_config = *config; 3704 3705 /* enable/disable Tx h/w time stamping */ 3706 regval = er32(TSYNCTXCTL); 3707 regval &= ~E1000_TSYNCTXCTL_ENABLED; 3708 regval |= tsync_tx_ctl; 3709 ew32(TSYNCTXCTL, regval); 3710 if ((er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) != 3711 (regval & E1000_TSYNCTXCTL_ENABLED)) { 3712 e_err("Timesync Tx Control register not set as expected\n"); 3713 return -EAGAIN; 3714 } 3715 3716 /* enable/disable Rx h/w time stamping */ 3717 regval = er32(TSYNCRXCTL); 3718 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK); 3719 regval |= tsync_rx_ctl; 3720 ew32(TSYNCRXCTL, regval); 3721 if ((er32(TSYNCRXCTL) & (E1000_TSYNCRXCTL_ENABLED | 3722 E1000_TSYNCRXCTL_TYPE_MASK)) != 3723 (regval & (E1000_TSYNCRXCTL_ENABLED | 3724 E1000_TSYNCRXCTL_TYPE_MASK))) { 3725 e_err("Timesync Rx Control register not set as expected\n"); 3726 return -EAGAIN; 3727 } 3728 3729 /* L2: define ethertype filter for time stamped packets */ 3730 if (is_l2) 3731 rxmtrl |= ETH_P_1588; 3732 3733 /* define which PTP packets get time stamped */ 3734 ew32(RXMTRL, rxmtrl); 3735 3736 /* Filter by destination port */ 3737 if (is_l4) { 3738 rxudp = PTP_EV_PORT; 3739 cpu_to_be16s(&rxudp); 3740 } 3741 ew32(RXUDP, rxudp); 3742 3743 e1e_flush(); 3744 3745 /* Clear TSYNCRXCTL_VALID & TSYNCTXCTL_VALID bit */ 3746 er32(RXSTMPH); 3747 er32(TXSTMPH); 3748 3749 return 0; 3750 } 3751 3752 /** 3753 * e1000_configure - configure the hardware for Rx and Tx 3754 * @adapter: private board structure 3755 **/ 3756 static void e1000_configure(struct e1000_adapter *adapter) 3757 { 3758 struct e1000_ring *rx_ring = adapter->rx_ring; 3759 3760 e1000e_set_rx_mode(adapter->netdev); 3761 3762 e1000_restore_vlan(adapter); 3763 e1000_init_manageability_pt(adapter); 3764 3765 e1000_configure_tx(adapter); 3766 3767 if (adapter->netdev->features & NETIF_F_RXHASH) 3768 e1000e_setup_rss_hash(adapter); 3769 e1000_setup_rctl(adapter); 3770 e1000_configure_rx(adapter); 3771 adapter->alloc_rx_buf(rx_ring, e1000_desc_unused(rx_ring), GFP_KERNEL); 3772 } 3773 3774 /** 3775 * e1000e_power_up_phy - restore link in case the phy was powered down 3776 * @adapter: address of board private structure 3777 * 3778 * The phy may be powered down to save power and turn off link when the 3779 * driver is unloaded and wake on lan is not enabled (among others) 3780 * *** this routine MUST be followed by a call to e1000e_reset *** 3781 **/ 3782 void e1000e_power_up_phy(struct e1000_adapter *adapter) 3783 { 3784 if (adapter->hw.phy.ops.power_up) 3785 adapter->hw.phy.ops.power_up(&adapter->hw); 3786 3787 adapter->hw.mac.ops.setup_link(&adapter->hw); 3788 } 3789 3790 /** 3791 * e1000_power_down_phy - Power down the PHY 3792 * @adapter: board private structure 3793 * 3794 * Power down the PHY so no link is implied when interface is down. 3795 * The PHY cannot be powered down if management or WoL is active. 3796 */ 3797 static void e1000_power_down_phy(struct e1000_adapter *adapter) 3798 { 3799 if (adapter->hw.phy.ops.power_down) 3800 adapter->hw.phy.ops.power_down(&adapter->hw); 3801 } 3802 3803 /** 3804 * e1000_flush_tx_ring - remove all descriptors from the tx_ring 3805 * @adapter: board private structure 3806 * 3807 * We want to clear all pending descriptors from the TX ring. 3808 * zeroing happens when the HW reads the regs. We assign the ring itself as 3809 * the data of the next descriptor. We don't care about the data we are about 3810 * to reset the HW. 3811 */ 3812 static void e1000_flush_tx_ring(struct e1000_adapter *adapter) 3813 { 3814 struct e1000_hw *hw = &adapter->hw; 3815 struct e1000_ring *tx_ring = adapter->tx_ring; 3816 struct e1000_tx_desc *tx_desc = NULL; 3817 u32 tdt, tctl, txd_lower = E1000_TXD_CMD_IFCS; 3818 u16 size = 512; 3819 3820 tctl = er32(TCTL); 3821 ew32(TCTL, tctl | E1000_TCTL_EN); 3822 tdt = er32(TDT(0)); 3823 BUG_ON(tdt != tx_ring->next_to_use); 3824 tx_desc = E1000_TX_DESC(*tx_ring, tx_ring->next_to_use); 3825 tx_desc->buffer_addr = cpu_to_le64(tx_ring->dma); 3826 3827 tx_desc->lower.data = cpu_to_le32(txd_lower | size); 3828 tx_desc->upper.data = 0; 3829 /* flush descriptors to memory before notifying the HW */ 3830 wmb(); 3831 tx_ring->next_to_use++; 3832 if (tx_ring->next_to_use == tx_ring->count) 3833 tx_ring->next_to_use = 0; 3834 ew32(TDT(0), tx_ring->next_to_use); 3835 usleep_range(200, 250); 3836 } 3837 3838 /** 3839 * e1000_flush_rx_ring - remove all descriptors from the rx_ring 3840 * @adapter: board private structure 3841 * 3842 * Mark all descriptors in the RX ring as consumed and disable the rx ring 3843 */ 3844 static void e1000_flush_rx_ring(struct e1000_adapter *adapter) 3845 { 3846 u32 rctl, rxdctl; 3847 struct e1000_hw *hw = &adapter->hw; 3848 3849 rctl = er32(RCTL); 3850 ew32(RCTL, rctl & ~E1000_RCTL_EN); 3851 e1e_flush(); 3852 usleep_range(100, 150); 3853 3854 rxdctl = er32(RXDCTL(0)); 3855 /* zero the lower 14 bits (prefetch and host thresholds) */ 3856 rxdctl &= 0xffffc000; 3857 3858 /* update thresholds: prefetch threshold to 31, host threshold to 1 3859 * and make sure the granularity is "descriptors" and not "cache lines" 3860 */ 3861 rxdctl |= (0x1F | BIT(8) | E1000_RXDCTL_THRESH_UNIT_DESC); 3862 3863 ew32(RXDCTL(0), rxdctl); 3864 /* momentarily enable the RX ring for the changes to take effect */ 3865 ew32(RCTL, rctl | E1000_RCTL_EN); 3866 e1e_flush(); 3867 usleep_range(100, 150); 3868 ew32(RCTL, rctl & ~E1000_RCTL_EN); 3869 } 3870 3871 /** 3872 * e1000_flush_desc_rings - remove all descriptors from the descriptor rings 3873 * @adapter: board private structure 3874 * 3875 * In i219, the descriptor rings must be emptied before resetting the HW 3876 * or before changing the device state to D3 during runtime (runtime PM). 3877 * 3878 * Failure to do this will cause the HW to enter a unit hang state which can 3879 * only be released by PCI reset on the device 3880 * 3881 */ 3882 3883 static void e1000_flush_desc_rings(struct e1000_adapter *adapter) 3884 { 3885 u16 hang_state; 3886 u32 fext_nvm11, tdlen; 3887 struct e1000_hw *hw = &adapter->hw; 3888 3889 /* First, disable MULR fix in FEXTNVM11 */ 3890 fext_nvm11 = er32(FEXTNVM11); 3891 fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX; 3892 ew32(FEXTNVM11, fext_nvm11); 3893 /* do nothing if we're not in faulty state, or if the queue is empty */ 3894 tdlen = er32(TDLEN(0)); 3895 pci_read_config_word(adapter->pdev, PCICFG_DESC_RING_STATUS, 3896 &hang_state); 3897 if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen) 3898 return; 3899 e1000_flush_tx_ring(adapter); 3900 /* recheck, maybe the fault is caused by the rx ring */ 3901 pci_read_config_word(adapter->pdev, PCICFG_DESC_RING_STATUS, 3902 &hang_state); 3903 if (hang_state & FLUSH_DESC_REQUIRED) 3904 e1000_flush_rx_ring(adapter); 3905 } 3906 3907 /** 3908 * e1000e_systim_reset - reset the timesync registers after a hardware reset 3909 * @adapter: board private structure 3910 * 3911 * When the MAC is reset, all hardware bits for timesync will be reset to the 3912 * default values. This function will restore the settings last in place. 3913 * Since the clock SYSTIME registers are reset, we will simply restore the 3914 * cyclecounter to the kernel real clock time. 3915 **/ 3916 static void e1000e_systim_reset(struct e1000_adapter *adapter) 3917 { 3918 struct ptp_clock_info *info = &adapter->ptp_clock_info; 3919 struct e1000_hw *hw = &adapter->hw; 3920 unsigned long flags; 3921 u32 timinca; 3922 s32 ret_val; 3923 3924 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP)) 3925 return; 3926 3927 if (info->adjfreq) { 3928 /* restore the previous ptp frequency delta */ 3929 ret_val = info->adjfreq(info, adapter->ptp_delta); 3930 } else { 3931 /* set the default base frequency if no adjustment possible */ 3932 ret_val = e1000e_get_base_timinca(adapter, &timinca); 3933 if (!ret_val) 3934 ew32(TIMINCA, timinca); 3935 } 3936 3937 if (ret_val) { 3938 dev_warn(&adapter->pdev->dev, 3939 "Failed to restore TIMINCA clock rate delta: %d\n", 3940 ret_val); 3941 return; 3942 } 3943 3944 /* reset the systim ns time counter */ 3945 spin_lock_irqsave(&adapter->systim_lock, flags); 3946 timecounter_init(&adapter->tc, &adapter->cc, 3947 ktime_to_ns(ktime_get_real())); 3948 spin_unlock_irqrestore(&adapter->systim_lock, flags); 3949 3950 /* restore the previous hwtstamp configuration settings */ 3951 e1000e_config_hwtstamp(adapter, &adapter->hwtstamp_config); 3952 } 3953 3954 /** 3955 * e1000e_reset - bring the hardware into a known good state 3956 * @adapter: board private structure 3957 * 3958 * This function boots the hardware and enables some settings that 3959 * require a configuration cycle of the hardware - those cannot be 3960 * set/changed during runtime. After reset the device needs to be 3961 * properly configured for Rx, Tx etc. 3962 */ 3963 void e1000e_reset(struct e1000_adapter *adapter) 3964 { 3965 struct e1000_mac_info *mac = &adapter->hw.mac; 3966 struct e1000_fc_info *fc = &adapter->hw.fc; 3967 struct e1000_hw *hw = &adapter->hw; 3968 u32 tx_space, min_tx_space, min_rx_space; 3969 u32 pba = adapter->pba; 3970 u16 hwm; 3971 3972 /* reset Packet Buffer Allocation to default */ 3973 ew32(PBA, pba); 3974 3975 if (adapter->max_frame_size > (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)) { 3976 /* To maintain wire speed transmits, the Tx FIFO should be 3977 * large enough to accommodate two full transmit packets, 3978 * rounded up to the next 1KB and expressed in KB. Likewise, 3979 * the Rx FIFO should be large enough to accommodate at least 3980 * one full receive packet and is similarly rounded up and 3981 * expressed in KB. 3982 */ 3983 pba = er32(PBA); 3984 /* upper 16 bits has Tx packet buffer allocation size in KB */ 3985 tx_space = pba >> 16; 3986 /* lower 16 bits has Rx packet buffer allocation size in KB */ 3987 pba &= 0xffff; 3988 /* the Tx fifo also stores 16 bytes of information about the Tx 3989 * but don't include ethernet FCS because hardware appends it 3990 */ 3991 min_tx_space = (adapter->max_frame_size + 3992 sizeof(struct e1000_tx_desc) - ETH_FCS_LEN) * 2; 3993 min_tx_space = ALIGN(min_tx_space, 1024); 3994 min_tx_space >>= 10; 3995 /* software strips receive CRC, so leave room for it */ 3996 min_rx_space = adapter->max_frame_size; 3997 min_rx_space = ALIGN(min_rx_space, 1024); 3998 min_rx_space >>= 10; 3999 4000 /* If current Tx allocation is less than the min Tx FIFO size, 4001 * and the min Tx FIFO size is less than the current Rx FIFO 4002 * allocation, take space away from current Rx allocation 4003 */ 4004 if ((tx_space < min_tx_space) && 4005 ((min_tx_space - tx_space) < pba)) { 4006 pba -= min_tx_space - tx_space; 4007 4008 /* if short on Rx space, Rx wins and must trump Tx 4009 * adjustment 4010 */ 4011 if (pba < min_rx_space) 4012 pba = min_rx_space; 4013 } 4014 4015 ew32(PBA, pba); 4016 } 4017 4018 /* flow control settings 4019 * 4020 * The high water mark must be low enough to fit one full frame 4021 * (or the size used for early receive) above it in the Rx FIFO. 4022 * Set it to the lower of: 4023 * - 90% of the Rx FIFO size, and 4024 * - the full Rx FIFO size minus one full frame 4025 */ 4026 if (adapter->flags & FLAG_DISABLE_FC_PAUSE_TIME) 4027 fc->pause_time = 0xFFFF; 4028 else 4029 fc->pause_time = E1000_FC_PAUSE_TIME; 4030 fc->send_xon = true; 4031 fc->current_mode = fc->requested_mode; 4032 4033 switch (hw->mac.type) { 4034 case e1000_ich9lan: 4035 case e1000_ich10lan: 4036 if (adapter->netdev->mtu > ETH_DATA_LEN) { 4037 pba = 14; 4038 ew32(PBA, pba); 4039 fc->high_water = 0x2800; 4040 fc->low_water = fc->high_water - 8; 4041 break; 4042 } 4043 fallthrough; 4044 default: 4045 hwm = min(((pba << 10) * 9 / 10), 4046 ((pba << 10) - adapter->max_frame_size)); 4047 4048 fc->high_water = hwm & E1000_FCRTH_RTH; /* 8-byte granularity */ 4049 fc->low_water = fc->high_water - 8; 4050 break; 4051 case e1000_pchlan: 4052 /* Workaround PCH LOM adapter hangs with certain network 4053 * loads. If hangs persist, try disabling Tx flow control. 4054 */ 4055 if (adapter->netdev->mtu > ETH_DATA_LEN) { 4056 fc->high_water = 0x3500; 4057 fc->low_water = 0x1500; 4058 } else { 4059 fc->high_water = 0x5000; 4060 fc->low_water = 0x3000; 4061 } 4062 fc->refresh_time = 0x1000; 4063 break; 4064 case e1000_pch2lan: 4065 case e1000_pch_lpt: 4066 case e1000_pch_spt: 4067 case e1000_pch_cnp: 4068 case e1000_pch_tgp: 4069 case e1000_pch_adp: 4070 case e1000_pch_mtp: 4071 fc->refresh_time = 0xFFFF; 4072 fc->pause_time = 0xFFFF; 4073 4074 if (adapter->netdev->mtu <= ETH_DATA_LEN) { 4075 fc->high_water = 0x05C20; 4076 fc->low_water = 0x05048; 4077 break; 4078 } 4079 4080 pba = 14; 4081 ew32(PBA, pba); 4082 fc->high_water = ((pba << 10) * 9 / 10) & E1000_FCRTH_RTH; 4083 fc->low_water = ((pba << 10) * 8 / 10) & E1000_FCRTL_RTL; 4084 break; 4085 } 4086 4087 /* Alignment of Tx data is on an arbitrary byte boundary with the 4088 * maximum size per Tx descriptor limited only to the transmit 4089 * allocation of the packet buffer minus 96 bytes with an upper 4090 * limit of 24KB due to receive synchronization limitations. 4091 */ 4092 adapter->tx_fifo_limit = min_t(u32, ((er32(PBA) >> 16) << 10) - 96, 4093 24 << 10); 4094 4095 /* Disable Adaptive Interrupt Moderation if 2 full packets cannot 4096 * fit in receive buffer. 4097 */ 4098 if (adapter->itr_setting & 0x3) { 4099 if ((adapter->max_frame_size * 2) > (pba << 10)) { 4100 if (!(adapter->flags2 & FLAG2_DISABLE_AIM)) { 4101 dev_info(&adapter->pdev->dev, 4102 "Interrupt Throttle Rate off\n"); 4103 adapter->flags2 |= FLAG2_DISABLE_AIM; 4104 e1000e_write_itr(adapter, 0); 4105 } 4106 } else if (adapter->flags2 & FLAG2_DISABLE_AIM) { 4107 dev_info(&adapter->pdev->dev, 4108 "Interrupt Throttle Rate on\n"); 4109 adapter->flags2 &= ~FLAG2_DISABLE_AIM; 4110 adapter->itr = 20000; 4111 e1000e_write_itr(adapter, adapter->itr); 4112 } 4113 } 4114 4115 if (hw->mac.type >= e1000_pch_spt) 4116 e1000_flush_desc_rings(adapter); 4117 /* Allow time for pending master requests to run */ 4118 mac->ops.reset_hw(hw); 4119 4120 /* For parts with AMT enabled, let the firmware know 4121 * that the network interface is in control 4122 */ 4123 if (adapter->flags & FLAG_HAS_AMT) 4124 e1000e_get_hw_control(adapter); 4125 4126 ew32(WUC, 0); 4127 4128 if (mac->ops.init_hw(hw)) 4129 e_err("Hardware Error\n"); 4130 4131 e1000_update_mng_vlan(adapter); 4132 4133 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */ 4134 ew32(VET, ETH_P_8021Q); 4135 4136 e1000e_reset_adaptive(hw); 4137 4138 /* restore systim and hwtstamp settings */ 4139 e1000e_systim_reset(adapter); 4140 4141 /* Set EEE advertisement as appropriate */ 4142 if (adapter->flags2 & FLAG2_HAS_EEE) { 4143 s32 ret_val; 4144 u16 adv_addr; 4145 4146 switch (hw->phy.type) { 4147 case e1000_phy_82579: 4148 adv_addr = I82579_EEE_ADVERTISEMENT; 4149 break; 4150 case e1000_phy_i217: 4151 adv_addr = I217_EEE_ADVERTISEMENT; 4152 break; 4153 default: 4154 dev_err(&adapter->pdev->dev, 4155 "Invalid PHY type setting EEE advertisement\n"); 4156 return; 4157 } 4158 4159 ret_val = hw->phy.ops.acquire(hw); 4160 if (ret_val) { 4161 dev_err(&adapter->pdev->dev, 4162 "EEE advertisement - unable to acquire PHY\n"); 4163 return; 4164 } 4165 4166 e1000_write_emi_reg_locked(hw, adv_addr, 4167 hw->dev_spec.ich8lan.eee_disable ? 4168 0 : adapter->eee_advert); 4169 4170 hw->phy.ops.release(hw); 4171 } 4172 4173 if (!netif_running(adapter->netdev) && 4174 !test_bit(__E1000_TESTING, &adapter->state)) 4175 e1000_power_down_phy(adapter); 4176 4177 e1000_get_phy_info(hw); 4178 4179 if ((adapter->flags & FLAG_HAS_SMART_POWER_DOWN) && 4180 !(adapter->flags & FLAG_SMART_POWER_DOWN)) { 4181 u16 phy_data = 0; 4182 /* speed up time to link by disabling smart power down, ignore 4183 * the return value of this function because there is nothing 4184 * different we would do if it failed 4185 */ 4186 e1e_rphy(hw, IGP02E1000_PHY_POWER_MGMT, &phy_data); 4187 phy_data &= ~IGP02E1000_PM_SPD; 4188 e1e_wphy(hw, IGP02E1000_PHY_POWER_MGMT, phy_data); 4189 } 4190 if (hw->mac.type >= e1000_pch_spt && adapter->int_mode == 0) { 4191 u32 reg; 4192 4193 /* Fextnvm7 @ 0xe4[2] = 1 */ 4194 reg = er32(FEXTNVM7); 4195 reg |= E1000_FEXTNVM7_SIDE_CLK_UNGATE; 4196 ew32(FEXTNVM7, reg); 4197 /* Fextnvm9 @ 0x5bb4[13:12] = 11 */ 4198 reg = er32(FEXTNVM9); 4199 reg |= E1000_FEXTNVM9_IOSFSB_CLKGATE_DIS | 4200 E1000_FEXTNVM9_IOSFSB_CLKREQ_DIS; 4201 ew32(FEXTNVM9, reg); 4202 } 4203 4204 } 4205 4206 /** 4207 * e1000e_trigger_lsc - trigger an LSC interrupt 4208 * @adapter: 4209 * 4210 * Fire a link status change interrupt to start the watchdog. 4211 **/ 4212 static void e1000e_trigger_lsc(struct e1000_adapter *adapter) 4213 { 4214 struct e1000_hw *hw = &adapter->hw; 4215 4216 if (adapter->msix_entries) 4217 ew32(ICS, E1000_ICS_LSC | E1000_ICS_OTHER); 4218 else 4219 ew32(ICS, E1000_ICS_LSC); 4220 } 4221 4222 void e1000e_up(struct e1000_adapter *adapter) 4223 { 4224 /* hardware has been reset, we need to reload some things */ 4225 e1000_configure(adapter); 4226 4227 clear_bit(__E1000_DOWN, &adapter->state); 4228 4229 if (adapter->msix_entries) 4230 e1000_configure_msix(adapter); 4231 e1000_irq_enable(adapter); 4232 4233 /* Tx queue started by watchdog timer when link is up */ 4234 4235 e1000e_trigger_lsc(adapter); 4236 } 4237 4238 static void e1000e_flush_descriptors(struct e1000_adapter *adapter) 4239 { 4240 struct e1000_hw *hw = &adapter->hw; 4241 4242 if (!(adapter->flags2 & FLAG2_DMA_BURST)) 4243 return; 4244 4245 /* flush pending descriptor writebacks to memory */ 4246 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 4247 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD); 4248 4249 /* execute the writes immediately */ 4250 e1e_flush(); 4251 4252 /* due to rare timing issues, write to TIDV/RDTR again to ensure the 4253 * write is successful 4254 */ 4255 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 4256 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD); 4257 4258 /* execute the writes immediately */ 4259 e1e_flush(); 4260 } 4261 4262 static void e1000e_update_stats(struct e1000_adapter *adapter); 4263 4264 /** 4265 * e1000e_down - quiesce the device and optionally reset the hardware 4266 * @adapter: board private structure 4267 * @reset: boolean flag to reset the hardware or not 4268 */ 4269 void e1000e_down(struct e1000_adapter *adapter, bool reset) 4270 { 4271 struct net_device *netdev = adapter->netdev; 4272 struct e1000_hw *hw = &adapter->hw; 4273 u32 tctl, rctl; 4274 4275 /* signal that we're down so the interrupt handler does not 4276 * reschedule our watchdog timer 4277 */ 4278 set_bit(__E1000_DOWN, &adapter->state); 4279 4280 netif_carrier_off(netdev); 4281 4282 /* disable receives in the hardware */ 4283 rctl = er32(RCTL); 4284 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX)) 4285 ew32(RCTL, rctl & ~E1000_RCTL_EN); 4286 /* flush and sleep below */ 4287 4288 netif_stop_queue(netdev); 4289 4290 /* disable transmits in the hardware */ 4291 tctl = er32(TCTL); 4292 tctl &= ~E1000_TCTL_EN; 4293 ew32(TCTL, tctl); 4294 4295 /* flush both disables and wait for them to finish */ 4296 e1e_flush(); 4297 usleep_range(10000, 11000); 4298 4299 e1000_irq_disable(adapter); 4300 4301 napi_synchronize(&adapter->napi); 4302 4303 del_timer_sync(&adapter->watchdog_timer); 4304 del_timer_sync(&adapter->phy_info_timer); 4305 4306 spin_lock(&adapter->stats64_lock); 4307 e1000e_update_stats(adapter); 4308 spin_unlock(&adapter->stats64_lock); 4309 4310 e1000e_flush_descriptors(adapter); 4311 4312 adapter->link_speed = 0; 4313 adapter->link_duplex = 0; 4314 4315 /* Disable Si errata workaround on PCHx for jumbo frame flow */ 4316 if ((hw->mac.type >= e1000_pch2lan) && 4317 (adapter->netdev->mtu > ETH_DATA_LEN) && 4318 e1000_lv_jumbo_workaround_ich8lan(hw, false)) 4319 e_dbg("failed to disable jumbo frame workaround mode\n"); 4320 4321 if (!pci_channel_offline(adapter->pdev)) { 4322 if (reset) 4323 e1000e_reset(adapter); 4324 else if (hw->mac.type >= e1000_pch_spt) 4325 e1000_flush_desc_rings(adapter); 4326 } 4327 e1000_clean_tx_ring(adapter->tx_ring); 4328 e1000_clean_rx_ring(adapter->rx_ring); 4329 } 4330 4331 void e1000e_reinit_locked(struct e1000_adapter *adapter) 4332 { 4333 might_sleep(); 4334 while (test_and_set_bit(__E1000_RESETTING, &adapter->state)) 4335 usleep_range(1000, 1100); 4336 e1000e_down(adapter, true); 4337 e1000e_up(adapter); 4338 clear_bit(__E1000_RESETTING, &adapter->state); 4339 } 4340 4341 /** 4342 * e1000e_sanitize_systim - sanitize raw cycle counter reads 4343 * @hw: pointer to the HW structure 4344 * @systim: PHC time value read, sanitized and returned 4345 * @sts: structure to hold system time before and after reading SYSTIML, 4346 * may be NULL 4347 * 4348 * Errata for 82574/82583 possible bad bits read from SYSTIMH/L: 4349 * check to see that the time is incrementing at a reasonable 4350 * rate and is a multiple of incvalue. 4351 **/ 4352 static u64 e1000e_sanitize_systim(struct e1000_hw *hw, u64 systim, 4353 struct ptp_system_timestamp *sts) 4354 { 4355 u64 time_delta, rem, temp; 4356 u64 systim_next; 4357 u32 incvalue; 4358 int i; 4359 4360 incvalue = er32(TIMINCA) & E1000_TIMINCA_INCVALUE_MASK; 4361 for (i = 0; i < E1000_MAX_82574_SYSTIM_REREADS; i++) { 4362 /* latch SYSTIMH on read of SYSTIML */ 4363 ptp_read_system_prets(sts); 4364 systim_next = (u64)er32(SYSTIML); 4365 ptp_read_system_postts(sts); 4366 systim_next |= (u64)er32(SYSTIMH) << 32; 4367 4368 time_delta = systim_next - systim; 4369 temp = time_delta; 4370 /* VMWare users have seen incvalue of zero, don't div / 0 */ 4371 rem = incvalue ? do_div(temp, incvalue) : (time_delta != 0); 4372 4373 systim = systim_next; 4374 4375 if ((time_delta < E1000_82574_SYSTIM_EPSILON) && (rem == 0)) 4376 break; 4377 } 4378 4379 return systim; 4380 } 4381 4382 /** 4383 * e1000e_read_systim - read SYSTIM register 4384 * @adapter: board private structure 4385 * @sts: structure which will contain system time before and after reading 4386 * SYSTIML, may be NULL 4387 **/ 4388 u64 e1000e_read_systim(struct e1000_adapter *adapter, 4389 struct ptp_system_timestamp *sts) 4390 { 4391 struct e1000_hw *hw = &adapter->hw; 4392 u32 systimel, systimel_2, systimeh; 4393 u64 systim; 4394 /* SYSTIMH latching upon SYSTIML read does not work well. 4395 * This means that if SYSTIML overflows after we read it but before 4396 * we read SYSTIMH, the value of SYSTIMH has been incremented and we 4397 * will experience a huge non linear increment in the systime value 4398 * to fix that we test for overflow and if true, we re-read systime. 4399 */ 4400 ptp_read_system_prets(sts); 4401 systimel = er32(SYSTIML); 4402 ptp_read_system_postts(sts); 4403 systimeh = er32(SYSTIMH); 4404 /* Is systimel is so large that overflow is possible? */ 4405 if (systimel >= (u32)0xffffffff - E1000_TIMINCA_INCVALUE_MASK) { 4406 ptp_read_system_prets(sts); 4407 systimel_2 = er32(SYSTIML); 4408 ptp_read_system_postts(sts); 4409 if (systimel > systimel_2) { 4410 /* There was an overflow, read again SYSTIMH, and use 4411 * systimel_2 4412 */ 4413 systimeh = er32(SYSTIMH); 4414 systimel = systimel_2; 4415 } 4416 } 4417 systim = (u64)systimel; 4418 systim |= (u64)systimeh << 32; 4419 4420 if (adapter->flags2 & FLAG2_CHECK_SYSTIM_OVERFLOW) 4421 systim = e1000e_sanitize_systim(hw, systim, sts); 4422 4423 return systim; 4424 } 4425 4426 /** 4427 * e1000e_cyclecounter_read - read raw cycle counter (used by time counter) 4428 * @cc: cyclecounter structure 4429 **/ 4430 static u64 e1000e_cyclecounter_read(const struct cyclecounter *cc) 4431 { 4432 struct e1000_adapter *adapter = container_of(cc, struct e1000_adapter, 4433 cc); 4434 4435 return e1000e_read_systim(adapter, NULL); 4436 } 4437 4438 /** 4439 * e1000_sw_init - Initialize general software structures (struct e1000_adapter) 4440 * @adapter: board private structure to initialize 4441 * 4442 * e1000_sw_init initializes the Adapter private data structure. 4443 * Fields are initialized based on PCI device information and 4444 * OS network device settings (MTU size). 4445 **/ 4446 static int e1000_sw_init(struct e1000_adapter *adapter) 4447 { 4448 struct net_device *netdev = adapter->netdev; 4449 4450 adapter->rx_buffer_len = VLAN_ETH_FRAME_LEN + ETH_FCS_LEN; 4451 adapter->rx_ps_bsize0 = 128; 4452 adapter->max_frame_size = netdev->mtu + VLAN_ETH_HLEN + ETH_FCS_LEN; 4453 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 4454 adapter->tx_ring_count = E1000_DEFAULT_TXD; 4455 adapter->rx_ring_count = E1000_DEFAULT_RXD; 4456 4457 spin_lock_init(&adapter->stats64_lock); 4458 4459 e1000e_set_interrupt_capability(adapter); 4460 4461 if (e1000_alloc_queues(adapter)) 4462 return -ENOMEM; 4463 4464 /* Setup hardware time stamping cyclecounter */ 4465 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) { 4466 adapter->cc.read = e1000e_cyclecounter_read; 4467 adapter->cc.mask = CYCLECOUNTER_MASK(64); 4468 adapter->cc.mult = 1; 4469 /* cc.shift set in e1000e_get_base_tininca() */ 4470 4471 spin_lock_init(&adapter->systim_lock); 4472 INIT_WORK(&adapter->tx_hwtstamp_work, e1000e_tx_hwtstamp_work); 4473 } 4474 4475 /* Explicitly disable IRQ since the NIC can be in any state. */ 4476 e1000_irq_disable(adapter); 4477 4478 set_bit(__E1000_DOWN, &adapter->state); 4479 return 0; 4480 } 4481 4482 /** 4483 * e1000_intr_msi_test - Interrupt Handler 4484 * @irq: interrupt number 4485 * @data: pointer to a network interface device structure 4486 **/ 4487 static irqreturn_t e1000_intr_msi_test(int __always_unused irq, void *data) 4488 { 4489 struct net_device *netdev = data; 4490 struct e1000_adapter *adapter = netdev_priv(netdev); 4491 struct e1000_hw *hw = &adapter->hw; 4492 u32 icr = er32(ICR); 4493 4494 e_dbg("icr is %08X\n", icr); 4495 if (icr & E1000_ICR_RXSEQ) { 4496 adapter->flags &= ~FLAG_MSI_TEST_FAILED; 4497 /* Force memory writes to complete before acknowledging the 4498 * interrupt is handled. 4499 */ 4500 wmb(); 4501 } 4502 4503 return IRQ_HANDLED; 4504 } 4505 4506 /** 4507 * e1000_test_msi_interrupt - Returns 0 for successful test 4508 * @adapter: board private struct 4509 * 4510 * code flow taken from tg3.c 4511 **/ 4512 static int e1000_test_msi_interrupt(struct e1000_adapter *adapter) 4513 { 4514 struct net_device *netdev = adapter->netdev; 4515 struct e1000_hw *hw = &adapter->hw; 4516 int err; 4517 4518 /* poll_enable hasn't been called yet, so don't need disable */ 4519 /* clear any pending events */ 4520 er32(ICR); 4521 4522 /* free the real vector and request a test handler */ 4523 e1000_free_irq(adapter); 4524 e1000e_reset_interrupt_capability(adapter); 4525 4526 /* Assume that the test fails, if it succeeds then the test 4527 * MSI irq handler will unset this flag 4528 */ 4529 adapter->flags |= FLAG_MSI_TEST_FAILED; 4530 4531 err = pci_enable_msi(adapter->pdev); 4532 if (err) 4533 goto msi_test_failed; 4534 4535 err = request_irq(adapter->pdev->irq, e1000_intr_msi_test, 0, 4536 netdev->name, netdev); 4537 if (err) { 4538 pci_disable_msi(adapter->pdev); 4539 goto msi_test_failed; 4540 } 4541 4542 /* Force memory writes to complete before enabling and firing an 4543 * interrupt. 4544 */ 4545 wmb(); 4546 4547 e1000_irq_enable(adapter); 4548 4549 /* fire an unusual interrupt on the test handler */ 4550 ew32(ICS, E1000_ICS_RXSEQ); 4551 e1e_flush(); 4552 msleep(100); 4553 4554 e1000_irq_disable(adapter); 4555 4556 rmb(); /* read flags after interrupt has been fired */ 4557 4558 if (adapter->flags & FLAG_MSI_TEST_FAILED) { 4559 adapter->int_mode = E1000E_INT_MODE_LEGACY; 4560 e_info("MSI interrupt test failed, using legacy interrupt.\n"); 4561 } else { 4562 e_dbg("MSI interrupt test succeeded!\n"); 4563 } 4564 4565 free_irq(adapter->pdev->irq, netdev); 4566 pci_disable_msi(adapter->pdev); 4567 4568 msi_test_failed: 4569 e1000e_set_interrupt_capability(adapter); 4570 return e1000_request_irq(adapter); 4571 } 4572 4573 /** 4574 * e1000_test_msi - Returns 0 if MSI test succeeds or INTx mode is restored 4575 * @adapter: board private struct 4576 * 4577 * code flow taken from tg3.c, called with e1000 interrupts disabled. 4578 **/ 4579 static int e1000_test_msi(struct e1000_adapter *adapter) 4580 { 4581 int err; 4582 u16 pci_cmd; 4583 4584 if (!(adapter->flags & FLAG_MSI_ENABLED)) 4585 return 0; 4586 4587 /* disable SERR in case the MSI write causes a master abort */ 4588 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd); 4589 if (pci_cmd & PCI_COMMAND_SERR) 4590 pci_write_config_word(adapter->pdev, PCI_COMMAND, 4591 pci_cmd & ~PCI_COMMAND_SERR); 4592 4593 err = e1000_test_msi_interrupt(adapter); 4594 4595 /* re-enable SERR */ 4596 if (pci_cmd & PCI_COMMAND_SERR) { 4597 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd); 4598 pci_cmd |= PCI_COMMAND_SERR; 4599 pci_write_config_word(adapter->pdev, PCI_COMMAND, pci_cmd); 4600 } 4601 4602 return err; 4603 } 4604 4605 /** 4606 * e1000e_open - Called when a network interface is made active 4607 * @netdev: network interface device structure 4608 * 4609 * Returns 0 on success, negative value on failure 4610 * 4611 * The open entry point is called when a network interface is made 4612 * active by the system (IFF_UP). At this point all resources needed 4613 * for transmit and receive operations are allocated, the interrupt 4614 * handler is registered with the OS, the watchdog timer is started, 4615 * and the stack is notified that the interface is ready. 4616 **/ 4617 int e1000e_open(struct net_device *netdev) 4618 { 4619 struct e1000_adapter *adapter = netdev_priv(netdev); 4620 struct e1000_hw *hw = &adapter->hw; 4621 struct pci_dev *pdev = adapter->pdev; 4622 int err; 4623 4624 /* disallow open during test */ 4625 if (test_bit(__E1000_TESTING, &adapter->state)) 4626 return -EBUSY; 4627 4628 pm_runtime_get_sync(&pdev->dev); 4629 4630 netif_carrier_off(netdev); 4631 netif_stop_queue(netdev); 4632 4633 /* allocate transmit descriptors */ 4634 err = e1000e_setup_tx_resources(adapter->tx_ring); 4635 if (err) 4636 goto err_setup_tx; 4637 4638 /* allocate receive descriptors */ 4639 err = e1000e_setup_rx_resources(adapter->rx_ring); 4640 if (err) 4641 goto err_setup_rx; 4642 4643 /* If AMT is enabled, let the firmware know that the network 4644 * interface is now open and reset the part to a known state. 4645 */ 4646 if (adapter->flags & FLAG_HAS_AMT) { 4647 e1000e_get_hw_control(adapter); 4648 e1000e_reset(adapter); 4649 } 4650 4651 e1000e_power_up_phy(adapter); 4652 4653 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE; 4654 if ((adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN)) 4655 e1000_update_mng_vlan(adapter); 4656 4657 /* DMA latency requirement to workaround jumbo issue */ 4658 cpu_latency_qos_add_request(&adapter->pm_qos_req, PM_QOS_DEFAULT_VALUE); 4659 4660 /* before we allocate an interrupt, we must be ready to handle it. 4661 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt 4662 * as soon as we call pci_request_irq, so we have to setup our 4663 * clean_rx handler before we do so. 4664 */ 4665 e1000_configure(adapter); 4666 4667 err = e1000_request_irq(adapter); 4668 if (err) 4669 goto err_req_irq; 4670 4671 /* Work around PCIe errata with MSI interrupts causing some chipsets to 4672 * ignore e1000e MSI messages, which means we need to test our MSI 4673 * interrupt now 4674 */ 4675 if (adapter->int_mode != E1000E_INT_MODE_LEGACY) { 4676 err = e1000_test_msi(adapter); 4677 if (err) { 4678 e_err("Interrupt allocation failed\n"); 4679 goto err_req_irq; 4680 } 4681 } 4682 4683 /* From here on the code is the same as e1000e_up() */ 4684 clear_bit(__E1000_DOWN, &adapter->state); 4685 4686 napi_enable(&adapter->napi); 4687 4688 e1000_irq_enable(adapter); 4689 4690 adapter->tx_hang_recheck = false; 4691 4692 hw->mac.get_link_status = true; 4693 pm_runtime_put(&pdev->dev); 4694 4695 e1000e_trigger_lsc(adapter); 4696 4697 return 0; 4698 4699 err_req_irq: 4700 cpu_latency_qos_remove_request(&adapter->pm_qos_req); 4701 e1000e_release_hw_control(adapter); 4702 e1000_power_down_phy(adapter); 4703 e1000e_free_rx_resources(adapter->rx_ring); 4704 err_setup_rx: 4705 e1000e_free_tx_resources(adapter->tx_ring); 4706 err_setup_tx: 4707 e1000e_reset(adapter); 4708 pm_runtime_put_sync(&pdev->dev); 4709 4710 return err; 4711 } 4712 4713 /** 4714 * e1000e_close - Disables a network interface 4715 * @netdev: network interface device structure 4716 * 4717 * Returns 0, this is not allowed to fail 4718 * 4719 * The close entry point is called when an interface is de-activated 4720 * by the OS. The hardware is still under the drivers control, but 4721 * needs to be disabled. A global MAC reset is issued to stop the 4722 * hardware, and all transmit and receive resources are freed. 4723 **/ 4724 int e1000e_close(struct net_device *netdev) 4725 { 4726 struct e1000_adapter *adapter = netdev_priv(netdev); 4727 struct pci_dev *pdev = adapter->pdev; 4728 int count = E1000_CHECK_RESET_COUNT; 4729 4730 while (test_bit(__E1000_RESETTING, &adapter->state) && count--) 4731 usleep_range(10000, 11000); 4732 4733 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state)); 4734 4735 pm_runtime_get_sync(&pdev->dev); 4736 4737 if (netif_device_present(netdev)) { 4738 e1000e_down(adapter, true); 4739 e1000_free_irq(adapter); 4740 4741 /* Link status message must follow this format */ 4742 netdev_info(netdev, "NIC Link is Down\n"); 4743 } 4744 4745 napi_disable(&adapter->napi); 4746 4747 e1000e_free_tx_resources(adapter->tx_ring); 4748 e1000e_free_rx_resources(adapter->rx_ring); 4749 4750 /* kill manageability vlan ID if supported, but not if a vlan with 4751 * the same ID is registered on the host OS (let 8021q kill it) 4752 */ 4753 if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) 4754 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), 4755 adapter->mng_vlan_id); 4756 4757 /* If AMT is enabled, let the firmware know that the network 4758 * interface is now closed 4759 */ 4760 if ((adapter->flags & FLAG_HAS_AMT) && 4761 !test_bit(__E1000_TESTING, &adapter->state)) 4762 e1000e_release_hw_control(adapter); 4763 4764 cpu_latency_qos_remove_request(&adapter->pm_qos_req); 4765 4766 pm_runtime_put_sync(&pdev->dev); 4767 4768 return 0; 4769 } 4770 4771 /** 4772 * e1000_set_mac - Change the Ethernet Address of the NIC 4773 * @netdev: network interface device structure 4774 * @p: pointer to an address structure 4775 * 4776 * Returns 0 on success, negative on failure 4777 **/ 4778 static int e1000_set_mac(struct net_device *netdev, void *p) 4779 { 4780 struct e1000_adapter *adapter = netdev_priv(netdev); 4781 struct e1000_hw *hw = &adapter->hw; 4782 struct sockaddr *addr = p; 4783 4784 if (!is_valid_ether_addr(addr->sa_data)) 4785 return -EADDRNOTAVAIL; 4786 4787 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 4788 memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len); 4789 4790 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 4791 4792 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) { 4793 /* activate the work around */ 4794 e1000e_set_laa_state_82571(&adapter->hw, 1); 4795 4796 /* Hold a copy of the LAA in RAR[14] This is done so that 4797 * between the time RAR[0] gets clobbered and the time it 4798 * gets fixed (in e1000_watchdog), the actual LAA is in one 4799 * of the RARs and no incoming packets directed to this port 4800 * are dropped. Eventually the LAA will be in RAR[0] and 4801 * RAR[14] 4802 */ 4803 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 4804 adapter->hw.mac.rar_entry_count - 1); 4805 } 4806 4807 return 0; 4808 } 4809 4810 /** 4811 * e1000e_update_phy_task - work thread to update phy 4812 * @work: pointer to our work struct 4813 * 4814 * this worker thread exists because we must acquire a 4815 * semaphore to read the phy, which we could msleep while 4816 * waiting for it, and we can't msleep in a timer. 4817 **/ 4818 static void e1000e_update_phy_task(struct work_struct *work) 4819 { 4820 struct e1000_adapter *adapter = container_of(work, 4821 struct e1000_adapter, 4822 update_phy_task); 4823 struct e1000_hw *hw = &adapter->hw; 4824 4825 if (test_bit(__E1000_DOWN, &adapter->state)) 4826 return; 4827 4828 e1000_get_phy_info(hw); 4829 4830 /* Enable EEE on 82579 after link up */ 4831 if (hw->phy.type >= e1000_phy_82579) 4832 e1000_set_eee_pchlan(hw); 4833 } 4834 4835 /** 4836 * e1000_update_phy_info - timre call-back to update PHY info 4837 * @t: pointer to timer_list containing private info adapter 4838 * 4839 * Need to wait a few seconds after link up to get diagnostic information from 4840 * the phy 4841 **/ 4842 static void e1000_update_phy_info(struct timer_list *t) 4843 { 4844 struct e1000_adapter *adapter = from_timer(adapter, t, phy_info_timer); 4845 4846 if (test_bit(__E1000_DOWN, &adapter->state)) 4847 return; 4848 4849 schedule_work(&adapter->update_phy_task); 4850 } 4851 4852 /** 4853 * e1000e_update_phy_stats - Update the PHY statistics counters 4854 * @adapter: board private structure 4855 * 4856 * Read/clear the upper 16-bit PHY registers and read/accumulate lower 4857 **/ 4858 static void e1000e_update_phy_stats(struct e1000_adapter *adapter) 4859 { 4860 struct e1000_hw *hw = &adapter->hw; 4861 s32 ret_val; 4862 u16 phy_data; 4863 4864 ret_val = hw->phy.ops.acquire(hw); 4865 if (ret_val) 4866 return; 4867 4868 /* A page set is expensive so check if already on desired page. 4869 * If not, set to the page with the PHY status registers. 4870 */ 4871 hw->phy.addr = 1; 4872 ret_val = e1000e_read_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, 4873 &phy_data); 4874 if (ret_val) 4875 goto release; 4876 if (phy_data != (HV_STATS_PAGE << IGP_PAGE_SHIFT)) { 4877 ret_val = hw->phy.ops.set_page(hw, 4878 HV_STATS_PAGE << IGP_PAGE_SHIFT); 4879 if (ret_val) 4880 goto release; 4881 } 4882 4883 /* Single Collision Count */ 4884 hw->phy.ops.read_reg_page(hw, HV_SCC_UPPER, &phy_data); 4885 ret_val = hw->phy.ops.read_reg_page(hw, HV_SCC_LOWER, &phy_data); 4886 if (!ret_val) 4887 adapter->stats.scc += phy_data; 4888 4889 /* Excessive Collision Count */ 4890 hw->phy.ops.read_reg_page(hw, HV_ECOL_UPPER, &phy_data); 4891 ret_val = hw->phy.ops.read_reg_page(hw, HV_ECOL_LOWER, &phy_data); 4892 if (!ret_val) 4893 adapter->stats.ecol += phy_data; 4894 4895 /* Multiple Collision Count */ 4896 hw->phy.ops.read_reg_page(hw, HV_MCC_UPPER, &phy_data); 4897 ret_val = hw->phy.ops.read_reg_page(hw, HV_MCC_LOWER, &phy_data); 4898 if (!ret_val) 4899 adapter->stats.mcc += phy_data; 4900 4901 /* Late Collision Count */ 4902 hw->phy.ops.read_reg_page(hw, HV_LATECOL_UPPER, &phy_data); 4903 ret_val = hw->phy.ops.read_reg_page(hw, HV_LATECOL_LOWER, &phy_data); 4904 if (!ret_val) 4905 adapter->stats.latecol += phy_data; 4906 4907 /* Collision Count - also used for adaptive IFS */ 4908 hw->phy.ops.read_reg_page(hw, HV_COLC_UPPER, &phy_data); 4909 ret_val = hw->phy.ops.read_reg_page(hw, HV_COLC_LOWER, &phy_data); 4910 if (!ret_val) 4911 hw->mac.collision_delta = phy_data; 4912 4913 /* Defer Count */ 4914 hw->phy.ops.read_reg_page(hw, HV_DC_UPPER, &phy_data); 4915 ret_val = hw->phy.ops.read_reg_page(hw, HV_DC_LOWER, &phy_data); 4916 if (!ret_val) 4917 adapter->stats.dc += phy_data; 4918 4919 /* Transmit with no CRS */ 4920 hw->phy.ops.read_reg_page(hw, HV_TNCRS_UPPER, &phy_data); 4921 ret_val = hw->phy.ops.read_reg_page(hw, HV_TNCRS_LOWER, &phy_data); 4922 if (!ret_val) 4923 adapter->stats.tncrs += phy_data; 4924 4925 release: 4926 hw->phy.ops.release(hw); 4927 } 4928 4929 /** 4930 * e1000e_update_stats - Update the board statistics counters 4931 * @adapter: board private structure 4932 **/ 4933 static void e1000e_update_stats(struct e1000_adapter *adapter) 4934 { 4935 struct net_device *netdev = adapter->netdev; 4936 struct e1000_hw *hw = &adapter->hw; 4937 struct pci_dev *pdev = adapter->pdev; 4938 4939 /* Prevent stats update while adapter is being reset, or if the pci 4940 * connection is down. 4941 */ 4942 if (adapter->link_speed == 0) 4943 return; 4944 if (pci_channel_offline(pdev)) 4945 return; 4946 4947 adapter->stats.crcerrs += er32(CRCERRS); 4948 adapter->stats.gprc += er32(GPRC); 4949 adapter->stats.gorc += er32(GORCL); 4950 er32(GORCH); /* Clear gorc */ 4951 adapter->stats.bprc += er32(BPRC); 4952 adapter->stats.mprc += er32(MPRC); 4953 adapter->stats.roc += er32(ROC); 4954 4955 adapter->stats.mpc += er32(MPC); 4956 4957 /* Half-duplex statistics */ 4958 if (adapter->link_duplex == HALF_DUPLEX) { 4959 if (adapter->flags2 & FLAG2_HAS_PHY_STATS) { 4960 e1000e_update_phy_stats(adapter); 4961 } else { 4962 adapter->stats.scc += er32(SCC); 4963 adapter->stats.ecol += er32(ECOL); 4964 adapter->stats.mcc += er32(MCC); 4965 adapter->stats.latecol += er32(LATECOL); 4966 adapter->stats.dc += er32(DC); 4967 4968 hw->mac.collision_delta = er32(COLC); 4969 4970 if ((hw->mac.type != e1000_82574) && 4971 (hw->mac.type != e1000_82583)) 4972 adapter->stats.tncrs += er32(TNCRS); 4973 } 4974 adapter->stats.colc += hw->mac.collision_delta; 4975 } 4976 4977 adapter->stats.xonrxc += er32(XONRXC); 4978 adapter->stats.xontxc += er32(XONTXC); 4979 adapter->stats.xoffrxc += er32(XOFFRXC); 4980 adapter->stats.xofftxc += er32(XOFFTXC); 4981 adapter->stats.gptc += er32(GPTC); 4982 adapter->stats.gotc += er32(GOTCL); 4983 er32(GOTCH); /* Clear gotc */ 4984 adapter->stats.rnbc += er32(RNBC); 4985 adapter->stats.ruc += er32(RUC); 4986 4987 adapter->stats.mptc += er32(MPTC); 4988 adapter->stats.bptc += er32(BPTC); 4989 4990 /* used for adaptive IFS */ 4991 4992 hw->mac.tx_packet_delta = er32(TPT); 4993 adapter->stats.tpt += hw->mac.tx_packet_delta; 4994 4995 adapter->stats.algnerrc += er32(ALGNERRC); 4996 adapter->stats.rxerrc += er32(RXERRC); 4997 adapter->stats.cexterr += er32(CEXTERR); 4998 adapter->stats.tsctc += er32(TSCTC); 4999 adapter->stats.tsctfc += er32(TSCTFC); 5000 5001 /* Fill out the OS statistics structure */ 5002 netdev->stats.multicast = adapter->stats.mprc; 5003 netdev->stats.collisions = adapter->stats.colc; 5004 5005 /* Rx Errors */ 5006 5007 /* RLEC on some newer hardware can be incorrect so build 5008 * our own version based on RUC and ROC 5009 */ 5010 netdev->stats.rx_errors = adapter->stats.rxerrc + 5011 adapter->stats.crcerrs + adapter->stats.algnerrc + 5012 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr; 5013 netdev->stats.rx_length_errors = adapter->stats.ruc + 5014 adapter->stats.roc; 5015 netdev->stats.rx_crc_errors = adapter->stats.crcerrs; 5016 netdev->stats.rx_frame_errors = adapter->stats.algnerrc; 5017 netdev->stats.rx_missed_errors = adapter->stats.mpc; 5018 5019 /* Tx Errors */ 5020 netdev->stats.tx_errors = adapter->stats.ecol + adapter->stats.latecol; 5021 netdev->stats.tx_aborted_errors = adapter->stats.ecol; 5022 netdev->stats.tx_window_errors = adapter->stats.latecol; 5023 netdev->stats.tx_carrier_errors = adapter->stats.tncrs; 5024 5025 /* Tx Dropped needs to be maintained elsewhere */ 5026 5027 /* Management Stats */ 5028 adapter->stats.mgptc += er32(MGTPTC); 5029 adapter->stats.mgprc += er32(MGTPRC); 5030 adapter->stats.mgpdc += er32(MGTPDC); 5031 5032 /* Correctable ECC Errors */ 5033 if (hw->mac.type >= e1000_pch_lpt) { 5034 u32 pbeccsts = er32(PBECCSTS); 5035 5036 adapter->corr_errors += 5037 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 5038 adapter->uncorr_errors += 5039 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >> 5040 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT; 5041 } 5042 } 5043 5044 /** 5045 * e1000_phy_read_status - Update the PHY register status snapshot 5046 * @adapter: board private structure 5047 **/ 5048 static void e1000_phy_read_status(struct e1000_adapter *adapter) 5049 { 5050 struct e1000_hw *hw = &adapter->hw; 5051 struct e1000_phy_regs *phy = &adapter->phy_regs; 5052 5053 if (!pm_runtime_suspended((&adapter->pdev->dev)->parent) && 5054 (er32(STATUS) & E1000_STATUS_LU) && 5055 (adapter->hw.phy.media_type == e1000_media_type_copper)) { 5056 int ret_val; 5057 5058 ret_val = e1e_rphy(hw, MII_BMCR, &phy->bmcr); 5059 ret_val |= e1e_rphy(hw, MII_BMSR, &phy->bmsr); 5060 ret_val |= e1e_rphy(hw, MII_ADVERTISE, &phy->advertise); 5061 ret_val |= e1e_rphy(hw, MII_LPA, &phy->lpa); 5062 ret_val |= e1e_rphy(hw, MII_EXPANSION, &phy->expansion); 5063 ret_val |= e1e_rphy(hw, MII_CTRL1000, &phy->ctrl1000); 5064 ret_val |= e1e_rphy(hw, MII_STAT1000, &phy->stat1000); 5065 ret_val |= e1e_rphy(hw, MII_ESTATUS, &phy->estatus); 5066 if (ret_val) 5067 e_warn("Error reading PHY register\n"); 5068 } else { 5069 /* Do not read PHY registers if link is not up 5070 * Set values to typical power-on defaults 5071 */ 5072 phy->bmcr = (BMCR_SPEED1000 | BMCR_ANENABLE | BMCR_FULLDPLX); 5073 phy->bmsr = (BMSR_100FULL | BMSR_100HALF | BMSR_10FULL | 5074 BMSR_10HALF | BMSR_ESTATEN | BMSR_ANEGCAPABLE | 5075 BMSR_ERCAP); 5076 phy->advertise = (ADVERTISE_PAUSE_ASYM | ADVERTISE_PAUSE_CAP | 5077 ADVERTISE_ALL | ADVERTISE_CSMA); 5078 phy->lpa = 0; 5079 phy->expansion = EXPANSION_ENABLENPAGE; 5080 phy->ctrl1000 = ADVERTISE_1000FULL; 5081 phy->stat1000 = 0; 5082 phy->estatus = (ESTATUS_1000_TFULL | ESTATUS_1000_THALF); 5083 } 5084 } 5085 5086 static void e1000_print_link_info(struct e1000_adapter *adapter) 5087 { 5088 struct e1000_hw *hw = &adapter->hw; 5089 u32 ctrl = er32(CTRL); 5090 5091 /* Link status message must follow this format for user tools */ 5092 netdev_info(adapter->netdev, 5093 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 5094 adapter->link_speed, 5095 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half", 5096 (ctrl & E1000_CTRL_TFCE) && (ctrl & E1000_CTRL_RFCE) ? "Rx/Tx" : 5097 (ctrl & E1000_CTRL_RFCE) ? "Rx" : 5098 (ctrl & E1000_CTRL_TFCE) ? "Tx" : "None"); 5099 } 5100 5101 static bool e1000e_has_link(struct e1000_adapter *adapter) 5102 { 5103 struct e1000_hw *hw = &adapter->hw; 5104 bool link_active = false; 5105 s32 ret_val = 0; 5106 5107 /* get_link_status is set on LSC (link status) interrupt or 5108 * Rx sequence error interrupt. get_link_status will stay 5109 * true until the check_for_link establishes link 5110 * for copper adapters ONLY 5111 */ 5112 switch (hw->phy.media_type) { 5113 case e1000_media_type_copper: 5114 if (hw->mac.get_link_status) { 5115 ret_val = hw->mac.ops.check_for_link(hw); 5116 link_active = !hw->mac.get_link_status; 5117 } else { 5118 link_active = true; 5119 } 5120 break; 5121 case e1000_media_type_fiber: 5122 ret_val = hw->mac.ops.check_for_link(hw); 5123 link_active = !!(er32(STATUS) & E1000_STATUS_LU); 5124 break; 5125 case e1000_media_type_internal_serdes: 5126 ret_val = hw->mac.ops.check_for_link(hw); 5127 link_active = hw->mac.serdes_has_link; 5128 break; 5129 default: 5130 case e1000_media_type_unknown: 5131 break; 5132 } 5133 5134 if ((ret_val == -E1000_ERR_PHY) && (hw->phy.type == e1000_phy_igp_3) && 5135 (er32(CTRL) & E1000_PHY_CTRL_GBE_DISABLE)) { 5136 /* See e1000_kmrn_lock_loss_workaround_ich8lan() */ 5137 e_info("Gigabit has been disabled, downgrading speed\n"); 5138 } 5139 5140 return link_active; 5141 } 5142 5143 static void e1000e_enable_receives(struct e1000_adapter *adapter) 5144 { 5145 /* make sure the receive unit is started */ 5146 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) && 5147 (adapter->flags & FLAG_RESTART_NOW)) { 5148 struct e1000_hw *hw = &adapter->hw; 5149 u32 rctl = er32(RCTL); 5150 5151 ew32(RCTL, rctl | E1000_RCTL_EN); 5152 adapter->flags &= ~FLAG_RESTART_NOW; 5153 } 5154 } 5155 5156 static void e1000e_check_82574_phy_workaround(struct e1000_adapter *adapter) 5157 { 5158 struct e1000_hw *hw = &adapter->hw; 5159 5160 /* With 82574 controllers, PHY needs to be checked periodically 5161 * for hung state and reset, if two calls return true 5162 */ 5163 if (e1000_check_phy_82574(hw)) 5164 adapter->phy_hang_count++; 5165 else 5166 adapter->phy_hang_count = 0; 5167 5168 if (adapter->phy_hang_count > 1) { 5169 adapter->phy_hang_count = 0; 5170 e_dbg("PHY appears hung - resetting\n"); 5171 schedule_work(&adapter->reset_task); 5172 } 5173 } 5174 5175 /** 5176 * e1000_watchdog - Timer Call-back 5177 * @t: pointer to timer_list containing private info adapter 5178 **/ 5179 static void e1000_watchdog(struct timer_list *t) 5180 { 5181 struct e1000_adapter *adapter = from_timer(adapter, t, watchdog_timer); 5182 5183 /* Do the rest outside of interrupt context */ 5184 schedule_work(&adapter->watchdog_task); 5185 5186 /* TODO: make this use queue_delayed_work() */ 5187 } 5188 5189 static void e1000_watchdog_task(struct work_struct *work) 5190 { 5191 struct e1000_adapter *adapter = container_of(work, 5192 struct e1000_adapter, 5193 watchdog_task); 5194 struct net_device *netdev = adapter->netdev; 5195 struct e1000_mac_info *mac = &adapter->hw.mac; 5196 struct e1000_phy_info *phy = &adapter->hw.phy; 5197 struct e1000_ring *tx_ring = adapter->tx_ring; 5198 u32 dmoff_exit_timeout = 100, tries = 0; 5199 struct e1000_hw *hw = &adapter->hw; 5200 u32 link, tctl, pcim_state; 5201 5202 if (test_bit(__E1000_DOWN, &adapter->state)) 5203 return; 5204 5205 link = e1000e_has_link(adapter); 5206 if ((netif_carrier_ok(netdev)) && link) { 5207 /* Cancel scheduled suspend requests. */ 5208 pm_runtime_resume(netdev->dev.parent); 5209 5210 e1000e_enable_receives(adapter); 5211 goto link_up; 5212 } 5213 5214 if ((e1000e_enable_tx_pkt_filtering(hw)) && 5215 (adapter->mng_vlan_id != adapter->hw.mng_cookie.vlan_id)) 5216 e1000_update_mng_vlan(adapter); 5217 5218 if (link) { 5219 if (!netif_carrier_ok(netdev)) { 5220 bool txb2b = true; 5221 5222 /* Cancel scheduled suspend requests. */ 5223 pm_runtime_resume(netdev->dev.parent); 5224 5225 /* Checking if MAC is in DMoff state*/ 5226 if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID) { 5227 pcim_state = er32(STATUS); 5228 while (pcim_state & E1000_STATUS_PCIM_STATE) { 5229 if (tries++ == dmoff_exit_timeout) { 5230 e_dbg("Error in exiting dmoff\n"); 5231 break; 5232 } 5233 usleep_range(10000, 20000); 5234 pcim_state = er32(STATUS); 5235 5236 /* Checking if MAC exited DMoff state */ 5237 if (!(pcim_state & E1000_STATUS_PCIM_STATE)) 5238 e1000_phy_hw_reset(&adapter->hw); 5239 } 5240 } 5241 5242 /* update snapshot of PHY registers on LSC */ 5243 e1000_phy_read_status(adapter); 5244 mac->ops.get_link_up_info(&adapter->hw, 5245 &adapter->link_speed, 5246 &adapter->link_duplex); 5247 e1000_print_link_info(adapter); 5248 5249 /* check if SmartSpeed worked */ 5250 e1000e_check_downshift(hw); 5251 if (phy->speed_downgraded) 5252 netdev_warn(netdev, 5253 "Link Speed was downgraded by SmartSpeed\n"); 5254 5255 /* On supported PHYs, check for duplex mismatch only 5256 * if link has autonegotiated at 10/100 half 5257 */ 5258 if ((hw->phy.type == e1000_phy_igp_3 || 5259 hw->phy.type == e1000_phy_bm) && 5260 hw->mac.autoneg && 5261 (adapter->link_speed == SPEED_10 || 5262 adapter->link_speed == SPEED_100) && 5263 (adapter->link_duplex == HALF_DUPLEX)) { 5264 u16 autoneg_exp; 5265 5266 e1e_rphy(hw, MII_EXPANSION, &autoneg_exp); 5267 5268 if (!(autoneg_exp & EXPANSION_NWAY)) 5269 e_info("Autonegotiated half duplex but link partner cannot autoneg. Try forcing full duplex if link gets many collisions.\n"); 5270 } 5271 5272 /* adjust timeout factor according to speed/duplex */ 5273 adapter->tx_timeout_factor = 1; 5274 switch (adapter->link_speed) { 5275 case SPEED_10: 5276 txb2b = false; 5277 adapter->tx_timeout_factor = 16; 5278 break; 5279 case SPEED_100: 5280 txb2b = false; 5281 adapter->tx_timeout_factor = 10; 5282 break; 5283 } 5284 5285 /* workaround: re-program speed mode bit after 5286 * link-up event 5287 */ 5288 if ((adapter->flags & FLAG_TARC_SPEED_MODE_BIT) && 5289 !txb2b) { 5290 u32 tarc0; 5291 5292 tarc0 = er32(TARC(0)); 5293 tarc0 &= ~SPEED_MODE_BIT; 5294 ew32(TARC(0), tarc0); 5295 } 5296 5297 /* disable TSO for pcie and 10/100 speeds, to avoid 5298 * some hardware issues 5299 */ 5300 if (!(adapter->flags & FLAG_TSO_FORCE)) { 5301 switch (adapter->link_speed) { 5302 case SPEED_10: 5303 case SPEED_100: 5304 e_info("10/100 speed: disabling TSO\n"); 5305 netdev->features &= ~NETIF_F_TSO; 5306 netdev->features &= ~NETIF_F_TSO6; 5307 break; 5308 case SPEED_1000: 5309 netdev->features |= NETIF_F_TSO; 5310 netdev->features |= NETIF_F_TSO6; 5311 break; 5312 default: 5313 /* oops */ 5314 break; 5315 } 5316 if (hw->mac.type == e1000_pch_spt) { 5317 netdev->features &= ~NETIF_F_TSO; 5318 netdev->features &= ~NETIF_F_TSO6; 5319 } 5320 } 5321 5322 /* enable transmits in the hardware, need to do this 5323 * after setting TARC(0) 5324 */ 5325 tctl = er32(TCTL); 5326 tctl |= E1000_TCTL_EN; 5327 ew32(TCTL, tctl); 5328 5329 /* Perform any post-link-up configuration before 5330 * reporting link up. 5331 */ 5332 if (phy->ops.cfg_on_link_up) 5333 phy->ops.cfg_on_link_up(hw); 5334 5335 netif_wake_queue(netdev); 5336 netif_carrier_on(netdev); 5337 5338 if (!test_bit(__E1000_DOWN, &adapter->state)) 5339 mod_timer(&adapter->phy_info_timer, 5340 round_jiffies(jiffies + 2 * HZ)); 5341 } 5342 } else { 5343 if (netif_carrier_ok(netdev)) { 5344 adapter->link_speed = 0; 5345 adapter->link_duplex = 0; 5346 /* Link status message must follow this format */ 5347 netdev_info(netdev, "NIC Link is Down\n"); 5348 netif_carrier_off(netdev); 5349 netif_stop_queue(netdev); 5350 if (!test_bit(__E1000_DOWN, &adapter->state)) 5351 mod_timer(&adapter->phy_info_timer, 5352 round_jiffies(jiffies + 2 * HZ)); 5353 5354 /* 8000ES2LAN requires a Rx packet buffer work-around 5355 * on link down event; reset the controller to flush 5356 * the Rx packet buffer. 5357 */ 5358 if (adapter->flags & FLAG_RX_NEEDS_RESTART) 5359 adapter->flags |= FLAG_RESTART_NOW; 5360 else 5361 pm_schedule_suspend(netdev->dev.parent, 5362 LINK_TIMEOUT); 5363 } 5364 } 5365 5366 link_up: 5367 spin_lock(&adapter->stats64_lock); 5368 e1000e_update_stats(adapter); 5369 5370 mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old; 5371 adapter->tpt_old = adapter->stats.tpt; 5372 mac->collision_delta = adapter->stats.colc - adapter->colc_old; 5373 adapter->colc_old = adapter->stats.colc; 5374 5375 adapter->gorc = adapter->stats.gorc - adapter->gorc_old; 5376 adapter->gorc_old = adapter->stats.gorc; 5377 adapter->gotc = adapter->stats.gotc - adapter->gotc_old; 5378 adapter->gotc_old = adapter->stats.gotc; 5379 spin_unlock(&adapter->stats64_lock); 5380 5381 /* If the link is lost the controller stops DMA, but 5382 * if there is queued Tx work it cannot be done. So 5383 * reset the controller to flush the Tx packet buffers. 5384 */ 5385 if (!netif_carrier_ok(netdev) && 5386 (e1000_desc_unused(tx_ring) + 1 < tx_ring->count)) 5387 adapter->flags |= FLAG_RESTART_NOW; 5388 5389 /* If reset is necessary, do it outside of interrupt context. */ 5390 if (adapter->flags & FLAG_RESTART_NOW) { 5391 schedule_work(&adapter->reset_task); 5392 /* return immediately since reset is imminent */ 5393 return; 5394 } 5395 5396 e1000e_update_adaptive(&adapter->hw); 5397 5398 /* Simple mode for Interrupt Throttle Rate (ITR) */ 5399 if (adapter->itr_setting == 4) { 5400 /* Symmetric Tx/Rx gets a reduced ITR=2000; 5401 * Total asymmetrical Tx or Rx gets ITR=8000; 5402 * everyone else is between 2000-8000. 5403 */ 5404 u32 goc = (adapter->gotc + adapter->gorc) / 10000; 5405 u32 dif = (adapter->gotc > adapter->gorc ? 5406 adapter->gotc - adapter->gorc : 5407 adapter->gorc - adapter->gotc) / 10000; 5408 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000; 5409 5410 e1000e_write_itr(adapter, itr); 5411 } 5412 5413 /* Cause software interrupt to ensure Rx ring is cleaned */ 5414 if (adapter->msix_entries) 5415 ew32(ICS, adapter->rx_ring->ims_val); 5416 else 5417 ew32(ICS, E1000_ICS_RXDMT0); 5418 5419 /* flush pending descriptors to memory before detecting Tx hang */ 5420 e1000e_flush_descriptors(adapter); 5421 5422 /* Force detection of hung controller every watchdog period */ 5423 adapter->detect_tx_hung = true; 5424 5425 /* With 82571 controllers, LAA may be overwritten due to controller 5426 * reset from the other port. Set the appropriate LAA in RAR[0] 5427 */ 5428 if (e1000e_get_laa_state_82571(hw)) 5429 hw->mac.ops.rar_set(hw, adapter->hw.mac.addr, 0); 5430 5431 if (adapter->flags2 & FLAG2_CHECK_PHY_HANG) 5432 e1000e_check_82574_phy_workaround(adapter); 5433 5434 /* Clear valid timestamp stuck in RXSTMPL/H due to a Rx error */ 5435 if (adapter->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE) { 5436 if ((adapter->flags2 & FLAG2_CHECK_RX_HWTSTAMP) && 5437 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) { 5438 er32(RXSTMPH); 5439 adapter->rx_hwtstamp_cleared++; 5440 } else { 5441 adapter->flags2 |= FLAG2_CHECK_RX_HWTSTAMP; 5442 } 5443 } 5444 5445 /* Reset the timer */ 5446 if (!test_bit(__E1000_DOWN, &adapter->state)) 5447 mod_timer(&adapter->watchdog_timer, 5448 round_jiffies(jiffies + 2 * HZ)); 5449 } 5450 5451 #define E1000_TX_FLAGS_CSUM 0x00000001 5452 #define E1000_TX_FLAGS_VLAN 0x00000002 5453 #define E1000_TX_FLAGS_TSO 0x00000004 5454 #define E1000_TX_FLAGS_IPV4 0x00000008 5455 #define E1000_TX_FLAGS_NO_FCS 0x00000010 5456 #define E1000_TX_FLAGS_HWTSTAMP 0x00000020 5457 #define E1000_TX_FLAGS_VLAN_MASK 0xffff0000 5458 #define E1000_TX_FLAGS_VLAN_SHIFT 16 5459 5460 static int e1000_tso(struct e1000_ring *tx_ring, struct sk_buff *skb, 5461 __be16 protocol) 5462 { 5463 struct e1000_context_desc *context_desc; 5464 struct e1000_buffer *buffer_info; 5465 unsigned int i; 5466 u32 cmd_length = 0; 5467 u16 ipcse = 0, mss; 5468 u8 ipcss, ipcso, tucss, tucso, hdr_len; 5469 int err; 5470 5471 if (!skb_is_gso(skb)) 5472 return 0; 5473 5474 err = skb_cow_head(skb, 0); 5475 if (err < 0) 5476 return err; 5477 5478 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 5479 mss = skb_shinfo(skb)->gso_size; 5480 if (protocol == htons(ETH_P_IP)) { 5481 struct iphdr *iph = ip_hdr(skb); 5482 iph->tot_len = 0; 5483 iph->check = 0; 5484 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 5485 0, IPPROTO_TCP, 0); 5486 cmd_length = E1000_TXD_CMD_IP; 5487 ipcse = skb_transport_offset(skb) - 1; 5488 } else if (skb_is_gso_v6(skb)) { 5489 tcp_v6_gso_csum_prep(skb); 5490 ipcse = 0; 5491 } 5492 ipcss = skb_network_offset(skb); 5493 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data; 5494 tucss = skb_transport_offset(skb); 5495 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data; 5496 5497 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE | 5498 E1000_TXD_CMD_TCP | (skb->len - (hdr_len))); 5499 5500 i = tx_ring->next_to_use; 5501 context_desc = E1000_CONTEXT_DESC(*tx_ring, i); 5502 buffer_info = &tx_ring->buffer_info[i]; 5503 5504 context_desc->lower_setup.ip_fields.ipcss = ipcss; 5505 context_desc->lower_setup.ip_fields.ipcso = ipcso; 5506 context_desc->lower_setup.ip_fields.ipcse = cpu_to_le16(ipcse); 5507 context_desc->upper_setup.tcp_fields.tucss = tucss; 5508 context_desc->upper_setup.tcp_fields.tucso = tucso; 5509 context_desc->upper_setup.tcp_fields.tucse = 0; 5510 context_desc->tcp_seg_setup.fields.mss = cpu_to_le16(mss); 5511 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len; 5512 context_desc->cmd_and_length = cpu_to_le32(cmd_length); 5513 5514 buffer_info->time_stamp = jiffies; 5515 buffer_info->next_to_watch = i; 5516 5517 i++; 5518 if (i == tx_ring->count) 5519 i = 0; 5520 tx_ring->next_to_use = i; 5521 5522 return 1; 5523 } 5524 5525 static bool e1000_tx_csum(struct e1000_ring *tx_ring, struct sk_buff *skb, 5526 __be16 protocol) 5527 { 5528 struct e1000_adapter *adapter = tx_ring->adapter; 5529 struct e1000_context_desc *context_desc; 5530 struct e1000_buffer *buffer_info; 5531 unsigned int i; 5532 u8 css; 5533 u32 cmd_len = E1000_TXD_CMD_DEXT; 5534 5535 if (skb->ip_summed != CHECKSUM_PARTIAL) 5536 return false; 5537 5538 switch (protocol) { 5539 case cpu_to_be16(ETH_P_IP): 5540 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 5541 cmd_len |= E1000_TXD_CMD_TCP; 5542 break; 5543 case cpu_to_be16(ETH_P_IPV6): 5544 /* XXX not handling all IPV6 headers */ 5545 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 5546 cmd_len |= E1000_TXD_CMD_TCP; 5547 break; 5548 default: 5549 if (unlikely(net_ratelimit())) 5550 e_warn("checksum_partial proto=%x!\n", 5551 be16_to_cpu(protocol)); 5552 break; 5553 } 5554 5555 css = skb_checksum_start_offset(skb); 5556 5557 i = tx_ring->next_to_use; 5558 buffer_info = &tx_ring->buffer_info[i]; 5559 context_desc = E1000_CONTEXT_DESC(*tx_ring, i); 5560 5561 context_desc->lower_setup.ip_config = 0; 5562 context_desc->upper_setup.tcp_fields.tucss = css; 5563 context_desc->upper_setup.tcp_fields.tucso = css + skb->csum_offset; 5564 context_desc->upper_setup.tcp_fields.tucse = 0; 5565 context_desc->tcp_seg_setup.data = 0; 5566 context_desc->cmd_and_length = cpu_to_le32(cmd_len); 5567 5568 buffer_info->time_stamp = jiffies; 5569 buffer_info->next_to_watch = i; 5570 5571 i++; 5572 if (i == tx_ring->count) 5573 i = 0; 5574 tx_ring->next_to_use = i; 5575 5576 return true; 5577 } 5578 5579 static int e1000_tx_map(struct e1000_ring *tx_ring, struct sk_buff *skb, 5580 unsigned int first, unsigned int max_per_txd, 5581 unsigned int nr_frags) 5582 { 5583 struct e1000_adapter *adapter = tx_ring->adapter; 5584 struct pci_dev *pdev = adapter->pdev; 5585 struct e1000_buffer *buffer_info; 5586 unsigned int len = skb_headlen(skb); 5587 unsigned int offset = 0, size, count = 0, i; 5588 unsigned int f, bytecount, segs; 5589 5590 i = tx_ring->next_to_use; 5591 5592 while (len) { 5593 buffer_info = &tx_ring->buffer_info[i]; 5594 size = min(len, max_per_txd); 5595 5596 buffer_info->length = size; 5597 buffer_info->time_stamp = jiffies; 5598 buffer_info->next_to_watch = i; 5599 buffer_info->dma = dma_map_single(&pdev->dev, 5600 skb->data + offset, 5601 size, DMA_TO_DEVICE); 5602 buffer_info->mapped_as_page = false; 5603 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) 5604 goto dma_error; 5605 5606 len -= size; 5607 offset += size; 5608 count++; 5609 5610 if (len) { 5611 i++; 5612 if (i == tx_ring->count) 5613 i = 0; 5614 } 5615 } 5616 5617 for (f = 0; f < nr_frags; f++) { 5618 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 5619 5620 len = skb_frag_size(frag); 5621 offset = 0; 5622 5623 while (len) { 5624 i++; 5625 if (i == tx_ring->count) 5626 i = 0; 5627 5628 buffer_info = &tx_ring->buffer_info[i]; 5629 size = min(len, max_per_txd); 5630 5631 buffer_info->length = size; 5632 buffer_info->time_stamp = jiffies; 5633 buffer_info->next_to_watch = i; 5634 buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag, 5635 offset, size, 5636 DMA_TO_DEVICE); 5637 buffer_info->mapped_as_page = true; 5638 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) 5639 goto dma_error; 5640 5641 len -= size; 5642 offset += size; 5643 count++; 5644 } 5645 } 5646 5647 segs = skb_shinfo(skb)->gso_segs ? : 1; 5648 /* multiply data chunks by size of headers */ 5649 bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len; 5650 5651 tx_ring->buffer_info[i].skb = skb; 5652 tx_ring->buffer_info[i].segs = segs; 5653 tx_ring->buffer_info[i].bytecount = bytecount; 5654 tx_ring->buffer_info[first].next_to_watch = i; 5655 5656 return count; 5657 5658 dma_error: 5659 dev_err(&pdev->dev, "Tx DMA map failed\n"); 5660 buffer_info->dma = 0; 5661 if (count) 5662 count--; 5663 5664 while (count--) { 5665 if (i == 0) 5666 i += tx_ring->count; 5667 i--; 5668 buffer_info = &tx_ring->buffer_info[i]; 5669 e1000_put_txbuf(tx_ring, buffer_info, true); 5670 } 5671 5672 return 0; 5673 } 5674 5675 static void e1000_tx_queue(struct e1000_ring *tx_ring, int tx_flags, int count) 5676 { 5677 struct e1000_adapter *adapter = tx_ring->adapter; 5678 struct e1000_tx_desc *tx_desc = NULL; 5679 struct e1000_buffer *buffer_info; 5680 u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS; 5681 unsigned int i; 5682 5683 if (tx_flags & E1000_TX_FLAGS_TSO) { 5684 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D | 5685 E1000_TXD_CMD_TSE; 5686 txd_upper |= E1000_TXD_POPTS_TXSM << 8; 5687 5688 if (tx_flags & E1000_TX_FLAGS_IPV4) 5689 txd_upper |= E1000_TXD_POPTS_IXSM << 8; 5690 } 5691 5692 if (tx_flags & E1000_TX_FLAGS_CSUM) { 5693 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 5694 txd_upper |= E1000_TXD_POPTS_TXSM << 8; 5695 } 5696 5697 if (tx_flags & E1000_TX_FLAGS_VLAN) { 5698 txd_lower |= E1000_TXD_CMD_VLE; 5699 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK); 5700 } 5701 5702 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS)) 5703 txd_lower &= ~(E1000_TXD_CMD_IFCS); 5704 5705 if (unlikely(tx_flags & E1000_TX_FLAGS_HWTSTAMP)) { 5706 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 5707 txd_upper |= E1000_TXD_EXTCMD_TSTAMP; 5708 } 5709 5710 i = tx_ring->next_to_use; 5711 5712 do { 5713 buffer_info = &tx_ring->buffer_info[i]; 5714 tx_desc = E1000_TX_DESC(*tx_ring, i); 5715 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma); 5716 tx_desc->lower.data = cpu_to_le32(txd_lower | 5717 buffer_info->length); 5718 tx_desc->upper.data = cpu_to_le32(txd_upper); 5719 5720 i++; 5721 if (i == tx_ring->count) 5722 i = 0; 5723 } while (--count > 0); 5724 5725 tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd); 5726 5727 /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */ 5728 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS)) 5729 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS)); 5730 5731 /* Force memory writes to complete before letting h/w 5732 * know there are new descriptors to fetch. (Only 5733 * applicable for weak-ordered memory model archs, 5734 * such as IA-64). 5735 */ 5736 wmb(); 5737 5738 tx_ring->next_to_use = i; 5739 } 5740 5741 #define MINIMUM_DHCP_PACKET_SIZE 282 5742 static int e1000_transfer_dhcp_info(struct e1000_adapter *adapter, 5743 struct sk_buff *skb) 5744 { 5745 struct e1000_hw *hw = &adapter->hw; 5746 u16 length, offset; 5747 5748 if (skb_vlan_tag_present(skb) && 5749 !((skb_vlan_tag_get(skb) == adapter->hw.mng_cookie.vlan_id) && 5750 (adapter->hw.mng_cookie.status & 5751 E1000_MNG_DHCP_COOKIE_STATUS_VLAN))) 5752 return 0; 5753 5754 if (skb->len <= MINIMUM_DHCP_PACKET_SIZE) 5755 return 0; 5756 5757 if (((struct ethhdr *)skb->data)->h_proto != htons(ETH_P_IP)) 5758 return 0; 5759 5760 { 5761 const struct iphdr *ip = (struct iphdr *)((u8 *)skb->data + 14); 5762 struct udphdr *udp; 5763 5764 if (ip->protocol != IPPROTO_UDP) 5765 return 0; 5766 5767 udp = (struct udphdr *)((u8 *)ip + (ip->ihl << 2)); 5768 if (ntohs(udp->dest) != 67) 5769 return 0; 5770 5771 offset = (u8 *)udp + 8 - skb->data; 5772 length = skb->len - offset; 5773 return e1000e_mng_write_dhcp_info(hw, (u8 *)udp + 8, length); 5774 } 5775 5776 return 0; 5777 } 5778 5779 static int __e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size) 5780 { 5781 struct e1000_adapter *adapter = tx_ring->adapter; 5782 5783 netif_stop_queue(adapter->netdev); 5784 /* Herbert's original patch had: 5785 * smp_mb__after_netif_stop_queue(); 5786 * but since that doesn't exist yet, just open code it. 5787 */ 5788 smp_mb(); 5789 5790 /* We need to check again in a case another CPU has just 5791 * made room available. 5792 */ 5793 if (e1000_desc_unused(tx_ring) < size) 5794 return -EBUSY; 5795 5796 /* A reprieve! */ 5797 netif_start_queue(adapter->netdev); 5798 ++adapter->restart_queue; 5799 return 0; 5800 } 5801 5802 static int e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size) 5803 { 5804 BUG_ON(size > tx_ring->count); 5805 5806 if (e1000_desc_unused(tx_ring) >= size) 5807 return 0; 5808 return __e1000_maybe_stop_tx(tx_ring, size); 5809 } 5810 5811 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, 5812 struct net_device *netdev) 5813 { 5814 struct e1000_adapter *adapter = netdev_priv(netdev); 5815 struct e1000_ring *tx_ring = adapter->tx_ring; 5816 unsigned int first; 5817 unsigned int tx_flags = 0; 5818 unsigned int len = skb_headlen(skb); 5819 unsigned int nr_frags; 5820 unsigned int mss; 5821 int count = 0; 5822 int tso; 5823 unsigned int f; 5824 __be16 protocol = vlan_get_protocol(skb); 5825 5826 if (test_bit(__E1000_DOWN, &adapter->state)) { 5827 dev_kfree_skb_any(skb); 5828 return NETDEV_TX_OK; 5829 } 5830 5831 if (skb->len <= 0) { 5832 dev_kfree_skb_any(skb); 5833 return NETDEV_TX_OK; 5834 } 5835 5836 /* The minimum packet size with TCTL.PSP set is 17 bytes so 5837 * pad skb in order to meet this minimum size requirement 5838 */ 5839 if (skb_put_padto(skb, 17)) 5840 return NETDEV_TX_OK; 5841 5842 mss = skb_shinfo(skb)->gso_size; 5843 if (mss) { 5844 u8 hdr_len; 5845 5846 /* TSO Workaround for 82571/2/3 Controllers -- if skb->data 5847 * points to just header, pull a few bytes of payload from 5848 * frags into skb->data 5849 */ 5850 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 5851 /* we do this workaround for ES2LAN, but it is un-necessary, 5852 * avoiding it could save a lot of cycles 5853 */ 5854 if (skb->data_len && (hdr_len == len)) { 5855 unsigned int pull_size; 5856 5857 pull_size = min_t(unsigned int, 4, skb->data_len); 5858 if (!__pskb_pull_tail(skb, pull_size)) { 5859 e_err("__pskb_pull_tail failed.\n"); 5860 dev_kfree_skb_any(skb); 5861 return NETDEV_TX_OK; 5862 } 5863 len = skb_headlen(skb); 5864 } 5865 } 5866 5867 /* reserve a descriptor for the offload context */ 5868 if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL)) 5869 count++; 5870 count++; 5871 5872 count += DIV_ROUND_UP(len, adapter->tx_fifo_limit); 5873 5874 nr_frags = skb_shinfo(skb)->nr_frags; 5875 for (f = 0; f < nr_frags; f++) 5876 count += DIV_ROUND_UP(skb_frag_size(&skb_shinfo(skb)->frags[f]), 5877 adapter->tx_fifo_limit); 5878 5879 if (adapter->hw.mac.tx_pkt_filtering) 5880 e1000_transfer_dhcp_info(adapter, skb); 5881 5882 /* need: count + 2 desc gap to keep tail from touching 5883 * head, otherwise try next time 5884 */ 5885 if (e1000_maybe_stop_tx(tx_ring, count + 2)) 5886 return NETDEV_TX_BUSY; 5887 5888 if (skb_vlan_tag_present(skb)) { 5889 tx_flags |= E1000_TX_FLAGS_VLAN; 5890 tx_flags |= (skb_vlan_tag_get(skb) << 5891 E1000_TX_FLAGS_VLAN_SHIFT); 5892 } 5893 5894 first = tx_ring->next_to_use; 5895 5896 tso = e1000_tso(tx_ring, skb, protocol); 5897 if (tso < 0) { 5898 dev_kfree_skb_any(skb); 5899 return NETDEV_TX_OK; 5900 } 5901 5902 if (tso) 5903 tx_flags |= E1000_TX_FLAGS_TSO; 5904 else if (e1000_tx_csum(tx_ring, skb, protocol)) 5905 tx_flags |= E1000_TX_FLAGS_CSUM; 5906 5907 /* Old method was to assume IPv4 packet by default if TSO was enabled. 5908 * 82571 hardware supports TSO capabilities for IPv6 as well... 5909 * no longer assume, we must. 5910 */ 5911 if (protocol == htons(ETH_P_IP)) 5912 tx_flags |= E1000_TX_FLAGS_IPV4; 5913 5914 if (unlikely(skb->no_fcs)) 5915 tx_flags |= E1000_TX_FLAGS_NO_FCS; 5916 5917 /* if count is 0 then mapping error has occurred */ 5918 count = e1000_tx_map(tx_ring, skb, first, adapter->tx_fifo_limit, 5919 nr_frags); 5920 if (count) { 5921 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 5922 (adapter->flags & FLAG_HAS_HW_TIMESTAMP)) { 5923 if (!adapter->tx_hwtstamp_skb) { 5924 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 5925 tx_flags |= E1000_TX_FLAGS_HWTSTAMP; 5926 adapter->tx_hwtstamp_skb = skb_get(skb); 5927 adapter->tx_hwtstamp_start = jiffies; 5928 schedule_work(&adapter->tx_hwtstamp_work); 5929 } else { 5930 adapter->tx_hwtstamp_skipped++; 5931 } 5932 } 5933 5934 skb_tx_timestamp(skb); 5935 5936 netdev_sent_queue(netdev, skb->len); 5937 e1000_tx_queue(tx_ring, tx_flags, count); 5938 /* Make sure there is space in the ring for the next send. */ 5939 e1000_maybe_stop_tx(tx_ring, 5940 (MAX_SKB_FRAGS * 5941 DIV_ROUND_UP(PAGE_SIZE, 5942 adapter->tx_fifo_limit) + 2)); 5943 5944 if (!netdev_xmit_more() || 5945 netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) { 5946 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 5947 e1000e_update_tdt_wa(tx_ring, 5948 tx_ring->next_to_use); 5949 else 5950 writel(tx_ring->next_to_use, tx_ring->tail); 5951 } 5952 } else { 5953 dev_kfree_skb_any(skb); 5954 tx_ring->buffer_info[first].time_stamp = 0; 5955 tx_ring->next_to_use = first; 5956 } 5957 5958 return NETDEV_TX_OK; 5959 } 5960 5961 /** 5962 * e1000_tx_timeout - Respond to a Tx Hang 5963 * @netdev: network interface device structure 5964 * @txqueue: index of the hung queue (unused) 5965 **/ 5966 static void e1000_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue) 5967 { 5968 struct e1000_adapter *adapter = netdev_priv(netdev); 5969 5970 /* Do the reset outside of interrupt context */ 5971 adapter->tx_timeout_count++; 5972 schedule_work(&adapter->reset_task); 5973 } 5974 5975 static void e1000_reset_task(struct work_struct *work) 5976 { 5977 struct e1000_adapter *adapter; 5978 adapter = container_of(work, struct e1000_adapter, reset_task); 5979 5980 rtnl_lock(); 5981 /* don't run the task if already down */ 5982 if (test_bit(__E1000_DOWN, &adapter->state)) { 5983 rtnl_unlock(); 5984 return; 5985 } 5986 5987 if (!(adapter->flags & FLAG_RESTART_NOW)) { 5988 e1000e_dump(adapter); 5989 e_err("Reset adapter unexpectedly\n"); 5990 } 5991 e1000e_reinit_locked(adapter); 5992 rtnl_unlock(); 5993 } 5994 5995 /** 5996 * e1000e_get_stats64 - Get System Network Statistics 5997 * @netdev: network interface device structure 5998 * @stats: rtnl_link_stats64 pointer 5999 * 6000 * Returns the address of the device statistics structure. 6001 **/ 6002 void e1000e_get_stats64(struct net_device *netdev, 6003 struct rtnl_link_stats64 *stats) 6004 { 6005 struct e1000_adapter *adapter = netdev_priv(netdev); 6006 6007 spin_lock(&adapter->stats64_lock); 6008 e1000e_update_stats(adapter); 6009 /* Fill out the OS statistics structure */ 6010 stats->rx_bytes = adapter->stats.gorc; 6011 stats->rx_packets = adapter->stats.gprc; 6012 stats->tx_bytes = adapter->stats.gotc; 6013 stats->tx_packets = adapter->stats.gptc; 6014 stats->multicast = adapter->stats.mprc; 6015 stats->collisions = adapter->stats.colc; 6016 6017 /* Rx Errors */ 6018 6019 /* RLEC on some newer hardware can be incorrect so build 6020 * our own version based on RUC and ROC 6021 */ 6022 stats->rx_errors = adapter->stats.rxerrc + 6023 adapter->stats.crcerrs + adapter->stats.algnerrc + 6024 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr; 6025 stats->rx_length_errors = adapter->stats.ruc + adapter->stats.roc; 6026 stats->rx_crc_errors = adapter->stats.crcerrs; 6027 stats->rx_frame_errors = adapter->stats.algnerrc; 6028 stats->rx_missed_errors = adapter->stats.mpc; 6029 6030 /* Tx Errors */ 6031 stats->tx_errors = adapter->stats.ecol + adapter->stats.latecol; 6032 stats->tx_aborted_errors = adapter->stats.ecol; 6033 stats->tx_window_errors = adapter->stats.latecol; 6034 stats->tx_carrier_errors = adapter->stats.tncrs; 6035 6036 /* Tx Dropped needs to be maintained elsewhere */ 6037 6038 spin_unlock(&adapter->stats64_lock); 6039 } 6040 6041 /** 6042 * e1000_change_mtu - Change the Maximum Transfer Unit 6043 * @netdev: network interface device structure 6044 * @new_mtu: new value for maximum frame size 6045 * 6046 * Returns 0 on success, negative on failure 6047 **/ 6048 static int e1000_change_mtu(struct net_device *netdev, int new_mtu) 6049 { 6050 struct e1000_adapter *adapter = netdev_priv(netdev); 6051 int max_frame = new_mtu + VLAN_ETH_HLEN + ETH_FCS_LEN; 6052 6053 /* Jumbo frame support */ 6054 if ((new_mtu > ETH_DATA_LEN) && 6055 !(adapter->flags & FLAG_HAS_JUMBO_FRAMES)) { 6056 e_err("Jumbo Frames not supported.\n"); 6057 return -EINVAL; 6058 } 6059 6060 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */ 6061 if ((adapter->hw.mac.type >= e1000_pch2lan) && 6062 !(adapter->flags2 & FLAG2_CRC_STRIPPING) && 6063 (new_mtu > ETH_DATA_LEN)) { 6064 e_err("Jumbo Frames not supported on this device when CRC stripping is disabled.\n"); 6065 return -EINVAL; 6066 } 6067 6068 while (test_and_set_bit(__E1000_RESETTING, &adapter->state)) 6069 usleep_range(1000, 1100); 6070 /* e1000e_down -> e1000e_reset dependent on max_frame_size & mtu */ 6071 adapter->max_frame_size = max_frame; 6072 netdev_dbg(netdev, "changing MTU from %d to %d\n", 6073 netdev->mtu, new_mtu); 6074 netdev->mtu = new_mtu; 6075 6076 pm_runtime_get_sync(netdev->dev.parent); 6077 6078 if (netif_running(netdev)) 6079 e1000e_down(adapter, true); 6080 6081 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN 6082 * means we reserve 2 more, this pushes us to allocate from the next 6083 * larger slab size. 6084 * i.e. RXBUFFER_2048 --> size-4096 slab 6085 * However with the new *_jumbo_rx* routines, jumbo receives will use 6086 * fragmented skbs 6087 */ 6088 6089 if (max_frame <= 2048) 6090 adapter->rx_buffer_len = 2048; 6091 else 6092 adapter->rx_buffer_len = 4096; 6093 6094 /* adjust allocation if LPE protects us, and we aren't using SBP */ 6095 if (max_frame <= (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)) 6096 adapter->rx_buffer_len = VLAN_ETH_FRAME_LEN + ETH_FCS_LEN; 6097 6098 if (netif_running(netdev)) 6099 e1000e_up(adapter); 6100 else 6101 e1000e_reset(adapter); 6102 6103 pm_runtime_put_sync(netdev->dev.parent); 6104 6105 clear_bit(__E1000_RESETTING, &adapter->state); 6106 6107 return 0; 6108 } 6109 6110 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, 6111 int cmd) 6112 { 6113 struct e1000_adapter *adapter = netdev_priv(netdev); 6114 struct mii_ioctl_data *data = if_mii(ifr); 6115 6116 if (adapter->hw.phy.media_type != e1000_media_type_copper) 6117 return -EOPNOTSUPP; 6118 6119 switch (cmd) { 6120 case SIOCGMIIPHY: 6121 data->phy_id = adapter->hw.phy.addr; 6122 break; 6123 case SIOCGMIIREG: 6124 e1000_phy_read_status(adapter); 6125 6126 switch (data->reg_num & 0x1F) { 6127 case MII_BMCR: 6128 data->val_out = adapter->phy_regs.bmcr; 6129 break; 6130 case MII_BMSR: 6131 data->val_out = adapter->phy_regs.bmsr; 6132 break; 6133 case MII_PHYSID1: 6134 data->val_out = (adapter->hw.phy.id >> 16); 6135 break; 6136 case MII_PHYSID2: 6137 data->val_out = (adapter->hw.phy.id & 0xFFFF); 6138 break; 6139 case MII_ADVERTISE: 6140 data->val_out = adapter->phy_regs.advertise; 6141 break; 6142 case MII_LPA: 6143 data->val_out = adapter->phy_regs.lpa; 6144 break; 6145 case MII_EXPANSION: 6146 data->val_out = adapter->phy_regs.expansion; 6147 break; 6148 case MII_CTRL1000: 6149 data->val_out = adapter->phy_regs.ctrl1000; 6150 break; 6151 case MII_STAT1000: 6152 data->val_out = adapter->phy_regs.stat1000; 6153 break; 6154 case MII_ESTATUS: 6155 data->val_out = adapter->phy_regs.estatus; 6156 break; 6157 default: 6158 return -EIO; 6159 } 6160 break; 6161 case SIOCSMIIREG: 6162 default: 6163 return -EOPNOTSUPP; 6164 } 6165 return 0; 6166 } 6167 6168 /** 6169 * e1000e_hwtstamp_set - control hardware time stamping 6170 * @netdev: network interface device structure 6171 * @ifr: interface request 6172 * 6173 * Outgoing time stamping can be enabled and disabled. Play nice and 6174 * disable it when requested, although it shouldn't cause any overhead 6175 * when no packet needs it. At most one packet in the queue may be 6176 * marked for time stamping, otherwise it would be impossible to tell 6177 * for sure to which packet the hardware time stamp belongs. 6178 * 6179 * Incoming time stamping has to be configured via the hardware filters. 6180 * Not all combinations are supported, in particular event type has to be 6181 * specified. Matching the kind of event packet is not supported, with the 6182 * exception of "all V2 events regardless of level 2 or 4". 6183 **/ 6184 static int e1000e_hwtstamp_set(struct net_device *netdev, struct ifreq *ifr) 6185 { 6186 struct e1000_adapter *adapter = netdev_priv(netdev); 6187 struct hwtstamp_config config; 6188 int ret_val; 6189 6190 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 6191 return -EFAULT; 6192 6193 ret_val = e1000e_config_hwtstamp(adapter, &config); 6194 if (ret_val) 6195 return ret_val; 6196 6197 switch (config.rx_filter) { 6198 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 6199 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 6200 case HWTSTAMP_FILTER_PTP_V2_SYNC: 6201 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 6202 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 6203 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 6204 /* With V2 type filters which specify a Sync or Delay Request, 6205 * Path Delay Request/Response messages are also time stamped 6206 * by hardware so notify the caller the requested packets plus 6207 * some others are time stamped. 6208 */ 6209 config.rx_filter = HWTSTAMP_FILTER_SOME; 6210 break; 6211 default: 6212 break; 6213 } 6214 6215 return copy_to_user(ifr->ifr_data, &config, 6216 sizeof(config)) ? -EFAULT : 0; 6217 } 6218 6219 static int e1000e_hwtstamp_get(struct net_device *netdev, struct ifreq *ifr) 6220 { 6221 struct e1000_adapter *adapter = netdev_priv(netdev); 6222 6223 return copy_to_user(ifr->ifr_data, &adapter->hwtstamp_config, 6224 sizeof(adapter->hwtstamp_config)) ? -EFAULT : 0; 6225 } 6226 6227 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 6228 { 6229 switch (cmd) { 6230 case SIOCGMIIPHY: 6231 case SIOCGMIIREG: 6232 case SIOCSMIIREG: 6233 return e1000_mii_ioctl(netdev, ifr, cmd); 6234 case SIOCSHWTSTAMP: 6235 return e1000e_hwtstamp_set(netdev, ifr); 6236 case SIOCGHWTSTAMP: 6237 return e1000e_hwtstamp_get(netdev, ifr); 6238 default: 6239 return -EOPNOTSUPP; 6240 } 6241 } 6242 6243 static int e1000_init_phy_wakeup(struct e1000_adapter *adapter, u32 wufc) 6244 { 6245 struct e1000_hw *hw = &adapter->hw; 6246 u32 i, mac_reg, wuc; 6247 u16 phy_reg, wuc_enable; 6248 int retval; 6249 6250 /* copy MAC RARs to PHY RARs */ 6251 e1000_copy_rx_addrs_to_phy_ich8lan(hw); 6252 6253 retval = hw->phy.ops.acquire(hw); 6254 if (retval) { 6255 e_err("Could not acquire PHY\n"); 6256 return retval; 6257 } 6258 6259 /* Enable access to wakeup registers on and set page to BM_WUC_PAGE */ 6260 retval = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 6261 if (retval) 6262 goto release; 6263 6264 /* copy MAC MTA to PHY MTA - only needed for pchlan */ 6265 for (i = 0; i < adapter->hw.mac.mta_reg_count; i++) { 6266 mac_reg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 6267 hw->phy.ops.write_reg_page(hw, BM_MTA(i), 6268 (u16)(mac_reg & 0xFFFF)); 6269 hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1, 6270 (u16)((mac_reg >> 16) & 0xFFFF)); 6271 } 6272 6273 /* configure PHY Rx Control register */ 6274 hw->phy.ops.read_reg_page(&adapter->hw, BM_RCTL, &phy_reg); 6275 mac_reg = er32(RCTL); 6276 if (mac_reg & E1000_RCTL_UPE) 6277 phy_reg |= BM_RCTL_UPE; 6278 if (mac_reg & E1000_RCTL_MPE) 6279 phy_reg |= BM_RCTL_MPE; 6280 phy_reg &= ~(BM_RCTL_MO_MASK); 6281 if (mac_reg & E1000_RCTL_MO_3) 6282 phy_reg |= (((mac_reg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) 6283 << BM_RCTL_MO_SHIFT); 6284 if (mac_reg & E1000_RCTL_BAM) 6285 phy_reg |= BM_RCTL_BAM; 6286 if (mac_reg & E1000_RCTL_PMCF) 6287 phy_reg |= BM_RCTL_PMCF; 6288 mac_reg = er32(CTRL); 6289 if (mac_reg & E1000_CTRL_RFCE) 6290 phy_reg |= BM_RCTL_RFCE; 6291 hw->phy.ops.write_reg_page(&adapter->hw, BM_RCTL, phy_reg); 6292 6293 wuc = E1000_WUC_PME_EN; 6294 if (wufc & (E1000_WUFC_MAG | E1000_WUFC_LNKC)) 6295 wuc |= E1000_WUC_APME; 6296 6297 /* enable PHY wakeup in MAC register */ 6298 ew32(WUFC, wufc); 6299 ew32(WUC, (E1000_WUC_PHY_WAKE | E1000_WUC_APMPME | 6300 E1000_WUC_PME_STATUS | wuc)); 6301 6302 /* configure and enable PHY wakeup in PHY registers */ 6303 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUFC, wufc); 6304 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUC, wuc); 6305 6306 /* activate PHY wakeup */ 6307 wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 6308 retval = e1000_disable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 6309 if (retval) 6310 e_err("Could not set PHY Host Wakeup bit\n"); 6311 release: 6312 hw->phy.ops.release(hw); 6313 6314 return retval; 6315 } 6316 6317 static void e1000e_flush_lpic(struct pci_dev *pdev) 6318 { 6319 struct net_device *netdev = pci_get_drvdata(pdev); 6320 struct e1000_adapter *adapter = netdev_priv(netdev); 6321 struct e1000_hw *hw = &adapter->hw; 6322 u32 ret_val; 6323 6324 pm_runtime_get_sync(netdev->dev.parent); 6325 6326 ret_val = hw->phy.ops.acquire(hw); 6327 if (ret_val) 6328 goto fl_out; 6329 6330 pr_info("EEE TX LPI TIMER: %08X\n", 6331 er32(LPIC) >> E1000_LPIC_LPIET_SHIFT); 6332 6333 hw->phy.ops.release(hw); 6334 6335 fl_out: 6336 pm_runtime_put_sync(netdev->dev.parent); 6337 } 6338 6339 /* S0ix implementation */ 6340 static void e1000e_s0ix_entry_flow(struct e1000_adapter *adapter) 6341 { 6342 struct e1000_hw *hw = &adapter->hw; 6343 u32 mac_data; 6344 u16 phy_data; 6345 6346 /* Disable the periodic inband message, 6347 * don't request PCIe clock in K1 page770_17[10:9] = 10b 6348 */ 6349 e1e_rphy(hw, HV_PM_CTRL, &phy_data); 6350 phy_data &= ~HV_PM_CTRL_K1_CLK_REQ; 6351 phy_data |= BIT(10); 6352 e1e_wphy(hw, HV_PM_CTRL, phy_data); 6353 6354 /* Make sure we don't exit K1 every time a new packet arrives 6355 * 772_29[5] = 1 CS_Mode_Stay_In_K1 6356 */ 6357 e1e_rphy(hw, I217_CGFREG, &phy_data); 6358 phy_data |= BIT(5); 6359 e1e_wphy(hw, I217_CGFREG, phy_data); 6360 6361 /* Change the MAC/PHY interface to SMBus 6362 * Force the SMBus in PHY page769_23[0] = 1 6363 * Force the SMBus in MAC CTRL_EXT[11] = 1 6364 */ 6365 e1e_rphy(hw, CV_SMB_CTRL, &phy_data); 6366 phy_data |= CV_SMB_CTRL_FORCE_SMBUS; 6367 e1e_wphy(hw, CV_SMB_CTRL, phy_data); 6368 mac_data = er32(CTRL_EXT); 6369 mac_data |= E1000_CTRL_EXT_FORCE_SMBUS; 6370 ew32(CTRL_EXT, mac_data); 6371 6372 /* DFT control: PHY bit: page769_20[0] = 1 6373 * Gate PPW via EXTCNF_CTRL - set 0x0F00[7] = 1 6374 */ 6375 e1e_rphy(hw, I82579_DFT_CTRL, &phy_data); 6376 phy_data |= BIT(0); 6377 e1e_wphy(hw, I82579_DFT_CTRL, phy_data); 6378 6379 mac_data = er32(EXTCNF_CTRL); 6380 mac_data |= E1000_EXTCNF_CTRL_GATE_PHY_CFG; 6381 ew32(EXTCNF_CTRL, mac_data); 6382 6383 /* Check MAC Tx/Rx packet buffer pointers. 6384 * Reset MAC Tx/Rx packet buffer pointers to suppress any 6385 * pending traffic indication that would prevent power gating. 6386 */ 6387 mac_data = er32(TDFH); 6388 if (mac_data) 6389 ew32(TDFH, 0); 6390 mac_data = er32(TDFT); 6391 if (mac_data) 6392 ew32(TDFT, 0); 6393 mac_data = er32(TDFHS); 6394 if (mac_data) 6395 ew32(TDFHS, 0); 6396 mac_data = er32(TDFTS); 6397 if (mac_data) 6398 ew32(TDFTS, 0); 6399 mac_data = er32(TDFPC); 6400 if (mac_data) 6401 ew32(TDFPC, 0); 6402 mac_data = er32(RDFH); 6403 if (mac_data) 6404 ew32(RDFH, 0); 6405 mac_data = er32(RDFT); 6406 if (mac_data) 6407 ew32(RDFT, 0); 6408 mac_data = er32(RDFHS); 6409 if (mac_data) 6410 ew32(RDFHS, 0); 6411 mac_data = er32(RDFTS); 6412 if (mac_data) 6413 ew32(RDFTS, 0); 6414 mac_data = er32(RDFPC); 6415 if (mac_data) 6416 ew32(RDFPC, 0); 6417 6418 /* Enable the Dynamic Power Gating in the MAC */ 6419 mac_data = er32(FEXTNVM7); 6420 mac_data |= BIT(22); 6421 ew32(FEXTNVM7, mac_data); 6422 6423 /* Disable the time synchronization clock */ 6424 mac_data = er32(FEXTNVM7); 6425 mac_data |= BIT(31); 6426 mac_data &= ~BIT(0); 6427 ew32(FEXTNVM7, mac_data); 6428 6429 /* Dynamic Power Gating Enable */ 6430 mac_data = er32(CTRL_EXT); 6431 mac_data |= BIT(3); 6432 ew32(CTRL_EXT, mac_data); 6433 6434 /* Disable disconnected cable conditioning for Power Gating */ 6435 mac_data = er32(DPGFR); 6436 mac_data |= BIT(2); 6437 ew32(DPGFR, mac_data); 6438 6439 /* Don't wake from dynamic Power Gating with clock request */ 6440 mac_data = er32(FEXTNVM12); 6441 mac_data |= BIT(12); 6442 ew32(FEXTNVM12, mac_data); 6443 6444 /* Ungate PGCB clock */ 6445 mac_data = er32(FEXTNVM9); 6446 mac_data &= ~BIT(28); 6447 ew32(FEXTNVM9, mac_data); 6448 6449 /* Enable K1 off to enable mPHY Power Gating */ 6450 mac_data = er32(FEXTNVM6); 6451 mac_data |= BIT(31); 6452 ew32(FEXTNVM6, mac_data); 6453 6454 /* Enable mPHY power gating for any link and speed */ 6455 mac_data = er32(FEXTNVM8); 6456 mac_data |= BIT(9); 6457 ew32(FEXTNVM8, mac_data); 6458 6459 /* Enable the Dynamic Clock Gating in the DMA and MAC */ 6460 mac_data = er32(CTRL_EXT); 6461 mac_data |= E1000_CTRL_EXT_DMA_DYN_CLK_EN; 6462 ew32(CTRL_EXT, mac_data); 6463 6464 /* No MAC DPG gating SLP_S0 in modern standby 6465 * Switch the logic of the lanphypc to use PMC counter 6466 */ 6467 mac_data = er32(FEXTNVM5); 6468 mac_data |= BIT(7); 6469 ew32(FEXTNVM5, mac_data); 6470 } 6471 6472 static void e1000e_s0ix_exit_flow(struct e1000_adapter *adapter) 6473 { 6474 struct e1000_hw *hw = &adapter->hw; 6475 u32 mac_data; 6476 u16 phy_data; 6477 6478 /* Disable the Dynamic Power Gating in the MAC */ 6479 mac_data = er32(FEXTNVM7); 6480 mac_data &= 0xFFBFFFFF; 6481 ew32(FEXTNVM7, mac_data); 6482 6483 /* Enable the time synchronization clock */ 6484 mac_data = er32(FEXTNVM7); 6485 mac_data |= BIT(0); 6486 ew32(FEXTNVM7, mac_data); 6487 6488 /* Disable mPHY power gating for any link and speed */ 6489 mac_data = er32(FEXTNVM8); 6490 mac_data &= ~BIT(9); 6491 ew32(FEXTNVM8, mac_data); 6492 6493 /* Disable K1 off */ 6494 mac_data = er32(FEXTNVM6); 6495 mac_data &= ~BIT(31); 6496 ew32(FEXTNVM6, mac_data); 6497 6498 /* Disable Ungate PGCB clock */ 6499 mac_data = er32(FEXTNVM9); 6500 mac_data |= BIT(28); 6501 ew32(FEXTNVM9, mac_data); 6502 6503 /* Cancel not waking from dynamic 6504 * Power Gating with clock request 6505 */ 6506 mac_data = er32(FEXTNVM12); 6507 mac_data &= ~BIT(12); 6508 ew32(FEXTNVM12, mac_data); 6509 6510 /* Cancel disable disconnected cable conditioning 6511 * for Power Gating 6512 */ 6513 mac_data = er32(DPGFR); 6514 mac_data &= ~BIT(2); 6515 ew32(DPGFR, mac_data); 6516 6517 /* Disable Dynamic Power Gating */ 6518 mac_data = er32(CTRL_EXT); 6519 mac_data &= 0xFFFFFFF7; 6520 ew32(CTRL_EXT, mac_data); 6521 6522 /* Disable the Dynamic Clock Gating in the DMA and MAC */ 6523 mac_data = er32(CTRL_EXT); 6524 mac_data &= 0xFFF7FFFF; 6525 ew32(CTRL_EXT, mac_data); 6526 6527 /* Revert the lanphypc logic to use the internal Gbe counter 6528 * and not the PMC counter 6529 */ 6530 mac_data = er32(FEXTNVM5); 6531 mac_data &= 0xFFFFFF7F; 6532 ew32(FEXTNVM5, mac_data); 6533 6534 /* Enable the periodic inband message, 6535 * Request PCIe clock in K1 page770_17[10:9] =01b 6536 */ 6537 e1e_rphy(hw, HV_PM_CTRL, &phy_data); 6538 phy_data &= 0xFBFF; 6539 phy_data |= HV_PM_CTRL_K1_CLK_REQ; 6540 e1e_wphy(hw, HV_PM_CTRL, phy_data); 6541 6542 /* Return back configuration 6543 * 772_29[5] = 0 CS_Mode_Stay_In_K1 6544 */ 6545 e1e_rphy(hw, I217_CGFREG, &phy_data); 6546 phy_data &= 0xFFDF; 6547 e1e_wphy(hw, I217_CGFREG, phy_data); 6548 6549 /* Change the MAC/PHY interface to Kumeran 6550 * Unforce the SMBus in PHY page769_23[0] = 0 6551 * Unforce the SMBus in MAC CTRL_EXT[11] = 0 6552 */ 6553 e1e_rphy(hw, CV_SMB_CTRL, &phy_data); 6554 phy_data &= ~CV_SMB_CTRL_FORCE_SMBUS; 6555 e1e_wphy(hw, CV_SMB_CTRL, phy_data); 6556 mac_data = er32(CTRL_EXT); 6557 mac_data &= ~E1000_CTRL_EXT_FORCE_SMBUS; 6558 ew32(CTRL_EXT, mac_data); 6559 } 6560 6561 static int e1000e_pm_freeze(struct device *dev) 6562 { 6563 struct net_device *netdev = dev_get_drvdata(dev); 6564 struct e1000_adapter *adapter = netdev_priv(netdev); 6565 bool present; 6566 6567 rtnl_lock(); 6568 6569 present = netif_device_present(netdev); 6570 netif_device_detach(netdev); 6571 6572 if (present && netif_running(netdev)) { 6573 int count = E1000_CHECK_RESET_COUNT; 6574 6575 while (test_bit(__E1000_RESETTING, &adapter->state) && count--) 6576 usleep_range(10000, 11000); 6577 6578 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state)); 6579 6580 /* Quiesce the device without resetting the hardware */ 6581 e1000e_down(adapter, false); 6582 e1000_free_irq(adapter); 6583 } 6584 rtnl_unlock(); 6585 6586 e1000e_reset_interrupt_capability(adapter); 6587 6588 /* Allow time for pending master requests to run */ 6589 e1000e_disable_pcie_master(&adapter->hw); 6590 6591 return 0; 6592 } 6593 6594 static int __e1000_shutdown(struct pci_dev *pdev, bool runtime) 6595 { 6596 struct net_device *netdev = pci_get_drvdata(pdev); 6597 struct e1000_adapter *adapter = netdev_priv(netdev); 6598 struct e1000_hw *hw = &adapter->hw; 6599 u32 ctrl, ctrl_ext, rctl, status, wufc; 6600 int retval = 0; 6601 6602 /* Runtime suspend should only enable wakeup for link changes */ 6603 if (runtime) 6604 wufc = E1000_WUFC_LNKC; 6605 else if (device_may_wakeup(&pdev->dev)) 6606 wufc = adapter->wol; 6607 else 6608 wufc = 0; 6609 6610 status = er32(STATUS); 6611 if (status & E1000_STATUS_LU) 6612 wufc &= ~E1000_WUFC_LNKC; 6613 6614 if (wufc) { 6615 e1000_setup_rctl(adapter); 6616 e1000e_set_rx_mode(netdev); 6617 6618 /* turn on all-multi mode if wake on multicast is enabled */ 6619 if (wufc & E1000_WUFC_MC) { 6620 rctl = er32(RCTL); 6621 rctl |= E1000_RCTL_MPE; 6622 ew32(RCTL, rctl); 6623 } 6624 6625 ctrl = er32(CTRL); 6626 ctrl |= E1000_CTRL_ADVD3WUC; 6627 if (!(adapter->flags2 & FLAG2_HAS_PHY_WAKEUP)) 6628 ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT; 6629 ew32(CTRL, ctrl); 6630 6631 if (adapter->hw.phy.media_type == e1000_media_type_fiber || 6632 adapter->hw.phy.media_type == 6633 e1000_media_type_internal_serdes) { 6634 /* keep the laser running in D3 */ 6635 ctrl_ext = er32(CTRL_EXT); 6636 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 6637 ew32(CTRL_EXT, ctrl_ext); 6638 } 6639 6640 if (!runtime) 6641 e1000e_power_up_phy(adapter); 6642 6643 if (adapter->flags & FLAG_IS_ICH) 6644 e1000_suspend_workarounds_ich8lan(&adapter->hw); 6645 6646 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) { 6647 /* enable wakeup by the PHY */ 6648 retval = e1000_init_phy_wakeup(adapter, wufc); 6649 if (retval) 6650 return retval; 6651 } else { 6652 /* enable wakeup by the MAC */ 6653 ew32(WUFC, wufc); 6654 ew32(WUC, E1000_WUC_PME_EN); 6655 } 6656 } else { 6657 ew32(WUC, 0); 6658 ew32(WUFC, 0); 6659 6660 e1000_power_down_phy(adapter); 6661 } 6662 6663 if (adapter->hw.phy.type == e1000_phy_igp_3) { 6664 e1000e_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw); 6665 } else if (hw->mac.type >= e1000_pch_lpt) { 6666 if (wufc && !(wufc & (E1000_WUFC_EX | E1000_WUFC_MC | E1000_WUFC_BC))) 6667 /* ULP does not support wake from unicast, multicast 6668 * or broadcast. 6669 */ 6670 retval = e1000_enable_ulp_lpt_lp(hw, !runtime); 6671 6672 if (retval) 6673 return retval; 6674 } 6675 6676 /* Ensure that the appropriate bits are set in LPI_CTRL 6677 * for EEE in Sx 6678 */ 6679 if ((hw->phy.type >= e1000_phy_i217) && 6680 adapter->eee_advert && hw->dev_spec.ich8lan.eee_lp_ability) { 6681 u16 lpi_ctrl = 0; 6682 6683 retval = hw->phy.ops.acquire(hw); 6684 if (!retval) { 6685 retval = e1e_rphy_locked(hw, I82579_LPI_CTRL, 6686 &lpi_ctrl); 6687 if (!retval) { 6688 if (adapter->eee_advert & 6689 hw->dev_spec.ich8lan.eee_lp_ability & 6690 I82579_EEE_100_SUPPORTED) 6691 lpi_ctrl |= I82579_LPI_CTRL_100_ENABLE; 6692 if (adapter->eee_advert & 6693 hw->dev_spec.ich8lan.eee_lp_ability & 6694 I82579_EEE_1000_SUPPORTED) 6695 lpi_ctrl |= I82579_LPI_CTRL_1000_ENABLE; 6696 6697 retval = e1e_wphy_locked(hw, I82579_LPI_CTRL, 6698 lpi_ctrl); 6699 } 6700 } 6701 hw->phy.ops.release(hw); 6702 } 6703 6704 /* Release control of h/w to f/w. If f/w is AMT enabled, this 6705 * would have already happened in close and is redundant. 6706 */ 6707 e1000e_release_hw_control(adapter); 6708 6709 pci_clear_master(pdev); 6710 6711 /* The pci-e switch on some quad port adapters will report a 6712 * correctable error when the MAC transitions from D0 to D3. To 6713 * prevent this we need to mask off the correctable errors on the 6714 * downstream port of the pci-e switch. 6715 * 6716 * We don't have the associated upstream bridge while assigning 6717 * the PCI device into guest. For example, the KVM on power is 6718 * one of the cases. 6719 */ 6720 if (adapter->flags & FLAG_IS_QUAD_PORT) { 6721 struct pci_dev *us_dev = pdev->bus->self; 6722 u16 devctl; 6723 6724 if (!us_dev) 6725 return 0; 6726 6727 pcie_capability_read_word(us_dev, PCI_EXP_DEVCTL, &devctl); 6728 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, 6729 (devctl & ~PCI_EXP_DEVCTL_CERE)); 6730 6731 pci_save_state(pdev); 6732 pci_prepare_to_sleep(pdev); 6733 6734 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, devctl); 6735 } 6736 6737 return 0; 6738 } 6739 6740 /** 6741 * __e1000e_disable_aspm - Disable ASPM states 6742 * @pdev: pointer to PCI device struct 6743 * @state: bit-mask of ASPM states to disable 6744 * @locked: indication if this context holds pci_bus_sem locked. 6745 * 6746 * Some devices *must* have certain ASPM states disabled per hardware errata. 6747 **/ 6748 static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state, int locked) 6749 { 6750 struct pci_dev *parent = pdev->bus->self; 6751 u16 aspm_dis_mask = 0; 6752 u16 pdev_aspmc, parent_aspmc; 6753 6754 switch (state) { 6755 case PCIE_LINK_STATE_L0S: 6756 case PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1: 6757 aspm_dis_mask |= PCI_EXP_LNKCTL_ASPM_L0S; 6758 fallthrough; /* can't have L1 without L0s */ 6759 case PCIE_LINK_STATE_L1: 6760 aspm_dis_mask |= PCI_EXP_LNKCTL_ASPM_L1; 6761 break; 6762 default: 6763 return; 6764 } 6765 6766 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &pdev_aspmc); 6767 pdev_aspmc &= PCI_EXP_LNKCTL_ASPMC; 6768 6769 if (parent) { 6770 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, 6771 &parent_aspmc); 6772 parent_aspmc &= PCI_EXP_LNKCTL_ASPMC; 6773 } 6774 6775 /* Nothing to do if the ASPM states to be disabled already are */ 6776 if (!(pdev_aspmc & aspm_dis_mask) && 6777 (!parent || !(parent_aspmc & aspm_dis_mask))) 6778 return; 6779 6780 dev_info(&pdev->dev, "Disabling ASPM %s %s\n", 6781 (aspm_dis_mask & pdev_aspmc & PCI_EXP_LNKCTL_ASPM_L0S) ? 6782 "L0s" : "", 6783 (aspm_dis_mask & pdev_aspmc & PCI_EXP_LNKCTL_ASPM_L1) ? 6784 "L1" : ""); 6785 6786 #ifdef CONFIG_PCIEASPM 6787 if (locked) 6788 pci_disable_link_state_locked(pdev, state); 6789 else 6790 pci_disable_link_state(pdev, state); 6791 6792 /* Double-check ASPM control. If not disabled by the above, the 6793 * BIOS is preventing that from happening (or CONFIG_PCIEASPM is 6794 * not enabled); override by writing PCI config space directly. 6795 */ 6796 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &pdev_aspmc); 6797 pdev_aspmc &= PCI_EXP_LNKCTL_ASPMC; 6798 6799 if (!(aspm_dis_mask & pdev_aspmc)) 6800 return; 6801 #endif 6802 6803 /* Both device and parent should have the same ASPM setting. 6804 * Disable ASPM in downstream component first and then upstream. 6805 */ 6806 pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, aspm_dis_mask); 6807 6808 if (parent) 6809 pcie_capability_clear_word(parent, PCI_EXP_LNKCTL, 6810 aspm_dis_mask); 6811 } 6812 6813 /** 6814 * e1000e_disable_aspm - Disable ASPM states. 6815 * @pdev: pointer to PCI device struct 6816 * @state: bit-mask of ASPM states to disable 6817 * 6818 * This function acquires the pci_bus_sem! 6819 * Some devices *must* have certain ASPM states disabled per hardware errata. 6820 **/ 6821 static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state) 6822 { 6823 __e1000e_disable_aspm(pdev, state, 0); 6824 } 6825 6826 /** 6827 * e1000e_disable_aspm_locked - Disable ASPM states. 6828 * @pdev: pointer to PCI device struct 6829 * @state: bit-mask of ASPM states to disable 6830 * 6831 * This function must be called with pci_bus_sem acquired! 6832 * Some devices *must* have certain ASPM states disabled per hardware errata. 6833 **/ 6834 static void e1000e_disable_aspm_locked(struct pci_dev *pdev, u16 state) 6835 { 6836 __e1000e_disable_aspm(pdev, state, 1); 6837 } 6838 6839 static int e1000e_pm_thaw(struct device *dev) 6840 { 6841 struct net_device *netdev = dev_get_drvdata(dev); 6842 struct e1000_adapter *adapter = netdev_priv(netdev); 6843 int rc = 0; 6844 6845 e1000e_set_interrupt_capability(adapter); 6846 6847 rtnl_lock(); 6848 if (netif_running(netdev)) { 6849 rc = e1000_request_irq(adapter); 6850 if (rc) 6851 goto err_irq; 6852 6853 e1000e_up(adapter); 6854 } 6855 6856 netif_device_attach(netdev); 6857 err_irq: 6858 rtnl_unlock(); 6859 6860 return rc; 6861 } 6862 6863 static int __e1000_resume(struct pci_dev *pdev) 6864 { 6865 struct net_device *netdev = pci_get_drvdata(pdev); 6866 struct e1000_adapter *adapter = netdev_priv(netdev); 6867 struct e1000_hw *hw = &adapter->hw; 6868 u16 aspm_disable_flag = 0; 6869 6870 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S) 6871 aspm_disable_flag = PCIE_LINK_STATE_L0S; 6872 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1) 6873 aspm_disable_flag |= PCIE_LINK_STATE_L1; 6874 if (aspm_disable_flag) 6875 e1000e_disable_aspm(pdev, aspm_disable_flag); 6876 6877 pci_set_master(pdev); 6878 6879 if (hw->mac.type >= e1000_pch2lan) 6880 e1000_resume_workarounds_pchlan(&adapter->hw); 6881 6882 e1000e_power_up_phy(adapter); 6883 6884 /* report the system wakeup cause from S3/S4 */ 6885 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) { 6886 u16 phy_data; 6887 6888 e1e_rphy(&adapter->hw, BM_WUS, &phy_data); 6889 if (phy_data) { 6890 e_info("PHY Wakeup cause - %s\n", 6891 phy_data & E1000_WUS_EX ? "Unicast Packet" : 6892 phy_data & E1000_WUS_MC ? "Multicast Packet" : 6893 phy_data & E1000_WUS_BC ? "Broadcast Packet" : 6894 phy_data & E1000_WUS_MAG ? "Magic Packet" : 6895 phy_data & E1000_WUS_LNKC ? 6896 "Link Status Change" : "other"); 6897 } 6898 e1e_wphy(&adapter->hw, BM_WUS, ~0); 6899 } else { 6900 u32 wus = er32(WUS); 6901 6902 if (wus) { 6903 e_info("MAC Wakeup cause - %s\n", 6904 wus & E1000_WUS_EX ? "Unicast Packet" : 6905 wus & E1000_WUS_MC ? "Multicast Packet" : 6906 wus & E1000_WUS_BC ? "Broadcast Packet" : 6907 wus & E1000_WUS_MAG ? "Magic Packet" : 6908 wus & E1000_WUS_LNKC ? "Link Status Change" : 6909 "other"); 6910 } 6911 ew32(WUS, ~0); 6912 } 6913 6914 e1000e_reset(adapter); 6915 6916 e1000_init_manageability_pt(adapter); 6917 6918 /* If the controller has AMT, do not set DRV_LOAD until the interface 6919 * is up. For all other cases, let the f/w know that the h/w is now 6920 * under the control of the driver. 6921 */ 6922 if (!(adapter->flags & FLAG_HAS_AMT)) 6923 e1000e_get_hw_control(adapter); 6924 6925 return 0; 6926 } 6927 6928 static __maybe_unused int e1000e_pm_prepare(struct device *dev) 6929 { 6930 return pm_runtime_suspended(dev) && 6931 pm_suspend_via_firmware(); 6932 } 6933 6934 static __maybe_unused int e1000e_pm_suspend(struct device *dev) 6935 { 6936 struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev)); 6937 struct e1000_adapter *adapter = netdev_priv(netdev); 6938 struct pci_dev *pdev = to_pci_dev(dev); 6939 int rc; 6940 6941 e1000e_flush_lpic(pdev); 6942 6943 e1000e_pm_freeze(dev); 6944 6945 rc = __e1000_shutdown(pdev, false); 6946 if (rc) { 6947 e1000e_pm_thaw(dev); 6948 } else { 6949 /* Introduce S0ix implementation */ 6950 if (adapter->flags2 & FLAG2_ENABLE_S0IX_FLOWS) 6951 e1000e_s0ix_entry_flow(adapter); 6952 } 6953 6954 return rc; 6955 } 6956 6957 static __maybe_unused int e1000e_pm_resume(struct device *dev) 6958 { 6959 struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev)); 6960 struct e1000_adapter *adapter = netdev_priv(netdev); 6961 struct pci_dev *pdev = to_pci_dev(dev); 6962 int rc; 6963 6964 /* Introduce S0ix implementation */ 6965 if (adapter->flags2 & FLAG2_ENABLE_S0IX_FLOWS) 6966 e1000e_s0ix_exit_flow(adapter); 6967 6968 rc = __e1000_resume(pdev); 6969 if (rc) 6970 return rc; 6971 6972 return e1000e_pm_thaw(dev); 6973 } 6974 6975 static __maybe_unused int e1000e_pm_runtime_idle(struct device *dev) 6976 { 6977 struct net_device *netdev = dev_get_drvdata(dev); 6978 struct e1000_adapter *adapter = netdev_priv(netdev); 6979 u16 eee_lp; 6980 6981 eee_lp = adapter->hw.dev_spec.ich8lan.eee_lp_ability; 6982 6983 if (!e1000e_has_link(adapter)) { 6984 adapter->hw.dev_spec.ich8lan.eee_lp_ability = eee_lp; 6985 pm_schedule_suspend(dev, 5 * MSEC_PER_SEC); 6986 } 6987 6988 return -EBUSY; 6989 } 6990 6991 static __maybe_unused int e1000e_pm_runtime_resume(struct device *dev) 6992 { 6993 struct pci_dev *pdev = to_pci_dev(dev); 6994 struct net_device *netdev = pci_get_drvdata(pdev); 6995 struct e1000_adapter *adapter = netdev_priv(netdev); 6996 int rc; 6997 6998 rc = __e1000_resume(pdev); 6999 if (rc) 7000 return rc; 7001 7002 if (netdev->flags & IFF_UP) 7003 e1000e_up(adapter); 7004 7005 return rc; 7006 } 7007 7008 static __maybe_unused int e1000e_pm_runtime_suspend(struct device *dev) 7009 { 7010 struct pci_dev *pdev = to_pci_dev(dev); 7011 struct net_device *netdev = pci_get_drvdata(pdev); 7012 struct e1000_adapter *adapter = netdev_priv(netdev); 7013 7014 if (netdev->flags & IFF_UP) { 7015 int count = E1000_CHECK_RESET_COUNT; 7016 7017 while (test_bit(__E1000_RESETTING, &adapter->state) && count--) 7018 usleep_range(10000, 11000); 7019 7020 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state)); 7021 7022 /* Down the device without resetting the hardware */ 7023 e1000e_down(adapter, false); 7024 } 7025 7026 if (__e1000_shutdown(pdev, true)) { 7027 e1000e_pm_runtime_resume(dev); 7028 return -EBUSY; 7029 } 7030 7031 return 0; 7032 } 7033 7034 static void e1000_shutdown(struct pci_dev *pdev) 7035 { 7036 e1000e_flush_lpic(pdev); 7037 7038 e1000e_pm_freeze(&pdev->dev); 7039 7040 __e1000_shutdown(pdev, false); 7041 } 7042 7043 #ifdef CONFIG_NET_POLL_CONTROLLER 7044 7045 static irqreturn_t e1000_intr_msix(int __always_unused irq, void *data) 7046 { 7047 struct net_device *netdev = data; 7048 struct e1000_adapter *adapter = netdev_priv(netdev); 7049 7050 if (adapter->msix_entries) { 7051 int vector, msix_irq; 7052 7053 vector = 0; 7054 msix_irq = adapter->msix_entries[vector].vector; 7055 if (disable_hardirq(msix_irq)) 7056 e1000_intr_msix_rx(msix_irq, netdev); 7057 enable_irq(msix_irq); 7058 7059 vector++; 7060 msix_irq = adapter->msix_entries[vector].vector; 7061 if (disable_hardirq(msix_irq)) 7062 e1000_intr_msix_tx(msix_irq, netdev); 7063 enable_irq(msix_irq); 7064 7065 vector++; 7066 msix_irq = adapter->msix_entries[vector].vector; 7067 if (disable_hardirq(msix_irq)) 7068 e1000_msix_other(msix_irq, netdev); 7069 enable_irq(msix_irq); 7070 } 7071 7072 return IRQ_HANDLED; 7073 } 7074 7075 /** 7076 * e1000_netpoll 7077 * @netdev: network interface device structure 7078 * 7079 * Polling 'interrupt' - used by things like netconsole to send skbs 7080 * without having to re-enable interrupts. It's not called while 7081 * the interrupt routine is executing. 7082 */ 7083 static void e1000_netpoll(struct net_device *netdev) 7084 { 7085 struct e1000_adapter *adapter = netdev_priv(netdev); 7086 7087 switch (adapter->int_mode) { 7088 case E1000E_INT_MODE_MSIX: 7089 e1000_intr_msix(adapter->pdev->irq, netdev); 7090 break; 7091 case E1000E_INT_MODE_MSI: 7092 if (disable_hardirq(adapter->pdev->irq)) 7093 e1000_intr_msi(adapter->pdev->irq, netdev); 7094 enable_irq(adapter->pdev->irq); 7095 break; 7096 default: /* E1000E_INT_MODE_LEGACY */ 7097 if (disable_hardirq(adapter->pdev->irq)) 7098 e1000_intr(adapter->pdev->irq, netdev); 7099 enable_irq(adapter->pdev->irq); 7100 break; 7101 } 7102 } 7103 #endif 7104 7105 /** 7106 * e1000_io_error_detected - called when PCI error is detected 7107 * @pdev: Pointer to PCI device 7108 * @state: The current pci connection state 7109 * 7110 * This function is called after a PCI bus error affecting 7111 * this device has been detected. 7112 */ 7113 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev, 7114 pci_channel_state_t state) 7115 { 7116 e1000e_pm_freeze(&pdev->dev); 7117 7118 if (state == pci_channel_io_perm_failure) 7119 return PCI_ERS_RESULT_DISCONNECT; 7120 7121 pci_disable_device(pdev); 7122 7123 /* Request a slot reset. */ 7124 return PCI_ERS_RESULT_NEED_RESET; 7125 } 7126 7127 /** 7128 * e1000_io_slot_reset - called after the pci bus has been reset. 7129 * @pdev: Pointer to PCI device 7130 * 7131 * Restart the card from scratch, as if from a cold-boot. Implementation 7132 * resembles the first-half of the e1000e_pm_resume routine. 7133 */ 7134 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev) 7135 { 7136 struct net_device *netdev = pci_get_drvdata(pdev); 7137 struct e1000_adapter *adapter = netdev_priv(netdev); 7138 struct e1000_hw *hw = &adapter->hw; 7139 u16 aspm_disable_flag = 0; 7140 int err; 7141 pci_ers_result_t result; 7142 7143 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S) 7144 aspm_disable_flag = PCIE_LINK_STATE_L0S; 7145 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1) 7146 aspm_disable_flag |= PCIE_LINK_STATE_L1; 7147 if (aspm_disable_flag) 7148 e1000e_disable_aspm_locked(pdev, aspm_disable_flag); 7149 7150 err = pci_enable_device_mem(pdev); 7151 if (err) { 7152 dev_err(&pdev->dev, 7153 "Cannot re-enable PCI device after reset.\n"); 7154 result = PCI_ERS_RESULT_DISCONNECT; 7155 } else { 7156 pdev->state_saved = true; 7157 pci_restore_state(pdev); 7158 pci_set_master(pdev); 7159 7160 pci_enable_wake(pdev, PCI_D3hot, 0); 7161 pci_enable_wake(pdev, PCI_D3cold, 0); 7162 7163 e1000e_reset(adapter); 7164 ew32(WUS, ~0); 7165 result = PCI_ERS_RESULT_RECOVERED; 7166 } 7167 7168 return result; 7169 } 7170 7171 /** 7172 * e1000_io_resume - called when traffic can start flowing again. 7173 * @pdev: Pointer to PCI device 7174 * 7175 * This callback is called when the error recovery driver tells us that 7176 * its OK to resume normal operation. Implementation resembles the 7177 * second-half of the e1000e_pm_resume routine. 7178 */ 7179 static void e1000_io_resume(struct pci_dev *pdev) 7180 { 7181 struct net_device *netdev = pci_get_drvdata(pdev); 7182 struct e1000_adapter *adapter = netdev_priv(netdev); 7183 7184 e1000_init_manageability_pt(adapter); 7185 7186 e1000e_pm_thaw(&pdev->dev); 7187 7188 /* If the controller has AMT, do not set DRV_LOAD until the interface 7189 * is up. For all other cases, let the f/w know that the h/w is now 7190 * under the control of the driver. 7191 */ 7192 if (!(adapter->flags & FLAG_HAS_AMT)) 7193 e1000e_get_hw_control(adapter); 7194 } 7195 7196 static void e1000_print_device_info(struct e1000_adapter *adapter) 7197 { 7198 struct e1000_hw *hw = &adapter->hw; 7199 struct net_device *netdev = adapter->netdev; 7200 u32 ret_val; 7201 u8 pba_str[E1000_PBANUM_LENGTH]; 7202 7203 /* print bus type/speed/width info */ 7204 e_info("(PCI Express:2.5GT/s:%s) %pM\n", 7205 /* bus width */ 7206 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" : 7207 "Width x1"), 7208 /* MAC address */ 7209 netdev->dev_addr); 7210 e_info("Intel(R) PRO/%s Network Connection\n", 7211 (hw->phy.type == e1000_phy_ife) ? "10/100" : "1000"); 7212 ret_val = e1000_read_pba_string_generic(hw, pba_str, 7213 E1000_PBANUM_LENGTH); 7214 if (ret_val) 7215 strlcpy((char *)pba_str, "Unknown", sizeof(pba_str)); 7216 e_info("MAC: %d, PHY: %d, PBA No: %s\n", 7217 hw->mac.type, hw->phy.type, pba_str); 7218 } 7219 7220 static void e1000_eeprom_checks(struct e1000_adapter *adapter) 7221 { 7222 struct e1000_hw *hw = &adapter->hw; 7223 int ret_val; 7224 u16 buf = 0; 7225 7226 if (hw->mac.type != e1000_82573) 7227 return; 7228 7229 ret_val = e1000_read_nvm(hw, NVM_INIT_CONTROL2_REG, 1, &buf); 7230 le16_to_cpus(&buf); 7231 if (!ret_val && (!(buf & BIT(0)))) { 7232 /* Deep Smart Power Down (DSPD) */ 7233 dev_warn(&adapter->pdev->dev, 7234 "Warning: detected DSPD enabled in EEPROM\n"); 7235 } 7236 } 7237 7238 static netdev_features_t e1000_fix_features(struct net_device *netdev, 7239 netdev_features_t features) 7240 { 7241 struct e1000_adapter *adapter = netdev_priv(netdev); 7242 struct e1000_hw *hw = &adapter->hw; 7243 7244 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */ 7245 if ((hw->mac.type >= e1000_pch2lan) && (netdev->mtu > ETH_DATA_LEN)) 7246 features &= ~NETIF_F_RXFCS; 7247 7248 /* Since there is no support for separate Rx/Tx vlan accel 7249 * enable/disable make sure Tx flag is always in same state as Rx. 7250 */ 7251 if (features & NETIF_F_HW_VLAN_CTAG_RX) 7252 features |= NETIF_F_HW_VLAN_CTAG_TX; 7253 else 7254 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 7255 7256 return features; 7257 } 7258 7259 static int e1000_set_features(struct net_device *netdev, 7260 netdev_features_t features) 7261 { 7262 struct e1000_adapter *adapter = netdev_priv(netdev); 7263 netdev_features_t changed = features ^ netdev->features; 7264 7265 if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) 7266 adapter->flags |= FLAG_TSO_FORCE; 7267 7268 if (!(changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX | 7269 NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_RXFCS | 7270 NETIF_F_RXALL))) 7271 return 0; 7272 7273 if (changed & NETIF_F_RXFCS) { 7274 if (features & NETIF_F_RXFCS) { 7275 adapter->flags2 &= ~FLAG2_CRC_STRIPPING; 7276 } else { 7277 /* We need to take it back to defaults, which might mean 7278 * stripping is still disabled at the adapter level. 7279 */ 7280 if (adapter->flags2 & FLAG2_DFLT_CRC_STRIPPING) 7281 adapter->flags2 |= FLAG2_CRC_STRIPPING; 7282 else 7283 adapter->flags2 &= ~FLAG2_CRC_STRIPPING; 7284 } 7285 } 7286 7287 netdev->features = features; 7288 7289 if (netif_running(netdev)) 7290 e1000e_reinit_locked(adapter); 7291 else 7292 e1000e_reset(adapter); 7293 7294 return 1; 7295 } 7296 7297 static const struct net_device_ops e1000e_netdev_ops = { 7298 .ndo_open = e1000e_open, 7299 .ndo_stop = e1000e_close, 7300 .ndo_start_xmit = e1000_xmit_frame, 7301 .ndo_get_stats64 = e1000e_get_stats64, 7302 .ndo_set_rx_mode = e1000e_set_rx_mode, 7303 .ndo_set_mac_address = e1000_set_mac, 7304 .ndo_change_mtu = e1000_change_mtu, 7305 .ndo_do_ioctl = e1000_ioctl, 7306 .ndo_tx_timeout = e1000_tx_timeout, 7307 .ndo_validate_addr = eth_validate_addr, 7308 7309 .ndo_vlan_rx_add_vid = e1000_vlan_rx_add_vid, 7310 .ndo_vlan_rx_kill_vid = e1000_vlan_rx_kill_vid, 7311 #ifdef CONFIG_NET_POLL_CONTROLLER 7312 .ndo_poll_controller = e1000_netpoll, 7313 #endif 7314 .ndo_set_features = e1000_set_features, 7315 .ndo_fix_features = e1000_fix_features, 7316 .ndo_features_check = passthru_features_check, 7317 }; 7318 7319 /** 7320 * e1000_probe - Device Initialization Routine 7321 * @pdev: PCI device information struct 7322 * @ent: entry in e1000_pci_tbl 7323 * 7324 * Returns 0 on success, negative on failure 7325 * 7326 * e1000_probe initializes an adapter identified by a pci_dev structure. 7327 * The OS initialization, configuring of the adapter private structure, 7328 * and a hardware reset occur. 7329 **/ 7330 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 7331 { 7332 struct net_device *netdev; 7333 struct e1000_adapter *adapter; 7334 struct e1000_hw *hw; 7335 const struct e1000_info *ei = e1000_info_tbl[ent->driver_data]; 7336 resource_size_t mmio_start, mmio_len; 7337 resource_size_t flash_start, flash_len; 7338 static int cards_found; 7339 u16 aspm_disable_flag = 0; 7340 int bars, i, err, pci_using_dac; 7341 u16 eeprom_data = 0; 7342 u16 eeprom_apme_mask = E1000_EEPROM_APME; 7343 s32 ret_val = 0; 7344 7345 if (ei->flags2 & FLAG2_DISABLE_ASPM_L0S) 7346 aspm_disable_flag = PCIE_LINK_STATE_L0S; 7347 if (ei->flags2 & FLAG2_DISABLE_ASPM_L1) 7348 aspm_disable_flag |= PCIE_LINK_STATE_L1; 7349 if (aspm_disable_flag) 7350 e1000e_disable_aspm(pdev, aspm_disable_flag); 7351 7352 err = pci_enable_device_mem(pdev); 7353 if (err) 7354 return err; 7355 7356 pci_using_dac = 0; 7357 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 7358 if (!err) { 7359 pci_using_dac = 1; 7360 } else { 7361 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 7362 if (err) { 7363 dev_err(&pdev->dev, 7364 "No usable DMA configuration, aborting\n"); 7365 goto err_dma; 7366 } 7367 } 7368 7369 bars = pci_select_bars(pdev, IORESOURCE_MEM); 7370 err = pci_request_selected_regions_exclusive(pdev, bars, 7371 e1000e_driver_name); 7372 if (err) 7373 goto err_pci_reg; 7374 7375 /* AER (Advanced Error Reporting) hooks */ 7376 pci_enable_pcie_error_reporting(pdev); 7377 7378 pci_set_master(pdev); 7379 /* PCI config space info */ 7380 err = pci_save_state(pdev); 7381 if (err) 7382 goto err_alloc_etherdev; 7383 7384 err = -ENOMEM; 7385 netdev = alloc_etherdev(sizeof(struct e1000_adapter)); 7386 if (!netdev) 7387 goto err_alloc_etherdev; 7388 7389 SET_NETDEV_DEV(netdev, &pdev->dev); 7390 7391 netdev->irq = pdev->irq; 7392 7393 pci_set_drvdata(pdev, netdev); 7394 adapter = netdev_priv(netdev); 7395 hw = &adapter->hw; 7396 adapter->netdev = netdev; 7397 adapter->pdev = pdev; 7398 adapter->ei = ei; 7399 adapter->pba = ei->pba; 7400 adapter->flags = ei->flags; 7401 adapter->flags2 = ei->flags2; 7402 adapter->hw.adapter = adapter; 7403 adapter->hw.mac.type = ei->mac; 7404 adapter->max_hw_frame_size = ei->max_hw_frame_size; 7405 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 7406 7407 mmio_start = pci_resource_start(pdev, 0); 7408 mmio_len = pci_resource_len(pdev, 0); 7409 7410 err = -EIO; 7411 adapter->hw.hw_addr = ioremap(mmio_start, mmio_len); 7412 if (!adapter->hw.hw_addr) 7413 goto err_ioremap; 7414 7415 if ((adapter->flags & FLAG_HAS_FLASH) && 7416 (pci_resource_flags(pdev, 1) & IORESOURCE_MEM) && 7417 (hw->mac.type < e1000_pch_spt)) { 7418 flash_start = pci_resource_start(pdev, 1); 7419 flash_len = pci_resource_len(pdev, 1); 7420 adapter->hw.flash_address = ioremap(flash_start, flash_len); 7421 if (!adapter->hw.flash_address) 7422 goto err_flashmap; 7423 } 7424 7425 /* Set default EEE advertisement */ 7426 if (adapter->flags2 & FLAG2_HAS_EEE) 7427 adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T; 7428 7429 /* construct the net_device struct */ 7430 netdev->netdev_ops = &e1000e_netdev_ops; 7431 e1000e_set_ethtool_ops(netdev); 7432 netdev->watchdog_timeo = 5 * HZ; 7433 netif_napi_add(netdev, &adapter->napi, e1000e_poll, 64); 7434 strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name)); 7435 7436 netdev->mem_start = mmio_start; 7437 netdev->mem_end = mmio_start + mmio_len; 7438 7439 adapter->bd_number = cards_found++; 7440 7441 e1000e_check_options(adapter); 7442 7443 /* setup adapter struct */ 7444 err = e1000_sw_init(adapter); 7445 if (err) 7446 goto err_sw_init; 7447 7448 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 7449 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops)); 7450 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 7451 7452 err = ei->get_variants(adapter); 7453 if (err) 7454 goto err_hw_init; 7455 7456 if ((adapter->flags & FLAG_IS_ICH) && 7457 (adapter->flags & FLAG_READ_ONLY_NVM) && 7458 (hw->mac.type < e1000_pch_spt)) 7459 e1000e_write_protect_nvm_ich8lan(&adapter->hw); 7460 7461 hw->mac.ops.get_bus_info(&adapter->hw); 7462 7463 adapter->hw.phy.autoneg_wait_to_complete = 0; 7464 7465 /* Copper options */ 7466 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 7467 adapter->hw.phy.mdix = AUTO_ALL_MODES; 7468 adapter->hw.phy.disable_polarity_correction = 0; 7469 adapter->hw.phy.ms_type = e1000_ms_hw_default; 7470 } 7471 7472 if (hw->phy.ops.check_reset_block && hw->phy.ops.check_reset_block(hw)) 7473 dev_info(&pdev->dev, 7474 "PHY reset is blocked due to SOL/IDER session.\n"); 7475 7476 /* Set initial default active device features */ 7477 netdev->features = (NETIF_F_SG | 7478 NETIF_F_HW_VLAN_CTAG_RX | 7479 NETIF_F_HW_VLAN_CTAG_TX | 7480 NETIF_F_TSO | 7481 NETIF_F_TSO6 | 7482 NETIF_F_RXHASH | 7483 NETIF_F_RXCSUM | 7484 NETIF_F_HW_CSUM); 7485 7486 /* Set user-changeable features (subset of all device features) */ 7487 netdev->hw_features = netdev->features; 7488 netdev->hw_features |= NETIF_F_RXFCS; 7489 netdev->priv_flags |= IFF_SUPP_NOFCS; 7490 netdev->hw_features |= NETIF_F_RXALL; 7491 7492 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) 7493 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7494 7495 netdev->vlan_features |= (NETIF_F_SG | 7496 NETIF_F_TSO | 7497 NETIF_F_TSO6 | 7498 NETIF_F_HW_CSUM); 7499 7500 netdev->priv_flags |= IFF_UNICAST_FLT; 7501 7502 if (pci_using_dac) { 7503 netdev->features |= NETIF_F_HIGHDMA; 7504 netdev->vlan_features |= NETIF_F_HIGHDMA; 7505 } 7506 7507 /* MTU range: 68 - max_hw_frame_size */ 7508 netdev->min_mtu = ETH_MIN_MTU; 7509 netdev->max_mtu = adapter->max_hw_frame_size - 7510 (VLAN_ETH_HLEN + ETH_FCS_LEN); 7511 7512 if (e1000e_enable_mng_pass_thru(&adapter->hw)) 7513 adapter->flags |= FLAG_MNG_PT_ENABLED; 7514 7515 /* before reading the NVM, reset the controller to 7516 * put the device in a known good starting state 7517 */ 7518 adapter->hw.mac.ops.reset_hw(&adapter->hw); 7519 7520 /* systems with ASPM and others may see the checksum fail on the first 7521 * attempt. Let's give it a few tries 7522 */ 7523 for (i = 0;; i++) { 7524 if (e1000_validate_nvm_checksum(&adapter->hw) >= 0) 7525 break; 7526 if (i == 2) { 7527 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 7528 err = -EIO; 7529 goto err_eeprom; 7530 } 7531 } 7532 7533 e1000_eeprom_checks(adapter); 7534 7535 /* copy the MAC address */ 7536 if (e1000e_read_mac_addr(&adapter->hw)) 7537 dev_err(&pdev->dev, 7538 "NVM Read Error while reading MAC address\n"); 7539 7540 memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len); 7541 7542 if (!is_valid_ether_addr(netdev->dev_addr)) { 7543 dev_err(&pdev->dev, "Invalid MAC Address: %pM\n", 7544 netdev->dev_addr); 7545 err = -EIO; 7546 goto err_eeprom; 7547 } 7548 7549 timer_setup(&adapter->watchdog_timer, e1000_watchdog, 0); 7550 timer_setup(&adapter->phy_info_timer, e1000_update_phy_info, 0); 7551 7552 INIT_WORK(&adapter->reset_task, e1000_reset_task); 7553 INIT_WORK(&adapter->watchdog_task, e1000_watchdog_task); 7554 INIT_WORK(&adapter->downshift_task, e1000e_downshift_workaround); 7555 INIT_WORK(&adapter->update_phy_task, e1000e_update_phy_task); 7556 INIT_WORK(&adapter->print_hang_task, e1000_print_hw_hang); 7557 7558 /* Initialize link parameters. User can change them with ethtool */ 7559 adapter->hw.mac.autoneg = 1; 7560 adapter->fc_autoneg = true; 7561 adapter->hw.fc.requested_mode = e1000_fc_default; 7562 adapter->hw.fc.current_mode = e1000_fc_default; 7563 adapter->hw.phy.autoneg_advertised = 0x2f; 7564 7565 /* Initial Wake on LAN setting - If APM wake is enabled in 7566 * the EEPROM, enable the ACPI Magic Packet filter 7567 */ 7568 if (adapter->flags & FLAG_APME_IN_WUC) { 7569 /* APME bit in EEPROM is mapped to WUC.APME */ 7570 eeprom_data = er32(WUC); 7571 eeprom_apme_mask = E1000_WUC_APME; 7572 if ((hw->mac.type > e1000_ich10lan) && 7573 (eeprom_data & E1000_WUC_PHY_WAKE)) 7574 adapter->flags2 |= FLAG2_HAS_PHY_WAKEUP; 7575 } else if (adapter->flags & FLAG_APME_IN_CTRL3) { 7576 if (adapter->flags & FLAG_APME_CHECK_PORT_B && 7577 (adapter->hw.bus.func == 1)) 7578 ret_val = e1000_read_nvm(&adapter->hw, 7579 NVM_INIT_CONTROL3_PORT_B, 7580 1, &eeprom_data); 7581 else 7582 ret_val = e1000_read_nvm(&adapter->hw, 7583 NVM_INIT_CONTROL3_PORT_A, 7584 1, &eeprom_data); 7585 } 7586 7587 /* fetch WoL from EEPROM */ 7588 if (ret_val) 7589 e_dbg("NVM read error getting WoL initial values: %d\n", ret_val); 7590 else if (eeprom_data & eeprom_apme_mask) 7591 adapter->eeprom_wol |= E1000_WUFC_MAG; 7592 7593 /* now that we have the eeprom settings, apply the special cases 7594 * where the eeprom may be wrong or the board simply won't support 7595 * wake on lan on a particular port 7596 */ 7597 if (!(adapter->flags & FLAG_HAS_WOL)) 7598 adapter->eeprom_wol = 0; 7599 7600 /* initialize the wol settings based on the eeprom settings */ 7601 adapter->wol = adapter->eeprom_wol; 7602 7603 /* make sure adapter isn't asleep if manageability is enabled */ 7604 if (adapter->wol || (adapter->flags & FLAG_MNG_PT_ENABLED) || 7605 (hw->mac.ops.check_mng_mode(hw))) 7606 device_wakeup_enable(&pdev->dev); 7607 7608 /* save off EEPROM version number */ 7609 ret_val = e1000_read_nvm(&adapter->hw, 5, 1, &adapter->eeprom_vers); 7610 7611 if (ret_val) { 7612 e_dbg("NVM read error getting EEPROM version: %d\n", ret_val); 7613 adapter->eeprom_vers = 0; 7614 } 7615 7616 /* init PTP hardware clock */ 7617 e1000e_ptp_init(adapter); 7618 7619 /* reset the hardware with the new settings */ 7620 e1000e_reset(adapter); 7621 7622 /* If the controller has AMT, do not set DRV_LOAD until the interface 7623 * is up. For all other cases, let the f/w know that the h/w is now 7624 * under the control of the driver. 7625 */ 7626 if (!(adapter->flags & FLAG_HAS_AMT)) 7627 e1000e_get_hw_control(adapter); 7628 7629 if (hw->mac.type >= e1000_pch_cnp) 7630 adapter->flags2 |= FLAG2_ENABLE_S0IX_FLOWS; 7631 7632 strlcpy(netdev->name, "eth%d", sizeof(netdev->name)); 7633 err = register_netdev(netdev); 7634 if (err) 7635 goto err_register; 7636 7637 /* carrier off reporting is important to ethtool even BEFORE open */ 7638 netif_carrier_off(netdev); 7639 7640 e1000_print_device_info(adapter); 7641 7642 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_SMART_PREPARE); 7643 7644 if (pci_dev_run_wake(pdev) && hw->mac.type != e1000_pch_cnp) 7645 pm_runtime_put_noidle(&pdev->dev); 7646 7647 return 0; 7648 7649 err_register: 7650 if (!(adapter->flags & FLAG_HAS_AMT)) 7651 e1000e_release_hw_control(adapter); 7652 err_eeprom: 7653 if (hw->phy.ops.check_reset_block && !hw->phy.ops.check_reset_block(hw)) 7654 e1000_phy_hw_reset(&adapter->hw); 7655 err_hw_init: 7656 kfree(adapter->tx_ring); 7657 kfree(adapter->rx_ring); 7658 err_sw_init: 7659 if ((adapter->hw.flash_address) && (hw->mac.type < e1000_pch_spt)) 7660 iounmap(adapter->hw.flash_address); 7661 e1000e_reset_interrupt_capability(adapter); 7662 err_flashmap: 7663 iounmap(adapter->hw.hw_addr); 7664 err_ioremap: 7665 free_netdev(netdev); 7666 err_alloc_etherdev: 7667 pci_release_mem_regions(pdev); 7668 err_pci_reg: 7669 err_dma: 7670 pci_disable_device(pdev); 7671 return err; 7672 } 7673 7674 /** 7675 * e1000_remove - Device Removal Routine 7676 * @pdev: PCI device information struct 7677 * 7678 * e1000_remove is called by the PCI subsystem to alert the driver 7679 * that it should release a PCI device. The could be caused by a 7680 * Hot-Plug event, or because the driver is going to be removed from 7681 * memory. 7682 **/ 7683 static void e1000_remove(struct pci_dev *pdev) 7684 { 7685 struct net_device *netdev = pci_get_drvdata(pdev); 7686 struct e1000_adapter *adapter = netdev_priv(netdev); 7687 7688 e1000e_ptp_remove(adapter); 7689 7690 /* The timers may be rescheduled, so explicitly disable them 7691 * from being rescheduled. 7692 */ 7693 set_bit(__E1000_DOWN, &adapter->state); 7694 del_timer_sync(&adapter->watchdog_timer); 7695 del_timer_sync(&adapter->phy_info_timer); 7696 7697 cancel_work_sync(&adapter->reset_task); 7698 cancel_work_sync(&adapter->watchdog_task); 7699 cancel_work_sync(&adapter->downshift_task); 7700 cancel_work_sync(&adapter->update_phy_task); 7701 cancel_work_sync(&adapter->print_hang_task); 7702 7703 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) { 7704 cancel_work_sync(&adapter->tx_hwtstamp_work); 7705 if (adapter->tx_hwtstamp_skb) { 7706 dev_consume_skb_any(adapter->tx_hwtstamp_skb); 7707 adapter->tx_hwtstamp_skb = NULL; 7708 } 7709 } 7710 7711 unregister_netdev(netdev); 7712 7713 if (pci_dev_run_wake(pdev)) 7714 pm_runtime_get_noresume(&pdev->dev); 7715 7716 /* Release control of h/w to f/w. If f/w is AMT enabled, this 7717 * would have already happened in close and is redundant. 7718 */ 7719 e1000e_release_hw_control(adapter); 7720 7721 e1000e_reset_interrupt_capability(adapter); 7722 kfree(adapter->tx_ring); 7723 kfree(adapter->rx_ring); 7724 7725 iounmap(adapter->hw.hw_addr); 7726 if ((adapter->hw.flash_address) && 7727 (adapter->hw.mac.type < e1000_pch_spt)) 7728 iounmap(adapter->hw.flash_address); 7729 pci_release_mem_regions(pdev); 7730 7731 free_netdev(netdev); 7732 7733 /* AER disable */ 7734 pci_disable_pcie_error_reporting(pdev); 7735 7736 pci_disable_device(pdev); 7737 } 7738 7739 /* PCI Error Recovery (ERS) */ 7740 static const struct pci_error_handlers e1000_err_handler = { 7741 .error_detected = e1000_io_error_detected, 7742 .slot_reset = e1000_io_slot_reset, 7743 .resume = e1000_io_resume, 7744 }; 7745 7746 static const struct pci_device_id e1000_pci_tbl[] = { 7747 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER), board_82571 }, 7748 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER), board_82571 }, 7749 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER), board_82571 }, 7750 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP), 7751 board_82571 }, 7752 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER), board_82571 }, 7753 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES), board_82571 }, 7754 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL), board_82571 }, 7755 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD), board_82571 }, 7756 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER), board_82571 }, 7757 7758 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI), board_82572 }, 7759 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER), board_82572 }, 7760 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER), board_82572 }, 7761 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES), board_82572 }, 7762 7763 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E), board_82573 }, 7764 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT), board_82573 }, 7765 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L), board_82573 }, 7766 7767 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L), board_82574 }, 7768 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA), board_82574 }, 7769 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V), board_82583 }, 7770 7771 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT), 7772 board_80003es2lan }, 7773 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT), 7774 board_80003es2lan }, 7775 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT), 7776 board_80003es2lan }, 7777 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT), 7778 board_80003es2lan }, 7779 7780 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE), board_ich8lan }, 7781 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G), board_ich8lan }, 7782 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT), board_ich8lan }, 7783 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT), board_ich8lan }, 7784 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C), board_ich8lan }, 7785 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M), board_ich8lan }, 7786 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT), board_ich8lan }, 7787 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3), board_ich8lan }, 7788 7789 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE), board_ich9lan }, 7790 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G), board_ich9lan }, 7791 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT), board_ich9lan }, 7792 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT), board_ich9lan }, 7793 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C), board_ich9lan }, 7794 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM), board_ich9lan }, 7795 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M), board_ich9lan }, 7796 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT), board_ich9lan }, 7797 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V), board_ich9lan }, 7798 7799 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM), board_ich9lan }, 7800 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF), board_ich9lan }, 7801 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V), board_ich9lan }, 7802 7803 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM), board_ich10lan }, 7804 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF), board_ich10lan }, 7805 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V), board_ich10lan }, 7806 7807 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM), board_pchlan }, 7808 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC), board_pchlan }, 7809 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM), board_pchlan }, 7810 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC), board_pchlan }, 7811 7812 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM), board_pch2lan }, 7813 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V), board_pch2lan }, 7814 7815 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_LM), board_pch_lpt }, 7816 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_V), board_pch_lpt }, 7817 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_LM), board_pch_lpt }, 7818 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_V), board_pch_lpt }, 7819 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM2), board_pch_lpt }, 7820 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V2), board_pch_lpt }, 7821 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM3), board_pch_lpt }, 7822 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V3), board_pch_lpt }, 7823 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM), board_pch_spt }, 7824 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V), board_pch_spt }, 7825 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM2), board_pch_spt }, 7826 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V2), board_pch_spt }, 7827 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LBG_I219_LM3), board_pch_spt }, 7828 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM4), board_pch_spt }, 7829 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V4), board_pch_spt }, 7830 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM5), board_pch_spt }, 7831 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V5), board_pch_spt }, 7832 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM6), board_pch_cnp }, 7833 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V6), board_pch_cnp }, 7834 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM7), board_pch_cnp }, 7835 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V7), board_pch_cnp }, 7836 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM8), board_pch_cnp }, 7837 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V8), board_pch_cnp }, 7838 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM9), board_pch_cnp }, 7839 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V9), board_pch_cnp }, 7840 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM10), board_pch_cnp }, 7841 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V10), board_pch_cnp }, 7842 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM11), board_pch_cnp }, 7843 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V11), board_pch_cnp }, 7844 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM12), board_pch_spt }, 7845 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V12), board_pch_spt }, 7846 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM13), board_pch_cnp }, 7847 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V13), board_pch_cnp }, 7848 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM14), board_pch_cnp }, 7849 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V14), board_pch_cnp }, 7850 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM15), board_pch_cnp }, 7851 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V15), board_pch_cnp }, 7852 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM16), board_pch_cnp }, 7853 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V16), board_pch_cnp }, 7854 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM17), board_pch_cnp }, 7855 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V17), board_pch_cnp }, 7856 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM18), board_pch_cnp }, 7857 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V18), board_pch_cnp }, 7858 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM19), board_pch_cnp }, 7859 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V19), board_pch_cnp }, 7860 7861 { 0, 0, 0, 0, 0, 0, 0 } /* terminate list */ 7862 }; 7863 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl); 7864 7865 static const struct dev_pm_ops e1000_pm_ops = { 7866 #ifdef CONFIG_PM_SLEEP 7867 .prepare = e1000e_pm_prepare, 7868 .suspend = e1000e_pm_suspend, 7869 .resume = e1000e_pm_resume, 7870 .freeze = e1000e_pm_freeze, 7871 .thaw = e1000e_pm_thaw, 7872 .poweroff = e1000e_pm_suspend, 7873 .restore = e1000e_pm_resume, 7874 #endif 7875 SET_RUNTIME_PM_OPS(e1000e_pm_runtime_suspend, e1000e_pm_runtime_resume, 7876 e1000e_pm_runtime_idle) 7877 }; 7878 7879 /* PCI Device API Driver */ 7880 static struct pci_driver e1000_driver = { 7881 .name = e1000e_driver_name, 7882 .id_table = e1000_pci_tbl, 7883 .probe = e1000_probe, 7884 .remove = e1000_remove, 7885 .driver = { 7886 .pm = &e1000_pm_ops, 7887 }, 7888 .shutdown = e1000_shutdown, 7889 .err_handler = &e1000_err_handler 7890 }; 7891 7892 /** 7893 * e1000_init_module - Driver Registration Routine 7894 * 7895 * e1000_init_module is the first routine called when the driver is 7896 * loaded. All it does is register with the PCI subsystem. 7897 **/ 7898 static int __init e1000_init_module(void) 7899 { 7900 pr_info("Intel(R) PRO/1000 Network Driver\n"); 7901 pr_info("Copyright(c) 1999 - 2015 Intel Corporation.\n"); 7902 7903 return pci_register_driver(&e1000_driver); 7904 } 7905 module_init(e1000_init_module); 7906 7907 /** 7908 * e1000_exit_module - Driver Exit Cleanup Routine 7909 * 7910 * e1000_exit_module is called just before the driver is removed 7911 * from memory. 7912 **/ 7913 static void __exit e1000_exit_module(void) 7914 { 7915 pci_unregister_driver(&e1000_driver); 7916 } 7917 module_exit(e1000_exit_module); 7918 7919 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 7920 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver"); 7921 MODULE_LICENSE("GPL v2"); 7922 7923 /* netdev.c */ 7924