1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 1999 - 2018 Intel Corporation. */ 3 4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 5 6 #include <linux/module.h> 7 #include <linux/types.h> 8 #include <linux/init.h> 9 #include <linux/pci.h> 10 #include <linux/vmalloc.h> 11 #include <linux/pagemap.h> 12 #include <linux/delay.h> 13 #include <linux/netdevice.h> 14 #include <linux/interrupt.h> 15 #include <linux/tcp.h> 16 #include <linux/ipv6.h> 17 #include <linux/slab.h> 18 #include <net/checksum.h> 19 #include <net/ip6_checksum.h> 20 #include <linux/ethtool.h> 21 #include <linux/if_vlan.h> 22 #include <linux/cpu.h> 23 #include <linux/smp.h> 24 #include <linux/pm_qos.h> 25 #include <linux/pm_runtime.h> 26 #include <linux/aer.h> 27 #include <linux/prefetch.h> 28 #include <linux/suspend.h> 29 30 #include "e1000.h" 31 32 char e1000e_driver_name[] = "e1000e"; 33 34 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 35 static int debug = -1; 36 module_param(debug, int, 0); 37 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 38 39 static const struct e1000_info *e1000_info_tbl[] = { 40 [board_82571] = &e1000_82571_info, 41 [board_82572] = &e1000_82572_info, 42 [board_82573] = &e1000_82573_info, 43 [board_82574] = &e1000_82574_info, 44 [board_82583] = &e1000_82583_info, 45 [board_80003es2lan] = &e1000_es2_info, 46 [board_ich8lan] = &e1000_ich8_info, 47 [board_ich9lan] = &e1000_ich9_info, 48 [board_ich10lan] = &e1000_ich10_info, 49 [board_pchlan] = &e1000_pch_info, 50 [board_pch2lan] = &e1000_pch2_info, 51 [board_pch_lpt] = &e1000_pch_lpt_info, 52 [board_pch_spt] = &e1000_pch_spt_info, 53 [board_pch_cnp] = &e1000_pch_cnp_info, 54 }; 55 56 struct e1000_reg_info { 57 u32 ofs; 58 char *name; 59 }; 60 61 static const struct e1000_reg_info e1000_reg_info_tbl[] = { 62 /* General Registers */ 63 {E1000_CTRL, "CTRL"}, 64 {E1000_STATUS, "STATUS"}, 65 {E1000_CTRL_EXT, "CTRL_EXT"}, 66 67 /* Interrupt Registers */ 68 {E1000_ICR, "ICR"}, 69 70 /* Rx Registers */ 71 {E1000_RCTL, "RCTL"}, 72 {E1000_RDLEN(0), "RDLEN"}, 73 {E1000_RDH(0), "RDH"}, 74 {E1000_RDT(0), "RDT"}, 75 {E1000_RDTR, "RDTR"}, 76 {E1000_RXDCTL(0), "RXDCTL"}, 77 {E1000_ERT, "ERT"}, 78 {E1000_RDBAL(0), "RDBAL"}, 79 {E1000_RDBAH(0), "RDBAH"}, 80 {E1000_RDFH, "RDFH"}, 81 {E1000_RDFT, "RDFT"}, 82 {E1000_RDFHS, "RDFHS"}, 83 {E1000_RDFTS, "RDFTS"}, 84 {E1000_RDFPC, "RDFPC"}, 85 86 /* Tx Registers */ 87 {E1000_TCTL, "TCTL"}, 88 {E1000_TDBAL(0), "TDBAL"}, 89 {E1000_TDBAH(0), "TDBAH"}, 90 {E1000_TDLEN(0), "TDLEN"}, 91 {E1000_TDH(0), "TDH"}, 92 {E1000_TDT(0), "TDT"}, 93 {E1000_TIDV, "TIDV"}, 94 {E1000_TXDCTL(0), "TXDCTL"}, 95 {E1000_TADV, "TADV"}, 96 {E1000_TARC(0), "TARC"}, 97 {E1000_TDFH, "TDFH"}, 98 {E1000_TDFT, "TDFT"}, 99 {E1000_TDFHS, "TDFHS"}, 100 {E1000_TDFTS, "TDFTS"}, 101 {E1000_TDFPC, "TDFPC"}, 102 103 /* List Terminator */ 104 {0, NULL} 105 }; 106 107 /** 108 * __ew32_prepare - prepare to write to MAC CSR register on certain parts 109 * @hw: pointer to the HW structure 110 * 111 * When updating the MAC CSR registers, the Manageability Engine (ME) could 112 * be accessing the registers at the same time. Normally, this is handled in 113 * h/w by an arbiter but on some parts there is a bug that acknowledges Host 114 * accesses later than it should which could result in the register to have 115 * an incorrect value. Workaround this by checking the FWSM register which 116 * has bit 24 set while ME is accessing MAC CSR registers, wait if it is set 117 * and try again a number of times. 118 **/ 119 static void __ew32_prepare(struct e1000_hw *hw) 120 { 121 s32 i = E1000_ICH_FWSM_PCIM2PCI_COUNT; 122 123 while ((er32(FWSM) & E1000_ICH_FWSM_PCIM2PCI) && --i) 124 udelay(50); 125 } 126 127 void __ew32(struct e1000_hw *hw, unsigned long reg, u32 val) 128 { 129 if (hw->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 130 __ew32_prepare(hw); 131 132 writel(val, hw->hw_addr + reg); 133 } 134 135 /** 136 * e1000_regdump - register printout routine 137 * @hw: pointer to the HW structure 138 * @reginfo: pointer to the register info table 139 **/ 140 static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo) 141 { 142 int n = 0; 143 char rname[16]; 144 u32 regs[8]; 145 146 switch (reginfo->ofs) { 147 case E1000_RXDCTL(0): 148 for (n = 0; n < 2; n++) 149 regs[n] = __er32(hw, E1000_RXDCTL(n)); 150 break; 151 case E1000_TXDCTL(0): 152 for (n = 0; n < 2; n++) 153 regs[n] = __er32(hw, E1000_TXDCTL(n)); 154 break; 155 case E1000_TARC(0): 156 for (n = 0; n < 2; n++) 157 regs[n] = __er32(hw, E1000_TARC(n)); 158 break; 159 default: 160 pr_info("%-15s %08x\n", 161 reginfo->name, __er32(hw, reginfo->ofs)); 162 return; 163 } 164 165 snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]"); 166 pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]); 167 } 168 169 static void e1000e_dump_ps_pages(struct e1000_adapter *adapter, 170 struct e1000_buffer *bi) 171 { 172 int i; 173 struct e1000_ps_page *ps_page; 174 175 for (i = 0; i < adapter->rx_ps_pages; i++) { 176 ps_page = &bi->ps_pages[i]; 177 178 if (ps_page->page) { 179 pr_info("packet dump for ps_page %d:\n", i); 180 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 181 16, 1, page_address(ps_page->page), 182 PAGE_SIZE, true); 183 } 184 } 185 } 186 187 /** 188 * e1000e_dump - Print registers, Tx-ring and Rx-ring 189 * @adapter: board private structure 190 **/ 191 static void e1000e_dump(struct e1000_adapter *adapter) 192 { 193 struct net_device *netdev = adapter->netdev; 194 struct e1000_hw *hw = &adapter->hw; 195 struct e1000_reg_info *reginfo; 196 struct e1000_ring *tx_ring = adapter->tx_ring; 197 struct e1000_tx_desc *tx_desc; 198 struct my_u0 { 199 __le64 a; 200 __le64 b; 201 } *u0; 202 struct e1000_buffer *buffer_info; 203 struct e1000_ring *rx_ring = adapter->rx_ring; 204 union e1000_rx_desc_packet_split *rx_desc_ps; 205 union e1000_rx_desc_extended *rx_desc; 206 struct my_u1 { 207 __le64 a; 208 __le64 b; 209 __le64 c; 210 __le64 d; 211 } *u1; 212 u32 staterr; 213 int i = 0; 214 215 if (!netif_msg_hw(adapter)) 216 return; 217 218 /* Print netdevice Info */ 219 if (netdev) { 220 dev_info(&adapter->pdev->dev, "Net device Info\n"); 221 pr_info("Device Name state trans_start\n"); 222 pr_info("%-15s %016lX %016lX\n", netdev->name, 223 netdev->state, dev_trans_start(netdev)); 224 } 225 226 /* Print Registers */ 227 dev_info(&adapter->pdev->dev, "Register Dump\n"); 228 pr_info(" Register Name Value\n"); 229 for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl; 230 reginfo->name; reginfo++) { 231 e1000_regdump(hw, reginfo); 232 } 233 234 /* Print Tx Ring Summary */ 235 if (!netdev || !netif_running(netdev)) 236 return; 237 238 dev_info(&adapter->pdev->dev, "Tx Ring Summary\n"); 239 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); 240 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean]; 241 pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n", 242 0, tx_ring->next_to_use, tx_ring->next_to_clean, 243 (unsigned long long)buffer_info->dma, 244 buffer_info->length, 245 buffer_info->next_to_watch, 246 (unsigned long long)buffer_info->time_stamp); 247 248 /* Print Tx Ring */ 249 if (!netif_msg_tx_done(adapter)) 250 goto rx_ring_summary; 251 252 dev_info(&adapter->pdev->dev, "Tx Ring Dump\n"); 253 254 /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended) 255 * 256 * Legacy Transmit Descriptor 257 * +--------------------------------------------------------------+ 258 * 0 | Buffer Address [63:0] (Reserved on Write Back) | 259 * +--------------------------------------------------------------+ 260 * 8 | Special | CSS | Status | CMD | CSO | Length | 261 * +--------------------------------------------------------------+ 262 * 63 48 47 36 35 32 31 24 23 16 15 0 263 * 264 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload 265 * 63 48 47 40 39 32 31 16 15 8 7 0 266 * +----------------------------------------------------------------+ 267 * 0 | TUCSE | TUCS0 | TUCSS | IPCSE | IPCS0 | IPCSS | 268 * +----------------------------------------------------------------+ 269 * 8 | MSS | HDRLEN | RSV | STA | TUCMD | DTYP | PAYLEN | 270 * +----------------------------------------------------------------+ 271 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0 272 * 273 * Extended Data Descriptor (DTYP=0x1) 274 * +----------------------------------------------------------------+ 275 * 0 | Buffer Address [63:0] | 276 * +----------------------------------------------------------------+ 277 * 8 | VLAN tag | POPTS | Rsvd | Status | Command | DTYP | DTALEN | 278 * +----------------------------------------------------------------+ 279 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0 280 */ 281 pr_info("Tl[desc] [address 63:0 ] [SpeCssSCmCsLen] [bi->dma ] leng ntw timestamp bi->skb <-- Legacy format\n"); 282 pr_info("Tc[desc] [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Context format\n"); 283 pr_info("Td[desc] [address 63:0 ] [VlaPoRSCm1Dlen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Data format\n"); 284 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { 285 const char *next_desc; 286 tx_desc = E1000_TX_DESC(*tx_ring, i); 287 buffer_info = &tx_ring->buffer_info[i]; 288 u0 = (struct my_u0 *)tx_desc; 289 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean) 290 next_desc = " NTC/U"; 291 else if (i == tx_ring->next_to_use) 292 next_desc = " NTU"; 293 else if (i == tx_ring->next_to_clean) 294 next_desc = " NTC"; 295 else 296 next_desc = ""; 297 pr_info("T%c[0x%03X] %016llX %016llX %016llX %04X %3X %016llX %p%s\n", 298 (!(le64_to_cpu(u0->b) & BIT(29)) ? 'l' : 299 ((le64_to_cpu(u0->b) & BIT(20)) ? 'd' : 'c')), 300 i, 301 (unsigned long long)le64_to_cpu(u0->a), 302 (unsigned long long)le64_to_cpu(u0->b), 303 (unsigned long long)buffer_info->dma, 304 buffer_info->length, buffer_info->next_to_watch, 305 (unsigned long long)buffer_info->time_stamp, 306 buffer_info->skb, next_desc); 307 308 if (netif_msg_pktdata(adapter) && buffer_info->skb) 309 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 310 16, 1, buffer_info->skb->data, 311 buffer_info->skb->len, true); 312 } 313 314 /* Print Rx Ring Summary */ 315 rx_ring_summary: 316 dev_info(&adapter->pdev->dev, "Rx Ring Summary\n"); 317 pr_info("Queue [NTU] [NTC]\n"); 318 pr_info(" %5d %5X %5X\n", 319 0, rx_ring->next_to_use, rx_ring->next_to_clean); 320 321 /* Print Rx Ring */ 322 if (!netif_msg_rx_status(adapter)) 323 return; 324 325 dev_info(&adapter->pdev->dev, "Rx Ring Dump\n"); 326 switch (adapter->rx_ps_pages) { 327 case 1: 328 case 2: 329 case 3: 330 /* [Extended] Packet Split Receive Descriptor Format 331 * 332 * +-----------------------------------------------------+ 333 * 0 | Buffer Address 0 [63:0] | 334 * +-----------------------------------------------------+ 335 * 8 | Buffer Address 1 [63:0] | 336 * +-----------------------------------------------------+ 337 * 16 | Buffer Address 2 [63:0] | 338 * +-----------------------------------------------------+ 339 * 24 | Buffer Address 3 [63:0] | 340 * +-----------------------------------------------------+ 341 */ 342 pr_info("R [desc] [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma ] [bi->skb] <-- Ext Pkt Split format\n"); 343 /* [Extended] Receive Descriptor (Write-Back) Format 344 * 345 * 63 48 47 32 31 13 12 8 7 4 3 0 346 * +------------------------------------------------------+ 347 * 0 | Packet | IP | Rsvd | MRQ | Rsvd | MRQ RSS | 348 * | Checksum | Ident | | Queue | | Type | 349 * +------------------------------------------------------+ 350 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 351 * +------------------------------------------------------+ 352 * 63 48 47 32 31 20 19 0 353 */ 354 pr_info("RWB[desc] [ck ipid mrqhsh] [vl l0 ee es] [ l3 l2 l1 hs] [reserved ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n"); 355 for (i = 0; i < rx_ring->count; i++) { 356 const char *next_desc; 357 buffer_info = &rx_ring->buffer_info[i]; 358 rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i); 359 u1 = (struct my_u1 *)rx_desc_ps; 360 staterr = 361 le32_to_cpu(rx_desc_ps->wb.middle.status_error); 362 363 if (i == rx_ring->next_to_use) 364 next_desc = " NTU"; 365 else if (i == rx_ring->next_to_clean) 366 next_desc = " NTC"; 367 else 368 next_desc = ""; 369 370 if (staterr & E1000_RXD_STAT_DD) { 371 /* Descriptor Done */ 372 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX ---------------- %p%s\n", 373 "RWB", i, 374 (unsigned long long)le64_to_cpu(u1->a), 375 (unsigned long long)le64_to_cpu(u1->b), 376 (unsigned long long)le64_to_cpu(u1->c), 377 (unsigned long long)le64_to_cpu(u1->d), 378 buffer_info->skb, next_desc); 379 } else { 380 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX %016llX %p%s\n", 381 "R ", i, 382 (unsigned long long)le64_to_cpu(u1->a), 383 (unsigned long long)le64_to_cpu(u1->b), 384 (unsigned long long)le64_to_cpu(u1->c), 385 (unsigned long long)le64_to_cpu(u1->d), 386 (unsigned long long)buffer_info->dma, 387 buffer_info->skb, next_desc); 388 389 if (netif_msg_pktdata(adapter)) 390 e1000e_dump_ps_pages(adapter, 391 buffer_info); 392 } 393 } 394 break; 395 default: 396 case 0: 397 /* Extended Receive Descriptor (Read) Format 398 * 399 * +-----------------------------------------------------+ 400 * 0 | Buffer Address [63:0] | 401 * +-----------------------------------------------------+ 402 * 8 | Reserved | 403 * +-----------------------------------------------------+ 404 */ 405 pr_info("R [desc] [buf addr 63:0 ] [reserved 63:0 ] [bi->dma ] [bi->skb] <-- Ext (Read) format\n"); 406 /* Extended Receive Descriptor (Write-Back) Format 407 * 408 * 63 48 47 32 31 24 23 4 3 0 409 * +------------------------------------------------------+ 410 * | RSS Hash | | | | 411 * 0 +-------------------+ Rsvd | Reserved | MRQ RSS | 412 * | Packet | IP | | | Type | 413 * | Checksum | Ident | | | | 414 * +------------------------------------------------------+ 415 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 416 * +------------------------------------------------------+ 417 * 63 48 47 32 31 20 19 0 418 */ 419 pr_info("RWB[desc] [cs ipid mrq] [vt ln xe xs] [bi->skb] <-- Ext (Write-Back) format\n"); 420 421 for (i = 0; i < rx_ring->count; i++) { 422 const char *next_desc; 423 424 buffer_info = &rx_ring->buffer_info[i]; 425 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 426 u1 = (struct my_u1 *)rx_desc; 427 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 428 429 if (i == rx_ring->next_to_use) 430 next_desc = " NTU"; 431 else if (i == rx_ring->next_to_clean) 432 next_desc = " NTC"; 433 else 434 next_desc = ""; 435 436 if (staterr & E1000_RXD_STAT_DD) { 437 /* Descriptor Done */ 438 pr_info("%s[0x%03X] %016llX %016llX ---------------- %p%s\n", 439 "RWB", i, 440 (unsigned long long)le64_to_cpu(u1->a), 441 (unsigned long long)le64_to_cpu(u1->b), 442 buffer_info->skb, next_desc); 443 } else { 444 pr_info("%s[0x%03X] %016llX %016llX %016llX %p%s\n", 445 "R ", i, 446 (unsigned long long)le64_to_cpu(u1->a), 447 (unsigned long long)le64_to_cpu(u1->b), 448 (unsigned long long)buffer_info->dma, 449 buffer_info->skb, next_desc); 450 451 if (netif_msg_pktdata(adapter) && 452 buffer_info->skb) 453 print_hex_dump(KERN_INFO, "", 454 DUMP_PREFIX_ADDRESS, 16, 455 1, 456 buffer_info->skb->data, 457 adapter->rx_buffer_len, 458 true); 459 } 460 } 461 } 462 } 463 464 /** 465 * e1000_desc_unused - calculate if we have unused descriptors 466 * @ring: pointer to ring struct to perform calculation on 467 **/ 468 static int e1000_desc_unused(struct e1000_ring *ring) 469 { 470 if (ring->next_to_clean > ring->next_to_use) 471 return ring->next_to_clean - ring->next_to_use - 1; 472 473 return ring->count + ring->next_to_clean - ring->next_to_use - 1; 474 } 475 476 /** 477 * e1000e_systim_to_hwtstamp - convert system time value to hw time stamp 478 * @adapter: board private structure 479 * @hwtstamps: time stamp structure to update 480 * @systim: unsigned 64bit system time value. 481 * 482 * Convert the system time value stored in the RX/TXSTMP registers into a 483 * hwtstamp which can be used by the upper level time stamping functions. 484 * 485 * The 'systim_lock' spinlock is used to protect the consistency of the 486 * system time value. This is needed because reading the 64 bit time 487 * value involves reading two 32 bit registers. The first read latches the 488 * value. 489 **/ 490 static void e1000e_systim_to_hwtstamp(struct e1000_adapter *adapter, 491 struct skb_shared_hwtstamps *hwtstamps, 492 u64 systim) 493 { 494 u64 ns; 495 unsigned long flags; 496 497 spin_lock_irqsave(&adapter->systim_lock, flags); 498 ns = timecounter_cyc2time(&adapter->tc, systim); 499 spin_unlock_irqrestore(&adapter->systim_lock, flags); 500 501 memset(hwtstamps, 0, sizeof(*hwtstamps)); 502 hwtstamps->hwtstamp = ns_to_ktime(ns); 503 } 504 505 /** 506 * e1000e_rx_hwtstamp - utility function which checks for Rx time stamp 507 * @adapter: board private structure 508 * @status: descriptor extended error and status field 509 * @skb: particular skb to include time stamp 510 * 511 * If the time stamp is valid, convert it into the timecounter ns value 512 * and store that result into the shhwtstamps structure which is passed 513 * up the network stack. 514 **/ 515 static void e1000e_rx_hwtstamp(struct e1000_adapter *adapter, u32 status, 516 struct sk_buff *skb) 517 { 518 struct e1000_hw *hw = &adapter->hw; 519 u64 rxstmp; 520 521 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP) || 522 !(status & E1000_RXDEXT_STATERR_TST) || 523 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) 524 return; 525 526 /* The Rx time stamp registers contain the time stamp. No other 527 * received packet will be time stamped until the Rx time stamp 528 * registers are read. Because only one packet can be time stamped 529 * at a time, the register values must belong to this packet and 530 * therefore none of the other additional attributes need to be 531 * compared. 532 */ 533 rxstmp = (u64)er32(RXSTMPL); 534 rxstmp |= (u64)er32(RXSTMPH) << 32; 535 e1000e_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), rxstmp); 536 537 adapter->flags2 &= ~FLAG2_CHECK_RX_HWTSTAMP; 538 } 539 540 /** 541 * e1000_receive_skb - helper function to handle Rx indications 542 * @adapter: board private structure 543 * @netdev: pointer to netdev struct 544 * @staterr: descriptor extended error and status field as written by hardware 545 * @vlan: descriptor vlan field as written by hardware (no le/be conversion) 546 * @skb: pointer to sk_buff to be indicated to stack 547 **/ 548 static void e1000_receive_skb(struct e1000_adapter *adapter, 549 struct net_device *netdev, struct sk_buff *skb, 550 u32 staterr, __le16 vlan) 551 { 552 u16 tag = le16_to_cpu(vlan); 553 554 e1000e_rx_hwtstamp(adapter, staterr, skb); 555 556 skb->protocol = eth_type_trans(skb, netdev); 557 558 if (staterr & E1000_RXD_STAT_VP) 559 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag); 560 561 napi_gro_receive(&adapter->napi, skb); 562 } 563 564 /** 565 * e1000_rx_checksum - Receive Checksum Offload 566 * @adapter: board private structure 567 * @status_err: receive descriptor status and error fields 568 * @skb: socket buffer with received data 569 **/ 570 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err, 571 struct sk_buff *skb) 572 { 573 u16 status = (u16)status_err; 574 u8 errors = (u8)(status_err >> 24); 575 576 skb_checksum_none_assert(skb); 577 578 /* Rx checksum disabled */ 579 if (!(adapter->netdev->features & NETIF_F_RXCSUM)) 580 return; 581 582 /* Ignore Checksum bit is set */ 583 if (status & E1000_RXD_STAT_IXSM) 584 return; 585 586 /* TCP/UDP checksum error bit or IP checksum error bit is set */ 587 if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) { 588 /* let the stack verify checksum errors */ 589 adapter->hw_csum_err++; 590 return; 591 } 592 593 /* TCP/UDP Checksum has not been calculated */ 594 if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) 595 return; 596 597 /* It must be a TCP or UDP packet with a valid checksum */ 598 skb->ip_summed = CHECKSUM_UNNECESSARY; 599 adapter->hw_csum_good++; 600 } 601 602 static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i) 603 { 604 struct e1000_adapter *adapter = rx_ring->adapter; 605 struct e1000_hw *hw = &adapter->hw; 606 607 __ew32_prepare(hw); 608 writel(i, rx_ring->tail); 609 610 if (unlikely(i != readl(rx_ring->tail))) { 611 u32 rctl = er32(RCTL); 612 613 ew32(RCTL, rctl & ~E1000_RCTL_EN); 614 e_err("ME firmware caused invalid RDT - resetting\n"); 615 schedule_work(&adapter->reset_task); 616 } 617 } 618 619 static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i) 620 { 621 struct e1000_adapter *adapter = tx_ring->adapter; 622 struct e1000_hw *hw = &adapter->hw; 623 624 __ew32_prepare(hw); 625 writel(i, tx_ring->tail); 626 627 if (unlikely(i != readl(tx_ring->tail))) { 628 u32 tctl = er32(TCTL); 629 630 ew32(TCTL, tctl & ~E1000_TCTL_EN); 631 e_err("ME firmware caused invalid TDT - resetting\n"); 632 schedule_work(&adapter->reset_task); 633 } 634 } 635 636 /** 637 * e1000_alloc_rx_buffers - Replace used receive buffers 638 * @rx_ring: Rx descriptor ring 639 * @cleaned_count: number to reallocate 640 * @gfp: flags for allocation 641 **/ 642 static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring, 643 int cleaned_count, gfp_t gfp) 644 { 645 struct e1000_adapter *adapter = rx_ring->adapter; 646 struct net_device *netdev = adapter->netdev; 647 struct pci_dev *pdev = adapter->pdev; 648 union e1000_rx_desc_extended *rx_desc; 649 struct e1000_buffer *buffer_info; 650 struct sk_buff *skb; 651 unsigned int i; 652 unsigned int bufsz = adapter->rx_buffer_len; 653 654 i = rx_ring->next_to_use; 655 buffer_info = &rx_ring->buffer_info[i]; 656 657 while (cleaned_count--) { 658 skb = buffer_info->skb; 659 if (skb) { 660 skb_trim(skb, 0); 661 goto map_skb; 662 } 663 664 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp); 665 if (!skb) { 666 /* Better luck next round */ 667 adapter->alloc_rx_buff_failed++; 668 break; 669 } 670 671 buffer_info->skb = skb; 672 map_skb: 673 buffer_info->dma = dma_map_single(&pdev->dev, skb->data, 674 adapter->rx_buffer_len, 675 DMA_FROM_DEVICE); 676 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 677 dev_err(&pdev->dev, "Rx DMA map failed\n"); 678 adapter->rx_dma_failed++; 679 break; 680 } 681 682 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 683 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma); 684 685 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) { 686 /* Force memory writes to complete before letting h/w 687 * know there are new descriptors to fetch. (Only 688 * applicable for weak-ordered memory model archs, 689 * such as IA-64). 690 */ 691 wmb(); 692 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 693 e1000e_update_rdt_wa(rx_ring, i); 694 else 695 writel(i, rx_ring->tail); 696 } 697 i++; 698 if (i == rx_ring->count) 699 i = 0; 700 buffer_info = &rx_ring->buffer_info[i]; 701 } 702 703 rx_ring->next_to_use = i; 704 } 705 706 /** 707 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split 708 * @rx_ring: Rx descriptor ring 709 * @cleaned_count: number to reallocate 710 * @gfp: flags for allocation 711 **/ 712 static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring, 713 int cleaned_count, gfp_t gfp) 714 { 715 struct e1000_adapter *adapter = rx_ring->adapter; 716 struct net_device *netdev = adapter->netdev; 717 struct pci_dev *pdev = adapter->pdev; 718 union e1000_rx_desc_packet_split *rx_desc; 719 struct e1000_buffer *buffer_info; 720 struct e1000_ps_page *ps_page; 721 struct sk_buff *skb; 722 unsigned int i, j; 723 724 i = rx_ring->next_to_use; 725 buffer_info = &rx_ring->buffer_info[i]; 726 727 while (cleaned_count--) { 728 rx_desc = E1000_RX_DESC_PS(*rx_ring, i); 729 730 for (j = 0; j < PS_PAGE_BUFFERS; j++) { 731 ps_page = &buffer_info->ps_pages[j]; 732 if (j >= adapter->rx_ps_pages) { 733 /* all unused desc entries get hw null ptr */ 734 rx_desc->read.buffer_addr[j + 1] = 735 ~cpu_to_le64(0); 736 continue; 737 } 738 if (!ps_page->page) { 739 ps_page->page = alloc_page(gfp); 740 if (!ps_page->page) { 741 adapter->alloc_rx_buff_failed++; 742 goto no_buffers; 743 } 744 ps_page->dma = dma_map_page(&pdev->dev, 745 ps_page->page, 746 0, PAGE_SIZE, 747 DMA_FROM_DEVICE); 748 if (dma_mapping_error(&pdev->dev, 749 ps_page->dma)) { 750 dev_err(&adapter->pdev->dev, 751 "Rx DMA page map failed\n"); 752 adapter->rx_dma_failed++; 753 goto no_buffers; 754 } 755 } 756 /* Refresh the desc even if buffer_addrs 757 * didn't change because each write-back 758 * erases this info. 759 */ 760 rx_desc->read.buffer_addr[j + 1] = 761 cpu_to_le64(ps_page->dma); 762 } 763 764 skb = __netdev_alloc_skb_ip_align(netdev, adapter->rx_ps_bsize0, 765 gfp); 766 767 if (!skb) { 768 adapter->alloc_rx_buff_failed++; 769 break; 770 } 771 772 buffer_info->skb = skb; 773 buffer_info->dma = dma_map_single(&pdev->dev, skb->data, 774 adapter->rx_ps_bsize0, 775 DMA_FROM_DEVICE); 776 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 777 dev_err(&pdev->dev, "Rx DMA map failed\n"); 778 adapter->rx_dma_failed++; 779 /* cleanup skb */ 780 dev_kfree_skb_any(skb); 781 buffer_info->skb = NULL; 782 break; 783 } 784 785 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma); 786 787 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) { 788 /* Force memory writes to complete before letting h/w 789 * know there are new descriptors to fetch. (Only 790 * applicable for weak-ordered memory model archs, 791 * such as IA-64). 792 */ 793 wmb(); 794 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 795 e1000e_update_rdt_wa(rx_ring, i << 1); 796 else 797 writel(i << 1, rx_ring->tail); 798 } 799 800 i++; 801 if (i == rx_ring->count) 802 i = 0; 803 buffer_info = &rx_ring->buffer_info[i]; 804 } 805 806 no_buffers: 807 rx_ring->next_to_use = i; 808 } 809 810 /** 811 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers 812 * @rx_ring: Rx descriptor ring 813 * @cleaned_count: number of buffers to allocate this pass 814 * @gfp: flags for allocation 815 **/ 816 817 static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring, 818 int cleaned_count, gfp_t gfp) 819 { 820 struct e1000_adapter *adapter = rx_ring->adapter; 821 struct net_device *netdev = adapter->netdev; 822 struct pci_dev *pdev = adapter->pdev; 823 union e1000_rx_desc_extended *rx_desc; 824 struct e1000_buffer *buffer_info; 825 struct sk_buff *skb; 826 unsigned int i; 827 unsigned int bufsz = 256 - 16; /* for skb_reserve */ 828 829 i = rx_ring->next_to_use; 830 buffer_info = &rx_ring->buffer_info[i]; 831 832 while (cleaned_count--) { 833 skb = buffer_info->skb; 834 if (skb) { 835 skb_trim(skb, 0); 836 goto check_page; 837 } 838 839 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp); 840 if (unlikely(!skb)) { 841 /* Better luck next round */ 842 adapter->alloc_rx_buff_failed++; 843 break; 844 } 845 846 buffer_info->skb = skb; 847 check_page: 848 /* allocate a new page if necessary */ 849 if (!buffer_info->page) { 850 buffer_info->page = alloc_page(gfp); 851 if (unlikely(!buffer_info->page)) { 852 adapter->alloc_rx_buff_failed++; 853 break; 854 } 855 } 856 857 if (!buffer_info->dma) { 858 buffer_info->dma = dma_map_page(&pdev->dev, 859 buffer_info->page, 0, 860 PAGE_SIZE, 861 DMA_FROM_DEVICE); 862 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 863 adapter->alloc_rx_buff_failed++; 864 break; 865 } 866 } 867 868 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 869 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma); 870 871 if (unlikely(++i == rx_ring->count)) 872 i = 0; 873 buffer_info = &rx_ring->buffer_info[i]; 874 } 875 876 if (likely(rx_ring->next_to_use != i)) { 877 rx_ring->next_to_use = i; 878 if (unlikely(i-- == 0)) 879 i = (rx_ring->count - 1); 880 881 /* Force memory writes to complete before letting h/w 882 * know there are new descriptors to fetch. (Only 883 * applicable for weak-ordered memory model archs, 884 * such as IA-64). 885 */ 886 wmb(); 887 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 888 e1000e_update_rdt_wa(rx_ring, i); 889 else 890 writel(i, rx_ring->tail); 891 } 892 } 893 894 static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss, 895 struct sk_buff *skb) 896 { 897 if (netdev->features & NETIF_F_RXHASH) 898 skb_set_hash(skb, le32_to_cpu(rss), PKT_HASH_TYPE_L3); 899 } 900 901 /** 902 * e1000_clean_rx_irq - Send received data up the network stack 903 * @rx_ring: Rx descriptor ring 904 * @work_done: output parameter for indicating completed work 905 * @work_to_do: how many packets we can clean 906 * 907 * the return value indicates whether actual cleaning was done, there 908 * is no guarantee that everything was cleaned 909 **/ 910 static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done, 911 int work_to_do) 912 { 913 struct e1000_adapter *adapter = rx_ring->adapter; 914 struct net_device *netdev = adapter->netdev; 915 struct pci_dev *pdev = adapter->pdev; 916 struct e1000_hw *hw = &adapter->hw; 917 union e1000_rx_desc_extended *rx_desc, *next_rxd; 918 struct e1000_buffer *buffer_info, *next_buffer; 919 u32 length, staterr; 920 unsigned int i; 921 int cleaned_count = 0; 922 bool cleaned = false; 923 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 924 925 i = rx_ring->next_to_clean; 926 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 927 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 928 buffer_info = &rx_ring->buffer_info[i]; 929 930 while (staterr & E1000_RXD_STAT_DD) { 931 struct sk_buff *skb; 932 933 if (*work_done >= work_to_do) 934 break; 935 (*work_done)++; 936 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ 937 938 skb = buffer_info->skb; 939 buffer_info->skb = NULL; 940 941 prefetch(skb->data - NET_IP_ALIGN); 942 943 i++; 944 if (i == rx_ring->count) 945 i = 0; 946 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i); 947 prefetch(next_rxd); 948 949 next_buffer = &rx_ring->buffer_info[i]; 950 951 cleaned = true; 952 cleaned_count++; 953 dma_unmap_single(&pdev->dev, buffer_info->dma, 954 adapter->rx_buffer_len, DMA_FROM_DEVICE); 955 buffer_info->dma = 0; 956 957 length = le16_to_cpu(rx_desc->wb.upper.length); 958 959 /* !EOP means multiple descriptors were used to store a single 960 * packet, if that's the case we need to toss it. In fact, we 961 * need to toss every packet with the EOP bit clear and the 962 * next frame that _does_ have the EOP bit set, as it is by 963 * definition only a frame fragment 964 */ 965 if (unlikely(!(staterr & E1000_RXD_STAT_EOP))) 966 adapter->flags2 |= FLAG2_IS_DISCARDING; 967 968 if (adapter->flags2 & FLAG2_IS_DISCARDING) { 969 /* All receives must fit into a single buffer */ 970 e_dbg("Receive packet consumed multiple buffers\n"); 971 /* recycle */ 972 buffer_info->skb = skb; 973 if (staterr & E1000_RXD_STAT_EOP) 974 adapter->flags2 &= ~FLAG2_IS_DISCARDING; 975 goto next_desc; 976 } 977 978 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) && 979 !(netdev->features & NETIF_F_RXALL))) { 980 /* recycle */ 981 buffer_info->skb = skb; 982 goto next_desc; 983 } 984 985 /* adjust length to remove Ethernet CRC */ 986 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) { 987 /* If configured to store CRC, don't subtract FCS, 988 * but keep the FCS bytes out of the total_rx_bytes 989 * counter 990 */ 991 if (netdev->features & NETIF_F_RXFCS) 992 total_rx_bytes -= 4; 993 else 994 length -= 4; 995 } 996 997 total_rx_bytes += length; 998 total_rx_packets++; 999 1000 /* code added for copybreak, this should improve 1001 * performance for small packets with large amounts 1002 * of reassembly being done in the stack 1003 */ 1004 if (length < copybreak) { 1005 struct sk_buff *new_skb = 1006 napi_alloc_skb(&adapter->napi, length); 1007 if (new_skb) { 1008 skb_copy_to_linear_data_offset(new_skb, 1009 -NET_IP_ALIGN, 1010 (skb->data - 1011 NET_IP_ALIGN), 1012 (length + 1013 NET_IP_ALIGN)); 1014 /* save the skb in buffer_info as good */ 1015 buffer_info->skb = skb; 1016 skb = new_skb; 1017 } 1018 /* else just continue with the old one */ 1019 } 1020 /* end copybreak code */ 1021 skb_put(skb, length); 1022 1023 /* Receive Checksum Offload */ 1024 e1000_rx_checksum(adapter, staterr, skb); 1025 1026 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb); 1027 1028 e1000_receive_skb(adapter, netdev, skb, staterr, 1029 rx_desc->wb.upper.vlan); 1030 1031 next_desc: 1032 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF); 1033 1034 /* return some buffers to hardware, one at a time is too slow */ 1035 if (cleaned_count >= E1000_RX_BUFFER_WRITE) { 1036 adapter->alloc_rx_buf(rx_ring, cleaned_count, 1037 GFP_ATOMIC); 1038 cleaned_count = 0; 1039 } 1040 1041 /* use prefetched values */ 1042 rx_desc = next_rxd; 1043 buffer_info = next_buffer; 1044 1045 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 1046 } 1047 rx_ring->next_to_clean = i; 1048 1049 cleaned_count = e1000_desc_unused(rx_ring); 1050 if (cleaned_count) 1051 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC); 1052 1053 adapter->total_rx_bytes += total_rx_bytes; 1054 adapter->total_rx_packets += total_rx_packets; 1055 return cleaned; 1056 } 1057 1058 static void e1000_put_txbuf(struct e1000_ring *tx_ring, 1059 struct e1000_buffer *buffer_info, 1060 bool drop) 1061 { 1062 struct e1000_adapter *adapter = tx_ring->adapter; 1063 1064 if (buffer_info->dma) { 1065 if (buffer_info->mapped_as_page) 1066 dma_unmap_page(&adapter->pdev->dev, buffer_info->dma, 1067 buffer_info->length, DMA_TO_DEVICE); 1068 else 1069 dma_unmap_single(&adapter->pdev->dev, buffer_info->dma, 1070 buffer_info->length, DMA_TO_DEVICE); 1071 buffer_info->dma = 0; 1072 } 1073 if (buffer_info->skb) { 1074 if (drop) 1075 dev_kfree_skb_any(buffer_info->skb); 1076 else 1077 dev_consume_skb_any(buffer_info->skb); 1078 buffer_info->skb = NULL; 1079 } 1080 buffer_info->time_stamp = 0; 1081 } 1082 1083 static void e1000_print_hw_hang(struct work_struct *work) 1084 { 1085 struct e1000_adapter *adapter = container_of(work, 1086 struct e1000_adapter, 1087 print_hang_task); 1088 struct net_device *netdev = adapter->netdev; 1089 struct e1000_ring *tx_ring = adapter->tx_ring; 1090 unsigned int i = tx_ring->next_to_clean; 1091 unsigned int eop = tx_ring->buffer_info[i].next_to_watch; 1092 struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop); 1093 struct e1000_hw *hw = &adapter->hw; 1094 u16 phy_status, phy_1000t_status, phy_ext_status; 1095 u16 pci_status; 1096 1097 if (test_bit(__E1000_DOWN, &adapter->state)) 1098 return; 1099 1100 if (!adapter->tx_hang_recheck && (adapter->flags2 & FLAG2_DMA_BURST)) { 1101 /* May be block on write-back, flush and detect again 1102 * flush pending descriptor writebacks to memory 1103 */ 1104 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 1105 /* execute the writes immediately */ 1106 e1e_flush(); 1107 /* Due to rare timing issues, write to TIDV again to ensure 1108 * the write is successful 1109 */ 1110 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 1111 /* execute the writes immediately */ 1112 e1e_flush(); 1113 adapter->tx_hang_recheck = true; 1114 return; 1115 } 1116 adapter->tx_hang_recheck = false; 1117 1118 if (er32(TDH(0)) == er32(TDT(0))) { 1119 e_dbg("false hang detected, ignoring\n"); 1120 return; 1121 } 1122 1123 /* Real hang detected */ 1124 netif_stop_queue(netdev); 1125 1126 e1e_rphy(hw, MII_BMSR, &phy_status); 1127 e1e_rphy(hw, MII_STAT1000, &phy_1000t_status); 1128 e1e_rphy(hw, MII_ESTATUS, &phy_ext_status); 1129 1130 pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status); 1131 1132 /* detected Hardware unit hang */ 1133 e_err("Detected Hardware Unit Hang:\n" 1134 " TDH <%x>\n" 1135 " TDT <%x>\n" 1136 " next_to_use <%x>\n" 1137 " next_to_clean <%x>\n" 1138 "buffer_info[next_to_clean]:\n" 1139 " time_stamp <%lx>\n" 1140 " next_to_watch <%x>\n" 1141 " jiffies <%lx>\n" 1142 " next_to_watch.status <%x>\n" 1143 "MAC Status <%x>\n" 1144 "PHY Status <%x>\n" 1145 "PHY 1000BASE-T Status <%x>\n" 1146 "PHY Extended Status <%x>\n" 1147 "PCI Status <%x>\n", 1148 readl(tx_ring->head), readl(tx_ring->tail), tx_ring->next_to_use, 1149 tx_ring->next_to_clean, tx_ring->buffer_info[eop].time_stamp, 1150 eop, jiffies, eop_desc->upper.fields.status, er32(STATUS), 1151 phy_status, phy_1000t_status, phy_ext_status, pci_status); 1152 1153 e1000e_dump(adapter); 1154 1155 /* Suggest workaround for known h/w issue */ 1156 if ((hw->mac.type == e1000_pchlan) && (er32(CTRL) & E1000_CTRL_TFCE)) 1157 e_err("Try turning off Tx pause (flow control) via ethtool\n"); 1158 } 1159 1160 /** 1161 * e1000e_tx_hwtstamp_work - check for Tx time stamp 1162 * @work: pointer to work struct 1163 * 1164 * This work function polls the TSYNCTXCTL valid bit to determine when a 1165 * timestamp has been taken for the current stored skb. The timestamp must 1166 * be for this skb because only one such packet is allowed in the queue. 1167 */ 1168 static void e1000e_tx_hwtstamp_work(struct work_struct *work) 1169 { 1170 struct e1000_adapter *adapter = container_of(work, struct e1000_adapter, 1171 tx_hwtstamp_work); 1172 struct e1000_hw *hw = &adapter->hw; 1173 1174 if (er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID) { 1175 struct sk_buff *skb = adapter->tx_hwtstamp_skb; 1176 struct skb_shared_hwtstamps shhwtstamps; 1177 u64 txstmp; 1178 1179 txstmp = er32(TXSTMPL); 1180 txstmp |= (u64)er32(TXSTMPH) << 32; 1181 1182 e1000e_systim_to_hwtstamp(adapter, &shhwtstamps, txstmp); 1183 1184 /* Clear the global tx_hwtstamp_skb pointer and force writes 1185 * prior to notifying the stack of a Tx timestamp. 1186 */ 1187 adapter->tx_hwtstamp_skb = NULL; 1188 wmb(); /* force write prior to skb_tstamp_tx */ 1189 1190 skb_tstamp_tx(skb, &shhwtstamps); 1191 dev_consume_skb_any(skb); 1192 } else if (time_after(jiffies, adapter->tx_hwtstamp_start 1193 + adapter->tx_timeout_factor * HZ)) { 1194 dev_kfree_skb_any(adapter->tx_hwtstamp_skb); 1195 adapter->tx_hwtstamp_skb = NULL; 1196 adapter->tx_hwtstamp_timeouts++; 1197 e_warn("clearing Tx timestamp hang\n"); 1198 } else { 1199 /* reschedule to check later */ 1200 schedule_work(&adapter->tx_hwtstamp_work); 1201 } 1202 } 1203 1204 /** 1205 * e1000_clean_tx_irq - Reclaim resources after transmit completes 1206 * @tx_ring: Tx descriptor ring 1207 * 1208 * the return value indicates whether actual cleaning was done, there 1209 * is no guarantee that everything was cleaned 1210 **/ 1211 static bool e1000_clean_tx_irq(struct e1000_ring *tx_ring) 1212 { 1213 struct e1000_adapter *adapter = tx_ring->adapter; 1214 struct net_device *netdev = adapter->netdev; 1215 struct e1000_hw *hw = &adapter->hw; 1216 struct e1000_tx_desc *tx_desc, *eop_desc; 1217 struct e1000_buffer *buffer_info; 1218 unsigned int i, eop; 1219 unsigned int count = 0; 1220 unsigned int total_tx_bytes = 0, total_tx_packets = 0; 1221 unsigned int bytes_compl = 0, pkts_compl = 0; 1222 1223 i = tx_ring->next_to_clean; 1224 eop = tx_ring->buffer_info[i].next_to_watch; 1225 eop_desc = E1000_TX_DESC(*tx_ring, eop); 1226 1227 while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) && 1228 (count < tx_ring->count)) { 1229 bool cleaned = false; 1230 1231 dma_rmb(); /* read buffer_info after eop_desc */ 1232 for (; !cleaned; count++) { 1233 tx_desc = E1000_TX_DESC(*tx_ring, i); 1234 buffer_info = &tx_ring->buffer_info[i]; 1235 cleaned = (i == eop); 1236 1237 if (cleaned) { 1238 total_tx_packets += buffer_info->segs; 1239 total_tx_bytes += buffer_info->bytecount; 1240 if (buffer_info->skb) { 1241 bytes_compl += buffer_info->skb->len; 1242 pkts_compl++; 1243 } 1244 } 1245 1246 e1000_put_txbuf(tx_ring, buffer_info, false); 1247 tx_desc->upper.data = 0; 1248 1249 i++; 1250 if (i == tx_ring->count) 1251 i = 0; 1252 } 1253 1254 if (i == tx_ring->next_to_use) 1255 break; 1256 eop = tx_ring->buffer_info[i].next_to_watch; 1257 eop_desc = E1000_TX_DESC(*tx_ring, eop); 1258 } 1259 1260 tx_ring->next_to_clean = i; 1261 1262 netdev_completed_queue(netdev, pkts_compl, bytes_compl); 1263 1264 #define TX_WAKE_THRESHOLD 32 1265 if (count && netif_carrier_ok(netdev) && 1266 e1000_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD) { 1267 /* Make sure that anybody stopping the queue after this 1268 * sees the new next_to_clean. 1269 */ 1270 smp_mb(); 1271 1272 if (netif_queue_stopped(netdev) && 1273 !(test_bit(__E1000_DOWN, &adapter->state))) { 1274 netif_wake_queue(netdev); 1275 ++adapter->restart_queue; 1276 } 1277 } 1278 1279 if (adapter->detect_tx_hung) { 1280 /* Detect a transmit hang in hardware, this serializes the 1281 * check with the clearing of time_stamp and movement of i 1282 */ 1283 adapter->detect_tx_hung = false; 1284 if (tx_ring->buffer_info[i].time_stamp && 1285 time_after(jiffies, tx_ring->buffer_info[i].time_stamp 1286 + (adapter->tx_timeout_factor * HZ)) && 1287 !(er32(STATUS) & E1000_STATUS_TXOFF)) 1288 schedule_work(&adapter->print_hang_task); 1289 else 1290 adapter->tx_hang_recheck = false; 1291 } 1292 adapter->total_tx_bytes += total_tx_bytes; 1293 adapter->total_tx_packets += total_tx_packets; 1294 return count < tx_ring->count; 1295 } 1296 1297 /** 1298 * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split 1299 * @rx_ring: Rx descriptor ring 1300 * @work_done: output parameter for indicating completed work 1301 * @work_to_do: how many packets we can clean 1302 * 1303 * the return value indicates whether actual cleaning was done, there 1304 * is no guarantee that everything was cleaned 1305 **/ 1306 static bool e1000_clean_rx_irq_ps(struct e1000_ring *rx_ring, int *work_done, 1307 int work_to_do) 1308 { 1309 struct e1000_adapter *adapter = rx_ring->adapter; 1310 struct e1000_hw *hw = &adapter->hw; 1311 union e1000_rx_desc_packet_split *rx_desc, *next_rxd; 1312 struct net_device *netdev = adapter->netdev; 1313 struct pci_dev *pdev = adapter->pdev; 1314 struct e1000_buffer *buffer_info, *next_buffer; 1315 struct e1000_ps_page *ps_page; 1316 struct sk_buff *skb; 1317 unsigned int i, j; 1318 u32 length, staterr; 1319 int cleaned_count = 0; 1320 bool cleaned = false; 1321 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1322 1323 i = rx_ring->next_to_clean; 1324 rx_desc = E1000_RX_DESC_PS(*rx_ring, i); 1325 staterr = le32_to_cpu(rx_desc->wb.middle.status_error); 1326 buffer_info = &rx_ring->buffer_info[i]; 1327 1328 while (staterr & E1000_RXD_STAT_DD) { 1329 if (*work_done >= work_to_do) 1330 break; 1331 (*work_done)++; 1332 skb = buffer_info->skb; 1333 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ 1334 1335 /* in the packet split case this is header only */ 1336 prefetch(skb->data - NET_IP_ALIGN); 1337 1338 i++; 1339 if (i == rx_ring->count) 1340 i = 0; 1341 next_rxd = E1000_RX_DESC_PS(*rx_ring, i); 1342 prefetch(next_rxd); 1343 1344 next_buffer = &rx_ring->buffer_info[i]; 1345 1346 cleaned = true; 1347 cleaned_count++; 1348 dma_unmap_single(&pdev->dev, buffer_info->dma, 1349 adapter->rx_ps_bsize0, DMA_FROM_DEVICE); 1350 buffer_info->dma = 0; 1351 1352 /* see !EOP comment in other Rx routine */ 1353 if (!(staterr & E1000_RXD_STAT_EOP)) 1354 adapter->flags2 |= FLAG2_IS_DISCARDING; 1355 1356 if (adapter->flags2 & FLAG2_IS_DISCARDING) { 1357 e_dbg("Packet Split buffers didn't pick up the full packet\n"); 1358 dev_kfree_skb_irq(skb); 1359 if (staterr & E1000_RXD_STAT_EOP) 1360 adapter->flags2 &= ~FLAG2_IS_DISCARDING; 1361 goto next_desc; 1362 } 1363 1364 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) && 1365 !(netdev->features & NETIF_F_RXALL))) { 1366 dev_kfree_skb_irq(skb); 1367 goto next_desc; 1368 } 1369 1370 length = le16_to_cpu(rx_desc->wb.middle.length0); 1371 1372 if (!length) { 1373 e_dbg("Last part of the packet spanning multiple descriptors\n"); 1374 dev_kfree_skb_irq(skb); 1375 goto next_desc; 1376 } 1377 1378 /* Good Receive */ 1379 skb_put(skb, length); 1380 1381 { 1382 /* this looks ugly, but it seems compiler issues make 1383 * it more efficient than reusing j 1384 */ 1385 int l1 = le16_to_cpu(rx_desc->wb.upper.length[0]); 1386 1387 /* page alloc/put takes too long and effects small 1388 * packet throughput, so unsplit small packets and 1389 * save the alloc/put only valid in softirq (napi) 1390 * context to call kmap_* 1391 */ 1392 if (l1 && (l1 <= copybreak) && 1393 ((length + l1) <= adapter->rx_ps_bsize0)) { 1394 u8 *vaddr; 1395 1396 ps_page = &buffer_info->ps_pages[0]; 1397 1398 /* there is no documentation about how to call 1399 * kmap_atomic, so we can't hold the mapping 1400 * very long 1401 */ 1402 dma_sync_single_for_cpu(&pdev->dev, 1403 ps_page->dma, 1404 PAGE_SIZE, 1405 DMA_FROM_DEVICE); 1406 vaddr = kmap_atomic(ps_page->page); 1407 memcpy(skb_tail_pointer(skb), vaddr, l1); 1408 kunmap_atomic(vaddr); 1409 dma_sync_single_for_device(&pdev->dev, 1410 ps_page->dma, 1411 PAGE_SIZE, 1412 DMA_FROM_DEVICE); 1413 1414 /* remove the CRC */ 1415 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) { 1416 if (!(netdev->features & NETIF_F_RXFCS)) 1417 l1 -= 4; 1418 } 1419 1420 skb_put(skb, l1); 1421 goto copydone; 1422 } /* if */ 1423 } 1424 1425 for (j = 0; j < PS_PAGE_BUFFERS; j++) { 1426 length = le16_to_cpu(rx_desc->wb.upper.length[j]); 1427 if (!length) 1428 break; 1429 1430 ps_page = &buffer_info->ps_pages[j]; 1431 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE, 1432 DMA_FROM_DEVICE); 1433 ps_page->dma = 0; 1434 skb_fill_page_desc(skb, j, ps_page->page, 0, length); 1435 ps_page->page = NULL; 1436 skb->len += length; 1437 skb->data_len += length; 1438 skb->truesize += PAGE_SIZE; 1439 } 1440 1441 /* strip the ethernet crc, problem is we're using pages now so 1442 * this whole operation can get a little cpu intensive 1443 */ 1444 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) { 1445 if (!(netdev->features & NETIF_F_RXFCS)) 1446 pskb_trim(skb, skb->len - 4); 1447 } 1448 1449 copydone: 1450 total_rx_bytes += skb->len; 1451 total_rx_packets++; 1452 1453 e1000_rx_checksum(adapter, staterr, skb); 1454 1455 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb); 1456 1457 if (rx_desc->wb.upper.header_status & 1458 cpu_to_le16(E1000_RXDPS_HDRSTAT_HDRSP)) 1459 adapter->rx_hdr_split++; 1460 1461 e1000_receive_skb(adapter, netdev, skb, staterr, 1462 rx_desc->wb.middle.vlan); 1463 1464 next_desc: 1465 rx_desc->wb.middle.status_error &= cpu_to_le32(~0xFF); 1466 buffer_info->skb = NULL; 1467 1468 /* return some buffers to hardware, one at a time is too slow */ 1469 if (cleaned_count >= E1000_RX_BUFFER_WRITE) { 1470 adapter->alloc_rx_buf(rx_ring, cleaned_count, 1471 GFP_ATOMIC); 1472 cleaned_count = 0; 1473 } 1474 1475 /* use prefetched values */ 1476 rx_desc = next_rxd; 1477 buffer_info = next_buffer; 1478 1479 staterr = le32_to_cpu(rx_desc->wb.middle.status_error); 1480 } 1481 rx_ring->next_to_clean = i; 1482 1483 cleaned_count = e1000_desc_unused(rx_ring); 1484 if (cleaned_count) 1485 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC); 1486 1487 adapter->total_rx_bytes += total_rx_bytes; 1488 adapter->total_rx_packets += total_rx_packets; 1489 return cleaned; 1490 } 1491 1492 static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb, 1493 u16 length) 1494 { 1495 bi->page = NULL; 1496 skb->len += length; 1497 skb->data_len += length; 1498 skb->truesize += PAGE_SIZE; 1499 } 1500 1501 /** 1502 * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy 1503 * @rx_ring: Rx descriptor ring 1504 * @work_done: output parameter for indicating completed work 1505 * @work_to_do: how many packets we can clean 1506 * 1507 * the return value indicates whether actual cleaning was done, there 1508 * is no guarantee that everything was cleaned 1509 **/ 1510 static bool e1000_clean_jumbo_rx_irq(struct e1000_ring *rx_ring, int *work_done, 1511 int work_to_do) 1512 { 1513 struct e1000_adapter *adapter = rx_ring->adapter; 1514 struct net_device *netdev = adapter->netdev; 1515 struct pci_dev *pdev = adapter->pdev; 1516 union e1000_rx_desc_extended *rx_desc, *next_rxd; 1517 struct e1000_buffer *buffer_info, *next_buffer; 1518 u32 length, staterr; 1519 unsigned int i; 1520 int cleaned_count = 0; 1521 bool cleaned = false; 1522 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1523 struct skb_shared_info *shinfo; 1524 1525 i = rx_ring->next_to_clean; 1526 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 1527 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 1528 buffer_info = &rx_ring->buffer_info[i]; 1529 1530 while (staterr & E1000_RXD_STAT_DD) { 1531 struct sk_buff *skb; 1532 1533 if (*work_done >= work_to_do) 1534 break; 1535 (*work_done)++; 1536 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ 1537 1538 skb = buffer_info->skb; 1539 buffer_info->skb = NULL; 1540 1541 ++i; 1542 if (i == rx_ring->count) 1543 i = 0; 1544 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i); 1545 prefetch(next_rxd); 1546 1547 next_buffer = &rx_ring->buffer_info[i]; 1548 1549 cleaned = true; 1550 cleaned_count++; 1551 dma_unmap_page(&pdev->dev, buffer_info->dma, PAGE_SIZE, 1552 DMA_FROM_DEVICE); 1553 buffer_info->dma = 0; 1554 1555 length = le16_to_cpu(rx_desc->wb.upper.length); 1556 1557 /* errors is only valid for DD + EOP descriptors */ 1558 if (unlikely((staterr & E1000_RXD_STAT_EOP) && 1559 ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) && 1560 !(netdev->features & NETIF_F_RXALL)))) { 1561 /* recycle both page and skb */ 1562 buffer_info->skb = skb; 1563 /* an error means any chain goes out the window too */ 1564 if (rx_ring->rx_skb_top) 1565 dev_kfree_skb_irq(rx_ring->rx_skb_top); 1566 rx_ring->rx_skb_top = NULL; 1567 goto next_desc; 1568 } 1569 #define rxtop (rx_ring->rx_skb_top) 1570 if (!(staterr & E1000_RXD_STAT_EOP)) { 1571 /* this descriptor is only the beginning (or middle) */ 1572 if (!rxtop) { 1573 /* this is the beginning of a chain */ 1574 rxtop = skb; 1575 skb_fill_page_desc(rxtop, 0, buffer_info->page, 1576 0, length); 1577 } else { 1578 /* this is the middle of a chain */ 1579 shinfo = skb_shinfo(rxtop); 1580 skb_fill_page_desc(rxtop, shinfo->nr_frags, 1581 buffer_info->page, 0, 1582 length); 1583 /* re-use the skb, only consumed the page */ 1584 buffer_info->skb = skb; 1585 } 1586 e1000_consume_page(buffer_info, rxtop, length); 1587 goto next_desc; 1588 } else { 1589 if (rxtop) { 1590 /* end of the chain */ 1591 shinfo = skb_shinfo(rxtop); 1592 skb_fill_page_desc(rxtop, shinfo->nr_frags, 1593 buffer_info->page, 0, 1594 length); 1595 /* re-use the current skb, we only consumed the 1596 * page 1597 */ 1598 buffer_info->skb = skb; 1599 skb = rxtop; 1600 rxtop = NULL; 1601 e1000_consume_page(buffer_info, skb, length); 1602 } else { 1603 /* no chain, got EOP, this buf is the packet 1604 * copybreak to save the put_page/alloc_page 1605 */ 1606 if (length <= copybreak && 1607 skb_tailroom(skb) >= length) { 1608 u8 *vaddr; 1609 vaddr = kmap_atomic(buffer_info->page); 1610 memcpy(skb_tail_pointer(skb), vaddr, 1611 length); 1612 kunmap_atomic(vaddr); 1613 /* re-use the page, so don't erase 1614 * buffer_info->page 1615 */ 1616 skb_put(skb, length); 1617 } else { 1618 skb_fill_page_desc(skb, 0, 1619 buffer_info->page, 0, 1620 length); 1621 e1000_consume_page(buffer_info, skb, 1622 length); 1623 } 1624 } 1625 } 1626 1627 /* Receive Checksum Offload */ 1628 e1000_rx_checksum(adapter, staterr, skb); 1629 1630 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb); 1631 1632 /* probably a little skewed due to removing CRC */ 1633 total_rx_bytes += skb->len; 1634 total_rx_packets++; 1635 1636 /* eth type trans needs skb->data to point to something */ 1637 if (!pskb_may_pull(skb, ETH_HLEN)) { 1638 e_err("pskb_may_pull failed.\n"); 1639 dev_kfree_skb_irq(skb); 1640 goto next_desc; 1641 } 1642 1643 e1000_receive_skb(adapter, netdev, skb, staterr, 1644 rx_desc->wb.upper.vlan); 1645 1646 next_desc: 1647 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF); 1648 1649 /* return some buffers to hardware, one at a time is too slow */ 1650 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) { 1651 adapter->alloc_rx_buf(rx_ring, cleaned_count, 1652 GFP_ATOMIC); 1653 cleaned_count = 0; 1654 } 1655 1656 /* use prefetched values */ 1657 rx_desc = next_rxd; 1658 buffer_info = next_buffer; 1659 1660 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 1661 } 1662 rx_ring->next_to_clean = i; 1663 1664 cleaned_count = e1000_desc_unused(rx_ring); 1665 if (cleaned_count) 1666 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC); 1667 1668 adapter->total_rx_bytes += total_rx_bytes; 1669 adapter->total_rx_packets += total_rx_packets; 1670 return cleaned; 1671 } 1672 1673 /** 1674 * e1000_clean_rx_ring - Free Rx Buffers per Queue 1675 * @rx_ring: Rx descriptor ring 1676 **/ 1677 static void e1000_clean_rx_ring(struct e1000_ring *rx_ring) 1678 { 1679 struct e1000_adapter *adapter = rx_ring->adapter; 1680 struct e1000_buffer *buffer_info; 1681 struct e1000_ps_page *ps_page; 1682 struct pci_dev *pdev = adapter->pdev; 1683 unsigned int i, j; 1684 1685 /* Free all the Rx ring sk_buffs */ 1686 for (i = 0; i < rx_ring->count; i++) { 1687 buffer_info = &rx_ring->buffer_info[i]; 1688 if (buffer_info->dma) { 1689 if (adapter->clean_rx == e1000_clean_rx_irq) 1690 dma_unmap_single(&pdev->dev, buffer_info->dma, 1691 adapter->rx_buffer_len, 1692 DMA_FROM_DEVICE); 1693 else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) 1694 dma_unmap_page(&pdev->dev, buffer_info->dma, 1695 PAGE_SIZE, DMA_FROM_DEVICE); 1696 else if (adapter->clean_rx == e1000_clean_rx_irq_ps) 1697 dma_unmap_single(&pdev->dev, buffer_info->dma, 1698 adapter->rx_ps_bsize0, 1699 DMA_FROM_DEVICE); 1700 buffer_info->dma = 0; 1701 } 1702 1703 if (buffer_info->page) { 1704 put_page(buffer_info->page); 1705 buffer_info->page = NULL; 1706 } 1707 1708 if (buffer_info->skb) { 1709 dev_kfree_skb(buffer_info->skb); 1710 buffer_info->skb = NULL; 1711 } 1712 1713 for (j = 0; j < PS_PAGE_BUFFERS; j++) { 1714 ps_page = &buffer_info->ps_pages[j]; 1715 if (!ps_page->page) 1716 break; 1717 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE, 1718 DMA_FROM_DEVICE); 1719 ps_page->dma = 0; 1720 put_page(ps_page->page); 1721 ps_page->page = NULL; 1722 } 1723 } 1724 1725 /* there also may be some cached data from a chained receive */ 1726 if (rx_ring->rx_skb_top) { 1727 dev_kfree_skb(rx_ring->rx_skb_top); 1728 rx_ring->rx_skb_top = NULL; 1729 } 1730 1731 /* Zero out the descriptor ring */ 1732 memset(rx_ring->desc, 0, rx_ring->size); 1733 1734 rx_ring->next_to_clean = 0; 1735 rx_ring->next_to_use = 0; 1736 adapter->flags2 &= ~FLAG2_IS_DISCARDING; 1737 } 1738 1739 static void e1000e_downshift_workaround(struct work_struct *work) 1740 { 1741 struct e1000_adapter *adapter = container_of(work, 1742 struct e1000_adapter, 1743 downshift_task); 1744 1745 if (test_bit(__E1000_DOWN, &adapter->state)) 1746 return; 1747 1748 e1000e_gig_downshift_workaround_ich8lan(&adapter->hw); 1749 } 1750 1751 /** 1752 * e1000_intr_msi - Interrupt Handler 1753 * @irq: interrupt number 1754 * @data: pointer to a network interface device structure 1755 **/ 1756 static irqreturn_t e1000_intr_msi(int __always_unused irq, void *data) 1757 { 1758 struct net_device *netdev = data; 1759 struct e1000_adapter *adapter = netdev_priv(netdev); 1760 struct e1000_hw *hw = &adapter->hw; 1761 u32 icr = er32(ICR); 1762 1763 /* read ICR disables interrupts using IAM */ 1764 if (icr & E1000_ICR_LSC) { 1765 hw->mac.get_link_status = true; 1766 /* ICH8 workaround-- Call gig speed drop workaround on cable 1767 * disconnect (LSC) before accessing any PHY registers 1768 */ 1769 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) && 1770 (!(er32(STATUS) & E1000_STATUS_LU))) 1771 schedule_work(&adapter->downshift_task); 1772 1773 /* 80003ES2LAN workaround-- For packet buffer work-around on 1774 * link down event; disable receives here in the ISR and reset 1775 * adapter in watchdog 1776 */ 1777 if (netif_carrier_ok(netdev) && 1778 adapter->flags & FLAG_RX_NEEDS_RESTART) { 1779 /* disable receives */ 1780 u32 rctl = er32(RCTL); 1781 1782 ew32(RCTL, rctl & ~E1000_RCTL_EN); 1783 adapter->flags |= FLAG_RESTART_NOW; 1784 } 1785 /* guard against interrupt when we're going down */ 1786 if (!test_bit(__E1000_DOWN, &adapter->state)) 1787 mod_timer(&adapter->watchdog_timer, jiffies + 1); 1788 } 1789 1790 /* Reset on uncorrectable ECC error */ 1791 if ((icr & E1000_ICR_ECCER) && (hw->mac.type >= e1000_pch_lpt)) { 1792 u32 pbeccsts = er32(PBECCSTS); 1793 1794 adapter->corr_errors += 1795 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 1796 adapter->uncorr_errors += 1797 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >> 1798 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT; 1799 1800 /* Do the reset outside of interrupt context */ 1801 schedule_work(&adapter->reset_task); 1802 1803 /* return immediately since reset is imminent */ 1804 return IRQ_HANDLED; 1805 } 1806 1807 if (napi_schedule_prep(&adapter->napi)) { 1808 adapter->total_tx_bytes = 0; 1809 adapter->total_tx_packets = 0; 1810 adapter->total_rx_bytes = 0; 1811 adapter->total_rx_packets = 0; 1812 __napi_schedule(&adapter->napi); 1813 } 1814 1815 return IRQ_HANDLED; 1816 } 1817 1818 /** 1819 * e1000_intr - Interrupt Handler 1820 * @irq: interrupt number 1821 * @data: pointer to a network interface device structure 1822 **/ 1823 static irqreturn_t e1000_intr(int __always_unused irq, void *data) 1824 { 1825 struct net_device *netdev = data; 1826 struct e1000_adapter *adapter = netdev_priv(netdev); 1827 struct e1000_hw *hw = &adapter->hw; 1828 u32 rctl, icr = er32(ICR); 1829 1830 if (!icr || test_bit(__E1000_DOWN, &adapter->state)) 1831 return IRQ_NONE; /* Not our interrupt */ 1832 1833 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 1834 * not set, then the adapter didn't send an interrupt 1835 */ 1836 if (!(icr & E1000_ICR_INT_ASSERTED)) 1837 return IRQ_NONE; 1838 1839 /* Interrupt Auto-Mask...upon reading ICR, 1840 * interrupts are masked. No need for the 1841 * IMC write 1842 */ 1843 1844 if (icr & E1000_ICR_LSC) { 1845 hw->mac.get_link_status = true; 1846 /* ICH8 workaround-- Call gig speed drop workaround on cable 1847 * disconnect (LSC) before accessing any PHY registers 1848 */ 1849 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) && 1850 (!(er32(STATUS) & E1000_STATUS_LU))) 1851 schedule_work(&adapter->downshift_task); 1852 1853 /* 80003ES2LAN workaround-- 1854 * For packet buffer work-around on link down event; 1855 * disable receives here in the ISR and 1856 * reset adapter in watchdog 1857 */ 1858 if (netif_carrier_ok(netdev) && 1859 (adapter->flags & FLAG_RX_NEEDS_RESTART)) { 1860 /* disable receives */ 1861 rctl = er32(RCTL); 1862 ew32(RCTL, rctl & ~E1000_RCTL_EN); 1863 adapter->flags |= FLAG_RESTART_NOW; 1864 } 1865 /* guard against interrupt when we're going down */ 1866 if (!test_bit(__E1000_DOWN, &adapter->state)) 1867 mod_timer(&adapter->watchdog_timer, jiffies + 1); 1868 } 1869 1870 /* Reset on uncorrectable ECC error */ 1871 if ((icr & E1000_ICR_ECCER) && (hw->mac.type >= e1000_pch_lpt)) { 1872 u32 pbeccsts = er32(PBECCSTS); 1873 1874 adapter->corr_errors += 1875 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 1876 adapter->uncorr_errors += 1877 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >> 1878 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT; 1879 1880 /* Do the reset outside of interrupt context */ 1881 schedule_work(&adapter->reset_task); 1882 1883 /* return immediately since reset is imminent */ 1884 return IRQ_HANDLED; 1885 } 1886 1887 if (napi_schedule_prep(&adapter->napi)) { 1888 adapter->total_tx_bytes = 0; 1889 adapter->total_tx_packets = 0; 1890 adapter->total_rx_bytes = 0; 1891 adapter->total_rx_packets = 0; 1892 __napi_schedule(&adapter->napi); 1893 } 1894 1895 return IRQ_HANDLED; 1896 } 1897 1898 static irqreturn_t e1000_msix_other(int __always_unused irq, void *data) 1899 { 1900 struct net_device *netdev = data; 1901 struct e1000_adapter *adapter = netdev_priv(netdev); 1902 struct e1000_hw *hw = &adapter->hw; 1903 u32 icr = er32(ICR); 1904 1905 if (icr & adapter->eiac_mask) 1906 ew32(ICS, (icr & adapter->eiac_mask)); 1907 1908 if (icr & E1000_ICR_LSC) { 1909 hw->mac.get_link_status = true; 1910 /* guard against interrupt when we're going down */ 1911 if (!test_bit(__E1000_DOWN, &adapter->state)) 1912 mod_timer(&adapter->watchdog_timer, jiffies + 1); 1913 } 1914 1915 if (!test_bit(__E1000_DOWN, &adapter->state)) 1916 ew32(IMS, E1000_IMS_OTHER | IMS_OTHER_MASK); 1917 1918 return IRQ_HANDLED; 1919 } 1920 1921 static irqreturn_t e1000_intr_msix_tx(int __always_unused irq, void *data) 1922 { 1923 struct net_device *netdev = data; 1924 struct e1000_adapter *adapter = netdev_priv(netdev); 1925 struct e1000_hw *hw = &adapter->hw; 1926 struct e1000_ring *tx_ring = adapter->tx_ring; 1927 1928 adapter->total_tx_bytes = 0; 1929 adapter->total_tx_packets = 0; 1930 1931 if (!e1000_clean_tx_irq(tx_ring)) 1932 /* Ring was not completely cleaned, so fire another interrupt */ 1933 ew32(ICS, tx_ring->ims_val); 1934 1935 if (!test_bit(__E1000_DOWN, &adapter->state)) 1936 ew32(IMS, adapter->tx_ring->ims_val); 1937 1938 return IRQ_HANDLED; 1939 } 1940 1941 static irqreturn_t e1000_intr_msix_rx(int __always_unused irq, void *data) 1942 { 1943 struct net_device *netdev = data; 1944 struct e1000_adapter *adapter = netdev_priv(netdev); 1945 struct e1000_ring *rx_ring = adapter->rx_ring; 1946 1947 /* Write the ITR value calculated at the end of the 1948 * previous interrupt. 1949 */ 1950 if (rx_ring->set_itr) { 1951 u32 itr = rx_ring->itr_val ? 1952 1000000000 / (rx_ring->itr_val * 256) : 0; 1953 1954 writel(itr, rx_ring->itr_register); 1955 rx_ring->set_itr = 0; 1956 } 1957 1958 if (napi_schedule_prep(&adapter->napi)) { 1959 adapter->total_rx_bytes = 0; 1960 adapter->total_rx_packets = 0; 1961 __napi_schedule(&adapter->napi); 1962 } 1963 return IRQ_HANDLED; 1964 } 1965 1966 /** 1967 * e1000_configure_msix - Configure MSI-X hardware 1968 * @adapter: board private structure 1969 * 1970 * e1000_configure_msix sets up the hardware to properly 1971 * generate MSI-X interrupts. 1972 **/ 1973 static void e1000_configure_msix(struct e1000_adapter *adapter) 1974 { 1975 struct e1000_hw *hw = &adapter->hw; 1976 struct e1000_ring *rx_ring = adapter->rx_ring; 1977 struct e1000_ring *tx_ring = adapter->tx_ring; 1978 int vector = 0; 1979 u32 ctrl_ext, ivar = 0; 1980 1981 adapter->eiac_mask = 0; 1982 1983 /* Workaround issue with spurious interrupts on 82574 in MSI-X mode */ 1984 if (hw->mac.type == e1000_82574) { 1985 u32 rfctl = er32(RFCTL); 1986 1987 rfctl |= E1000_RFCTL_ACK_DIS; 1988 ew32(RFCTL, rfctl); 1989 } 1990 1991 /* Configure Rx vector */ 1992 rx_ring->ims_val = E1000_IMS_RXQ0; 1993 adapter->eiac_mask |= rx_ring->ims_val; 1994 if (rx_ring->itr_val) 1995 writel(1000000000 / (rx_ring->itr_val * 256), 1996 rx_ring->itr_register); 1997 else 1998 writel(1, rx_ring->itr_register); 1999 ivar = E1000_IVAR_INT_ALLOC_VALID | vector; 2000 2001 /* Configure Tx vector */ 2002 tx_ring->ims_val = E1000_IMS_TXQ0; 2003 vector++; 2004 if (tx_ring->itr_val) 2005 writel(1000000000 / (tx_ring->itr_val * 256), 2006 tx_ring->itr_register); 2007 else 2008 writel(1, tx_ring->itr_register); 2009 adapter->eiac_mask |= tx_ring->ims_val; 2010 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 8); 2011 2012 /* set vector for Other Causes, e.g. link changes */ 2013 vector++; 2014 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 16); 2015 if (rx_ring->itr_val) 2016 writel(1000000000 / (rx_ring->itr_val * 256), 2017 hw->hw_addr + E1000_EITR_82574(vector)); 2018 else 2019 writel(1, hw->hw_addr + E1000_EITR_82574(vector)); 2020 2021 /* Cause Tx interrupts on every write back */ 2022 ivar |= BIT(31); 2023 2024 ew32(IVAR, ivar); 2025 2026 /* enable MSI-X PBA support */ 2027 ctrl_ext = er32(CTRL_EXT) & ~E1000_CTRL_EXT_IAME; 2028 ctrl_ext |= E1000_CTRL_EXT_PBA_CLR | E1000_CTRL_EXT_EIAME; 2029 ew32(CTRL_EXT, ctrl_ext); 2030 e1e_flush(); 2031 } 2032 2033 void e1000e_reset_interrupt_capability(struct e1000_adapter *adapter) 2034 { 2035 if (adapter->msix_entries) { 2036 pci_disable_msix(adapter->pdev); 2037 kfree(adapter->msix_entries); 2038 adapter->msix_entries = NULL; 2039 } else if (adapter->flags & FLAG_MSI_ENABLED) { 2040 pci_disable_msi(adapter->pdev); 2041 adapter->flags &= ~FLAG_MSI_ENABLED; 2042 } 2043 } 2044 2045 /** 2046 * e1000e_set_interrupt_capability - set MSI or MSI-X if supported 2047 * @adapter: board private structure 2048 * 2049 * Attempt to configure interrupts using the best available 2050 * capabilities of the hardware and kernel. 2051 **/ 2052 void e1000e_set_interrupt_capability(struct e1000_adapter *adapter) 2053 { 2054 int err; 2055 int i; 2056 2057 switch (adapter->int_mode) { 2058 case E1000E_INT_MODE_MSIX: 2059 if (adapter->flags & FLAG_HAS_MSIX) { 2060 adapter->num_vectors = 3; /* RxQ0, TxQ0 and other */ 2061 adapter->msix_entries = kcalloc(adapter->num_vectors, 2062 sizeof(struct 2063 msix_entry), 2064 GFP_KERNEL); 2065 if (adapter->msix_entries) { 2066 struct e1000_adapter *a = adapter; 2067 2068 for (i = 0; i < adapter->num_vectors; i++) 2069 adapter->msix_entries[i].entry = i; 2070 2071 err = pci_enable_msix_range(a->pdev, 2072 a->msix_entries, 2073 a->num_vectors, 2074 a->num_vectors); 2075 if (err > 0) 2076 return; 2077 } 2078 /* MSI-X failed, so fall through and try MSI */ 2079 e_err("Failed to initialize MSI-X interrupts. Falling back to MSI interrupts.\n"); 2080 e1000e_reset_interrupt_capability(adapter); 2081 } 2082 adapter->int_mode = E1000E_INT_MODE_MSI; 2083 fallthrough; 2084 case E1000E_INT_MODE_MSI: 2085 if (!pci_enable_msi(adapter->pdev)) { 2086 adapter->flags |= FLAG_MSI_ENABLED; 2087 } else { 2088 adapter->int_mode = E1000E_INT_MODE_LEGACY; 2089 e_err("Failed to initialize MSI interrupts. Falling back to legacy interrupts.\n"); 2090 } 2091 fallthrough; 2092 case E1000E_INT_MODE_LEGACY: 2093 /* Don't do anything; this is the system default */ 2094 break; 2095 } 2096 2097 /* store the number of vectors being used */ 2098 adapter->num_vectors = 1; 2099 } 2100 2101 /** 2102 * e1000_request_msix - Initialize MSI-X interrupts 2103 * @adapter: board private structure 2104 * 2105 * e1000_request_msix allocates MSI-X vectors and requests interrupts from the 2106 * kernel. 2107 **/ 2108 static int e1000_request_msix(struct e1000_adapter *adapter) 2109 { 2110 struct net_device *netdev = adapter->netdev; 2111 int err = 0, vector = 0; 2112 2113 if (strlen(netdev->name) < (IFNAMSIZ - 5)) 2114 snprintf(adapter->rx_ring->name, 2115 sizeof(adapter->rx_ring->name) - 1, 2116 "%.14s-rx-0", netdev->name); 2117 else 2118 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ); 2119 err = request_irq(adapter->msix_entries[vector].vector, 2120 e1000_intr_msix_rx, 0, adapter->rx_ring->name, 2121 netdev); 2122 if (err) 2123 return err; 2124 adapter->rx_ring->itr_register = adapter->hw.hw_addr + 2125 E1000_EITR_82574(vector); 2126 adapter->rx_ring->itr_val = adapter->itr; 2127 vector++; 2128 2129 if (strlen(netdev->name) < (IFNAMSIZ - 5)) 2130 snprintf(adapter->tx_ring->name, 2131 sizeof(adapter->tx_ring->name) - 1, 2132 "%.14s-tx-0", netdev->name); 2133 else 2134 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ); 2135 err = request_irq(adapter->msix_entries[vector].vector, 2136 e1000_intr_msix_tx, 0, adapter->tx_ring->name, 2137 netdev); 2138 if (err) 2139 return err; 2140 adapter->tx_ring->itr_register = adapter->hw.hw_addr + 2141 E1000_EITR_82574(vector); 2142 adapter->tx_ring->itr_val = adapter->itr; 2143 vector++; 2144 2145 err = request_irq(adapter->msix_entries[vector].vector, 2146 e1000_msix_other, 0, netdev->name, netdev); 2147 if (err) 2148 return err; 2149 2150 e1000_configure_msix(adapter); 2151 2152 return 0; 2153 } 2154 2155 /** 2156 * e1000_request_irq - initialize interrupts 2157 * @adapter: board private structure 2158 * 2159 * Attempts to configure interrupts using the best available 2160 * capabilities of the hardware and kernel. 2161 **/ 2162 static int e1000_request_irq(struct e1000_adapter *adapter) 2163 { 2164 struct net_device *netdev = adapter->netdev; 2165 int err; 2166 2167 if (adapter->msix_entries) { 2168 err = e1000_request_msix(adapter); 2169 if (!err) 2170 return err; 2171 /* fall back to MSI */ 2172 e1000e_reset_interrupt_capability(adapter); 2173 adapter->int_mode = E1000E_INT_MODE_MSI; 2174 e1000e_set_interrupt_capability(adapter); 2175 } 2176 if (adapter->flags & FLAG_MSI_ENABLED) { 2177 err = request_irq(adapter->pdev->irq, e1000_intr_msi, 0, 2178 netdev->name, netdev); 2179 if (!err) 2180 return err; 2181 2182 /* fall back to legacy interrupt */ 2183 e1000e_reset_interrupt_capability(adapter); 2184 adapter->int_mode = E1000E_INT_MODE_LEGACY; 2185 } 2186 2187 err = request_irq(adapter->pdev->irq, e1000_intr, IRQF_SHARED, 2188 netdev->name, netdev); 2189 if (err) 2190 e_err("Unable to allocate interrupt, Error: %d\n", err); 2191 2192 return err; 2193 } 2194 2195 static void e1000_free_irq(struct e1000_adapter *adapter) 2196 { 2197 struct net_device *netdev = adapter->netdev; 2198 2199 if (adapter->msix_entries) { 2200 int vector = 0; 2201 2202 free_irq(adapter->msix_entries[vector].vector, netdev); 2203 vector++; 2204 2205 free_irq(adapter->msix_entries[vector].vector, netdev); 2206 vector++; 2207 2208 /* Other Causes interrupt vector */ 2209 free_irq(adapter->msix_entries[vector].vector, netdev); 2210 return; 2211 } 2212 2213 free_irq(adapter->pdev->irq, netdev); 2214 } 2215 2216 /** 2217 * e1000_irq_disable - Mask off interrupt generation on the NIC 2218 * @adapter: board private structure 2219 **/ 2220 static void e1000_irq_disable(struct e1000_adapter *adapter) 2221 { 2222 struct e1000_hw *hw = &adapter->hw; 2223 2224 ew32(IMC, ~0); 2225 if (adapter->msix_entries) 2226 ew32(EIAC_82574, 0); 2227 e1e_flush(); 2228 2229 if (adapter->msix_entries) { 2230 int i; 2231 2232 for (i = 0; i < adapter->num_vectors; i++) 2233 synchronize_irq(adapter->msix_entries[i].vector); 2234 } else { 2235 synchronize_irq(adapter->pdev->irq); 2236 } 2237 } 2238 2239 /** 2240 * e1000_irq_enable - Enable default interrupt generation settings 2241 * @adapter: board private structure 2242 **/ 2243 static void e1000_irq_enable(struct e1000_adapter *adapter) 2244 { 2245 struct e1000_hw *hw = &adapter->hw; 2246 2247 if (adapter->msix_entries) { 2248 ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574); 2249 ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER | 2250 IMS_OTHER_MASK); 2251 } else if (hw->mac.type >= e1000_pch_lpt) { 2252 ew32(IMS, IMS_ENABLE_MASK | E1000_IMS_ECCER); 2253 } else { 2254 ew32(IMS, IMS_ENABLE_MASK); 2255 } 2256 e1e_flush(); 2257 } 2258 2259 /** 2260 * e1000e_get_hw_control - get control of the h/w from f/w 2261 * @adapter: address of board private structure 2262 * 2263 * e1000e_get_hw_control sets {CTRL_EXT|SWSM}:DRV_LOAD bit. 2264 * For ASF and Pass Through versions of f/w this means that 2265 * the driver is loaded. For AMT version (only with 82573) 2266 * of the f/w this means that the network i/f is open. 2267 **/ 2268 void e1000e_get_hw_control(struct e1000_adapter *adapter) 2269 { 2270 struct e1000_hw *hw = &adapter->hw; 2271 u32 ctrl_ext; 2272 u32 swsm; 2273 2274 /* Let firmware know the driver has taken over */ 2275 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) { 2276 swsm = er32(SWSM); 2277 ew32(SWSM, swsm | E1000_SWSM_DRV_LOAD); 2278 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) { 2279 ctrl_ext = er32(CTRL_EXT); 2280 ew32(CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2281 } 2282 } 2283 2284 /** 2285 * e1000e_release_hw_control - release control of the h/w to f/w 2286 * @adapter: address of board private structure 2287 * 2288 * e1000e_release_hw_control resets {CTRL_EXT|SWSM}:DRV_LOAD bit. 2289 * For ASF and Pass Through versions of f/w this means that the 2290 * driver is no longer loaded. For AMT version (only with 82573) i 2291 * of the f/w this means that the network i/f is closed. 2292 * 2293 **/ 2294 void e1000e_release_hw_control(struct e1000_adapter *adapter) 2295 { 2296 struct e1000_hw *hw = &adapter->hw; 2297 u32 ctrl_ext; 2298 u32 swsm; 2299 2300 /* Let firmware taken over control of h/w */ 2301 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) { 2302 swsm = er32(SWSM); 2303 ew32(SWSM, swsm & ~E1000_SWSM_DRV_LOAD); 2304 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) { 2305 ctrl_ext = er32(CTRL_EXT); 2306 ew32(CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2307 } 2308 } 2309 2310 /** 2311 * e1000_alloc_ring_dma - allocate memory for a ring structure 2312 * @adapter: board private structure 2313 * @ring: ring struct for which to allocate dma 2314 **/ 2315 static int e1000_alloc_ring_dma(struct e1000_adapter *adapter, 2316 struct e1000_ring *ring) 2317 { 2318 struct pci_dev *pdev = adapter->pdev; 2319 2320 ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma, 2321 GFP_KERNEL); 2322 if (!ring->desc) 2323 return -ENOMEM; 2324 2325 return 0; 2326 } 2327 2328 /** 2329 * e1000e_setup_tx_resources - allocate Tx resources (Descriptors) 2330 * @tx_ring: Tx descriptor ring 2331 * 2332 * Return 0 on success, negative on failure 2333 **/ 2334 int e1000e_setup_tx_resources(struct e1000_ring *tx_ring) 2335 { 2336 struct e1000_adapter *adapter = tx_ring->adapter; 2337 int err = -ENOMEM, size; 2338 2339 size = sizeof(struct e1000_buffer) * tx_ring->count; 2340 tx_ring->buffer_info = vzalloc(size); 2341 if (!tx_ring->buffer_info) 2342 goto err; 2343 2344 /* round up to nearest 4K */ 2345 tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc); 2346 tx_ring->size = ALIGN(tx_ring->size, 4096); 2347 2348 err = e1000_alloc_ring_dma(adapter, tx_ring); 2349 if (err) 2350 goto err; 2351 2352 tx_ring->next_to_use = 0; 2353 tx_ring->next_to_clean = 0; 2354 2355 return 0; 2356 err: 2357 vfree(tx_ring->buffer_info); 2358 e_err("Unable to allocate memory for the transmit descriptor ring\n"); 2359 return err; 2360 } 2361 2362 /** 2363 * e1000e_setup_rx_resources - allocate Rx resources (Descriptors) 2364 * @rx_ring: Rx descriptor ring 2365 * 2366 * Returns 0 on success, negative on failure 2367 **/ 2368 int e1000e_setup_rx_resources(struct e1000_ring *rx_ring) 2369 { 2370 struct e1000_adapter *adapter = rx_ring->adapter; 2371 struct e1000_buffer *buffer_info; 2372 int i, size, desc_len, err = -ENOMEM; 2373 2374 size = sizeof(struct e1000_buffer) * rx_ring->count; 2375 rx_ring->buffer_info = vzalloc(size); 2376 if (!rx_ring->buffer_info) 2377 goto err; 2378 2379 for (i = 0; i < rx_ring->count; i++) { 2380 buffer_info = &rx_ring->buffer_info[i]; 2381 buffer_info->ps_pages = kcalloc(PS_PAGE_BUFFERS, 2382 sizeof(struct e1000_ps_page), 2383 GFP_KERNEL); 2384 if (!buffer_info->ps_pages) 2385 goto err_pages; 2386 } 2387 2388 desc_len = sizeof(union e1000_rx_desc_packet_split); 2389 2390 /* Round up to nearest 4K */ 2391 rx_ring->size = rx_ring->count * desc_len; 2392 rx_ring->size = ALIGN(rx_ring->size, 4096); 2393 2394 err = e1000_alloc_ring_dma(adapter, rx_ring); 2395 if (err) 2396 goto err_pages; 2397 2398 rx_ring->next_to_clean = 0; 2399 rx_ring->next_to_use = 0; 2400 rx_ring->rx_skb_top = NULL; 2401 2402 return 0; 2403 2404 err_pages: 2405 for (i = 0; i < rx_ring->count; i++) { 2406 buffer_info = &rx_ring->buffer_info[i]; 2407 kfree(buffer_info->ps_pages); 2408 } 2409 err: 2410 vfree(rx_ring->buffer_info); 2411 e_err("Unable to allocate memory for the receive descriptor ring\n"); 2412 return err; 2413 } 2414 2415 /** 2416 * e1000_clean_tx_ring - Free Tx Buffers 2417 * @tx_ring: Tx descriptor ring 2418 **/ 2419 static void e1000_clean_tx_ring(struct e1000_ring *tx_ring) 2420 { 2421 struct e1000_adapter *adapter = tx_ring->adapter; 2422 struct e1000_buffer *buffer_info; 2423 unsigned long size; 2424 unsigned int i; 2425 2426 for (i = 0; i < tx_ring->count; i++) { 2427 buffer_info = &tx_ring->buffer_info[i]; 2428 e1000_put_txbuf(tx_ring, buffer_info, false); 2429 } 2430 2431 netdev_reset_queue(adapter->netdev); 2432 size = sizeof(struct e1000_buffer) * tx_ring->count; 2433 memset(tx_ring->buffer_info, 0, size); 2434 2435 memset(tx_ring->desc, 0, tx_ring->size); 2436 2437 tx_ring->next_to_use = 0; 2438 tx_ring->next_to_clean = 0; 2439 } 2440 2441 /** 2442 * e1000e_free_tx_resources - Free Tx Resources per Queue 2443 * @tx_ring: Tx descriptor ring 2444 * 2445 * Free all transmit software resources 2446 **/ 2447 void e1000e_free_tx_resources(struct e1000_ring *tx_ring) 2448 { 2449 struct e1000_adapter *adapter = tx_ring->adapter; 2450 struct pci_dev *pdev = adapter->pdev; 2451 2452 e1000_clean_tx_ring(tx_ring); 2453 2454 vfree(tx_ring->buffer_info); 2455 tx_ring->buffer_info = NULL; 2456 2457 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc, 2458 tx_ring->dma); 2459 tx_ring->desc = NULL; 2460 } 2461 2462 /** 2463 * e1000e_free_rx_resources - Free Rx Resources 2464 * @rx_ring: Rx descriptor ring 2465 * 2466 * Free all receive software resources 2467 **/ 2468 void e1000e_free_rx_resources(struct e1000_ring *rx_ring) 2469 { 2470 struct e1000_adapter *adapter = rx_ring->adapter; 2471 struct pci_dev *pdev = adapter->pdev; 2472 int i; 2473 2474 e1000_clean_rx_ring(rx_ring); 2475 2476 for (i = 0; i < rx_ring->count; i++) 2477 kfree(rx_ring->buffer_info[i].ps_pages); 2478 2479 vfree(rx_ring->buffer_info); 2480 rx_ring->buffer_info = NULL; 2481 2482 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, 2483 rx_ring->dma); 2484 rx_ring->desc = NULL; 2485 } 2486 2487 /** 2488 * e1000_update_itr - update the dynamic ITR value based on statistics 2489 * @itr_setting: current adapter->itr 2490 * @packets: the number of packets during this measurement interval 2491 * @bytes: the number of bytes during this measurement interval 2492 * 2493 * Stores a new ITR value based on packets and byte 2494 * counts during the last interrupt. The advantage of per interrupt 2495 * computation is faster updates and more accurate ITR for the current 2496 * traffic pattern. Constants in this function were computed 2497 * based on theoretical maximum wire speed and thresholds were set based 2498 * on testing data as well as attempting to minimize response time 2499 * while increasing bulk throughput. This functionality is controlled 2500 * by the InterruptThrottleRate module parameter. 2501 **/ 2502 static unsigned int e1000_update_itr(u16 itr_setting, int packets, int bytes) 2503 { 2504 unsigned int retval = itr_setting; 2505 2506 if (packets == 0) 2507 return itr_setting; 2508 2509 switch (itr_setting) { 2510 case lowest_latency: 2511 /* handle TSO and jumbo frames */ 2512 if (bytes / packets > 8000) 2513 retval = bulk_latency; 2514 else if ((packets < 5) && (bytes > 512)) 2515 retval = low_latency; 2516 break; 2517 case low_latency: /* 50 usec aka 20000 ints/s */ 2518 if (bytes > 10000) { 2519 /* this if handles the TSO accounting */ 2520 if (bytes / packets > 8000) 2521 retval = bulk_latency; 2522 else if ((packets < 10) || ((bytes / packets) > 1200)) 2523 retval = bulk_latency; 2524 else if ((packets > 35)) 2525 retval = lowest_latency; 2526 } else if (bytes / packets > 2000) { 2527 retval = bulk_latency; 2528 } else if (packets <= 2 && bytes < 512) { 2529 retval = lowest_latency; 2530 } 2531 break; 2532 case bulk_latency: /* 250 usec aka 4000 ints/s */ 2533 if (bytes > 25000) { 2534 if (packets > 35) 2535 retval = low_latency; 2536 } else if (bytes < 6000) { 2537 retval = low_latency; 2538 } 2539 break; 2540 } 2541 2542 return retval; 2543 } 2544 2545 static void e1000_set_itr(struct e1000_adapter *adapter) 2546 { 2547 u16 current_itr; 2548 u32 new_itr = adapter->itr; 2549 2550 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 2551 if (adapter->link_speed != SPEED_1000) { 2552 current_itr = 0; 2553 new_itr = 4000; 2554 goto set_itr_now; 2555 } 2556 2557 if (adapter->flags2 & FLAG2_DISABLE_AIM) { 2558 new_itr = 0; 2559 goto set_itr_now; 2560 } 2561 2562 adapter->tx_itr = e1000_update_itr(adapter->tx_itr, 2563 adapter->total_tx_packets, 2564 adapter->total_tx_bytes); 2565 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 2566 if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency) 2567 adapter->tx_itr = low_latency; 2568 2569 adapter->rx_itr = e1000_update_itr(adapter->rx_itr, 2570 adapter->total_rx_packets, 2571 adapter->total_rx_bytes); 2572 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 2573 if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency) 2574 adapter->rx_itr = low_latency; 2575 2576 current_itr = max(adapter->rx_itr, adapter->tx_itr); 2577 2578 /* counts and packets in update_itr are dependent on these numbers */ 2579 switch (current_itr) { 2580 case lowest_latency: 2581 new_itr = 70000; 2582 break; 2583 case low_latency: 2584 new_itr = 20000; /* aka hwitr = ~200 */ 2585 break; 2586 case bulk_latency: 2587 new_itr = 4000; 2588 break; 2589 default: 2590 break; 2591 } 2592 2593 set_itr_now: 2594 if (new_itr != adapter->itr) { 2595 /* this attempts to bias the interrupt rate towards Bulk 2596 * by adding intermediate steps when interrupt rate is 2597 * increasing 2598 */ 2599 new_itr = new_itr > adapter->itr ? 2600 min(adapter->itr + (new_itr >> 2), new_itr) : new_itr; 2601 adapter->itr = new_itr; 2602 adapter->rx_ring->itr_val = new_itr; 2603 if (adapter->msix_entries) 2604 adapter->rx_ring->set_itr = 1; 2605 else 2606 e1000e_write_itr(adapter, new_itr); 2607 } 2608 } 2609 2610 /** 2611 * e1000e_write_itr - write the ITR value to the appropriate registers 2612 * @adapter: address of board private structure 2613 * @itr: new ITR value to program 2614 * 2615 * e1000e_write_itr determines if the adapter is in MSI-X mode 2616 * and, if so, writes the EITR registers with the ITR value. 2617 * Otherwise, it writes the ITR value into the ITR register. 2618 **/ 2619 void e1000e_write_itr(struct e1000_adapter *adapter, u32 itr) 2620 { 2621 struct e1000_hw *hw = &adapter->hw; 2622 u32 new_itr = itr ? 1000000000 / (itr * 256) : 0; 2623 2624 if (adapter->msix_entries) { 2625 int vector; 2626 2627 for (vector = 0; vector < adapter->num_vectors; vector++) 2628 writel(new_itr, hw->hw_addr + E1000_EITR_82574(vector)); 2629 } else { 2630 ew32(ITR, new_itr); 2631 } 2632 } 2633 2634 /** 2635 * e1000_alloc_queues - Allocate memory for all rings 2636 * @adapter: board private structure to initialize 2637 **/ 2638 static int e1000_alloc_queues(struct e1000_adapter *adapter) 2639 { 2640 int size = sizeof(struct e1000_ring); 2641 2642 adapter->tx_ring = kzalloc(size, GFP_KERNEL); 2643 if (!adapter->tx_ring) 2644 goto err; 2645 adapter->tx_ring->count = adapter->tx_ring_count; 2646 adapter->tx_ring->adapter = adapter; 2647 2648 adapter->rx_ring = kzalloc(size, GFP_KERNEL); 2649 if (!adapter->rx_ring) 2650 goto err; 2651 adapter->rx_ring->count = adapter->rx_ring_count; 2652 adapter->rx_ring->adapter = adapter; 2653 2654 return 0; 2655 err: 2656 e_err("Unable to allocate memory for queues\n"); 2657 kfree(adapter->rx_ring); 2658 kfree(adapter->tx_ring); 2659 return -ENOMEM; 2660 } 2661 2662 /** 2663 * e1000e_poll - NAPI Rx polling callback 2664 * @napi: struct associated with this polling callback 2665 * @budget: number of packets driver is allowed to process this poll 2666 **/ 2667 static int e1000e_poll(struct napi_struct *napi, int budget) 2668 { 2669 struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter, 2670 napi); 2671 struct e1000_hw *hw = &adapter->hw; 2672 struct net_device *poll_dev = adapter->netdev; 2673 int tx_cleaned = 1, work_done = 0; 2674 2675 adapter = netdev_priv(poll_dev); 2676 2677 if (!adapter->msix_entries || 2678 (adapter->rx_ring->ims_val & adapter->tx_ring->ims_val)) 2679 tx_cleaned = e1000_clean_tx_irq(adapter->tx_ring); 2680 2681 adapter->clean_rx(adapter->rx_ring, &work_done, budget); 2682 2683 if (!tx_cleaned || work_done == budget) 2684 return budget; 2685 2686 /* Exit the polling mode, but don't re-enable interrupts if stack might 2687 * poll us due to busy-polling 2688 */ 2689 if (likely(napi_complete_done(napi, work_done))) { 2690 if (adapter->itr_setting & 3) 2691 e1000_set_itr(adapter); 2692 if (!test_bit(__E1000_DOWN, &adapter->state)) { 2693 if (adapter->msix_entries) 2694 ew32(IMS, adapter->rx_ring->ims_val); 2695 else 2696 e1000_irq_enable(adapter); 2697 } 2698 } 2699 2700 return work_done; 2701 } 2702 2703 static int e1000_vlan_rx_add_vid(struct net_device *netdev, 2704 __always_unused __be16 proto, u16 vid) 2705 { 2706 struct e1000_adapter *adapter = netdev_priv(netdev); 2707 struct e1000_hw *hw = &adapter->hw; 2708 u32 vfta, index; 2709 2710 /* don't update vlan cookie if already programmed */ 2711 if ((adapter->hw.mng_cookie.status & 2712 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) && 2713 (vid == adapter->mng_vlan_id)) 2714 return 0; 2715 2716 /* add VID to filter table */ 2717 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2718 index = (vid >> 5) & 0x7F; 2719 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index); 2720 vfta |= BIT((vid & 0x1F)); 2721 hw->mac.ops.write_vfta(hw, index, vfta); 2722 } 2723 2724 set_bit(vid, adapter->active_vlans); 2725 2726 return 0; 2727 } 2728 2729 static int e1000_vlan_rx_kill_vid(struct net_device *netdev, 2730 __always_unused __be16 proto, u16 vid) 2731 { 2732 struct e1000_adapter *adapter = netdev_priv(netdev); 2733 struct e1000_hw *hw = &adapter->hw; 2734 u32 vfta, index; 2735 2736 if ((adapter->hw.mng_cookie.status & 2737 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) && 2738 (vid == adapter->mng_vlan_id)) { 2739 /* release control to f/w */ 2740 e1000e_release_hw_control(adapter); 2741 return 0; 2742 } 2743 2744 /* remove VID from filter table */ 2745 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2746 index = (vid >> 5) & 0x7F; 2747 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index); 2748 vfta &= ~BIT((vid & 0x1F)); 2749 hw->mac.ops.write_vfta(hw, index, vfta); 2750 } 2751 2752 clear_bit(vid, adapter->active_vlans); 2753 2754 return 0; 2755 } 2756 2757 /** 2758 * e1000e_vlan_filter_disable - helper to disable hw VLAN filtering 2759 * @adapter: board private structure to initialize 2760 **/ 2761 static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter) 2762 { 2763 struct net_device *netdev = adapter->netdev; 2764 struct e1000_hw *hw = &adapter->hw; 2765 u32 rctl; 2766 2767 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2768 /* disable VLAN receive filtering */ 2769 rctl = er32(RCTL); 2770 rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN); 2771 ew32(RCTL, rctl); 2772 2773 if (adapter->mng_vlan_id != (u16)E1000_MNG_VLAN_NONE) { 2774 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), 2775 adapter->mng_vlan_id); 2776 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE; 2777 } 2778 } 2779 } 2780 2781 /** 2782 * e1000e_vlan_filter_enable - helper to enable HW VLAN filtering 2783 * @adapter: board private structure to initialize 2784 **/ 2785 static void e1000e_vlan_filter_enable(struct e1000_adapter *adapter) 2786 { 2787 struct e1000_hw *hw = &adapter->hw; 2788 u32 rctl; 2789 2790 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2791 /* enable VLAN receive filtering */ 2792 rctl = er32(RCTL); 2793 rctl |= E1000_RCTL_VFE; 2794 rctl &= ~E1000_RCTL_CFIEN; 2795 ew32(RCTL, rctl); 2796 } 2797 } 2798 2799 /** 2800 * e1000e_vlan_strip_disable - helper to disable HW VLAN stripping 2801 * @adapter: board private structure to initialize 2802 **/ 2803 static void e1000e_vlan_strip_disable(struct e1000_adapter *adapter) 2804 { 2805 struct e1000_hw *hw = &adapter->hw; 2806 u32 ctrl; 2807 2808 /* disable VLAN tag insert/strip */ 2809 ctrl = er32(CTRL); 2810 ctrl &= ~E1000_CTRL_VME; 2811 ew32(CTRL, ctrl); 2812 } 2813 2814 /** 2815 * e1000e_vlan_strip_enable - helper to enable HW VLAN stripping 2816 * @adapter: board private structure to initialize 2817 **/ 2818 static void e1000e_vlan_strip_enable(struct e1000_adapter *adapter) 2819 { 2820 struct e1000_hw *hw = &adapter->hw; 2821 u32 ctrl; 2822 2823 /* enable VLAN tag insert/strip */ 2824 ctrl = er32(CTRL); 2825 ctrl |= E1000_CTRL_VME; 2826 ew32(CTRL, ctrl); 2827 } 2828 2829 static void e1000_update_mng_vlan(struct e1000_adapter *adapter) 2830 { 2831 struct net_device *netdev = adapter->netdev; 2832 u16 vid = adapter->hw.mng_cookie.vlan_id; 2833 u16 old_vid = adapter->mng_vlan_id; 2834 2835 if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) { 2836 e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid); 2837 adapter->mng_vlan_id = vid; 2838 } 2839 2840 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && (vid != old_vid)) 2841 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), old_vid); 2842 } 2843 2844 static void e1000_restore_vlan(struct e1000_adapter *adapter) 2845 { 2846 u16 vid; 2847 2848 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0); 2849 2850 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2851 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid); 2852 } 2853 2854 static void e1000_init_manageability_pt(struct e1000_adapter *adapter) 2855 { 2856 struct e1000_hw *hw = &adapter->hw; 2857 u32 manc, manc2h, mdef, i, j; 2858 2859 if (!(adapter->flags & FLAG_MNG_PT_ENABLED)) 2860 return; 2861 2862 manc = er32(MANC); 2863 2864 /* enable receiving management packets to the host. this will probably 2865 * generate destination unreachable messages from the host OS, but 2866 * the packets will be handled on SMBUS 2867 */ 2868 manc |= E1000_MANC_EN_MNG2HOST; 2869 manc2h = er32(MANC2H); 2870 2871 switch (hw->mac.type) { 2872 default: 2873 manc2h |= (E1000_MANC2H_PORT_623 | E1000_MANC2H_PORT_664); 2874 break; 2875 case e1000_82574: 2876 case e1000_82583: 2877 /* Check if IPMI pass-through decision filter already exists; 2878 * if so, enable it. 2879 */ 2880 for (i = 0, j = 0; i < 8; i++) { 2881 mdef = er32(MDEF(i)); 2882 2883 /* Ignore filters with anything other than IPMI ports */ 2884 if (mdef & ~(E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664)) 2885 continue; 2886 2887 /* Enable this decision filter in MANC2H */ 2888 if (mdef) 2889 manc2h |= BIT(i); 2890 2891 j |= mdef; 2892 } 2893 2894 if (j == (E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664)) 2895 break; 2896 2897 /* Create new decision filter in an empty filter */ 2898 for (i = 0, j = 0; i < 8; i++) 2899 if (er32(MDEF(i)) == 0) { 2900 ew32(MDEF(i), (E1000_MDEF_PORT_623 | 2901 E1000_MDEF_PORT_664)); 2902 manc2h |= BIT(1); 2903 j++; 2904 break; 2905 } 2906 2907 if (!j) 2908 e_warn("Unable to create IPMI pass-through filter\n"); 2909 break; 2910 } 2911 2912 ew32(MANC2H, manc2h); 2913 ew32(MANC, manc); 2914 } 2915 2916 /** 2917 * e1000_configure_tx - Configure Transmit Unit after Reset 2918 * @adapter: board private structure 2919 * 2920 * Configure the Tx unit of the MAC after a reset. 2921 **/ 2922 static void e1000_configure_tx(struct e1000_adapter *adapter) 2923 { 2924 struct e1000_hw *hw = &adapter->hw; 2925 struct e1000_ring *tx_ring = adapter->tx_ring; 2926 u64 tdba; 2927 u32 tdlen, tctl, tarc; 2928 2929 /* Setup the HW Tx Head and Tail descriptor pointers */ 2930 tdba = tx_ring->dma; 2931 tdlen = tx_ring->count * sizeof(struct e1000_tx_desc); 2932 ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32))); 2933 ew32(TDBAH(0), (tdba >> 32)); 2934 ew32(TDLEN(0), tdlen); 2935 ew32(TDH(0), 0); 2936 ew32(TDT(0), 0); 2937 tx_ring->head = adapter->hw.hw_addr + E1000_TDH(0); 2938 tx_ring->tail = adapter->hw.hw_addr + E1000_TDT(0); 2939 2940 writel(0, tx_ring->head); 2941 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 2942 e1000e_update_tdt_wa(tx_ring, 0); 2943 else 2944 writel(0, tx_ring->tail); 2945 2946 /* Set the Tx Interrupt Delay register */ 2947 ew32(TIDV, adapter->tx_int_delay); 2948 /* Tx irq moderation */ 2949 ew32(TADV, adapter->tx_abs_int_delay); 2950 2951 if (adapter->flags2 & FLAG2_DMA_BURST) { 2952 u32 txdctl = er32(TXDCTL(0)); 2953 2954 txdctl &= ~(E1000_TXDCTL_PTHRESH | E1000_TXDCTL_HTHRESH | 2955 E1000_TXDCTL_WTHRESH); 2956 /* set up some performance related parameters to encourage the 2957 * hardware to use the bus more efficiently in bursts, depends 2958 * on the tx_int_delay to be enabled, 2959 * wthresh = 1 ==> burst write is disabled to avoid Tx stalls 2960 * hthresh = 1 ==> prefetch when one or more available 2961 * pthresh = 0x1f ==> prefetch if internal cache 31 or less 2962 * BEWARE: this seems to work but should be considered first if 2963 * there are Tx hangs or other Tx related bugs 2964 */ 2965 txdctl |= E1000_TXDCTL_DMA_BURST_ENABLE; 2966 ew32(TXDCTL(0), txdctl); 2967 } 2968 /* erratum work around: set txdctl the same for both queues */ 2969 ew32(TXDCTL(1), er32(TXDCTL(0))); 2970 2971 /* Program the Transmit Control Register */ 2972 tctl = er32(TCTL); 2973 tctl &= ~E1000_TCTL_CT; 2974 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | 2975 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2976 2977 if (adapter->flags & FLAG_TARC_SPEED_MODE_BIT) { 2978 tarc = er32(TARC(0)); 2979 /* set the speed mode bit, we'll clear it if we're not at 2980 * gigabit link later 2981 */ 2982 #define SPEED_MODE_BIT BIT(21) 2983 tarc |= SPEED_MODE_BIT; 2984 ew32(TARC(0), tarc); 2985 } 2986 2987 /* errata: program both queues to unweighted RR */ 2988 if (adapter->flags & FLAG_TARC_SET_BIT_ZERO) { 2989 tarc = er32(TARC(0)); 2990 tarc |= 1; 2991 ew32(TARC(0), tarc); 2992 tarc = er32(TARC(1)); 2993 tarc |= 1; 2994 ew32(TARC(1), tarc); 2995 } 2996 2997 /* Setup Transmit Descriptor Settings for eop descriptor */ 2998 adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS; 2999 3000 /* only set IDE if we are delaying interrupts using the timers */ 3001 if (adapter->tx_int_delay) 3002 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3003 3004 /* enable Report Status bit */ 3005 adapter->txd_cmd |= E1000_TXD_CMD_RS; 3006 3007 ew32(TCTL, tctl); 3008 3009 hw->mac.ops.config_collision_dist(hw); 3010 3011 /* SPT and KBL Si errata workaround to avoid data corruption */ 3012 if (hw->mac.type == e1000_pch_spt) { 3013 u32 reg_val; 3014 3015 reg_val = er32(IOSFPC); 3016 reg_val |= E1000_RCTL_RDMTS_HEX; 3017 ew32(IOSFPC, reg_val); 3018 3019 reg_val = er32(TARC(0)); 3020 /* SPT and KBL Si errata workaround to avoid Tx hang. 3021 * Dropping the number of outstanding requests from 3022 * 3 to 2 in order to avoid a buffer overrun. 3023 */ 3024 reg_val &= ~E1000_TARC0_CB_MULTIQ_3_REQ; 3025 reg_val |= E1000_TARC0_CB_MULTIQ_2_REQ; 3026 ew32(TARC(0), reg_val); 3027 } 3028 } 3029 3030 #define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \ 3031 (((S) & (PAGE_SIZE - 1)) ? 1 : 0)) 3032 3033 /** 3034 * e1000_setup_rctl - configure the receive control registers 3035 * @adapter: Board private structure 3036 **/ 3037 static void e1000_setup_rctl(struct e1000_adapter *adapter) 3038 { 3039 struct e1000_hw *hw = &adapter->hw; 3040 u32 rctl, rfctl; 3041 u32 pages = 0; 3042 3043 /* Workaround Si errata on PCHx - configure jumbo frame flow. 3044 * If jumbo frames not set, program related MAC/PHY registers 3045 * to h/w defaults 3046 */ 3047 if (hw->mac.type >= e1000_pch2lan) { 3048 s32 ret_val; 3049 3050 if (adapter->netdev->mtu > ETH_DATA_LEN) 3051 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, true); 3052 else 3053 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, false); 3054 3055 if (ret_val) 3056 e_dbg("failed to enable|disable jumbo frame workaround mode\n"); 3057 } 3058 3059 /* Program MC offset vector base */ 3060 rctl = er32(RCTL); 3061 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 3062 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 3063 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | 3064 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 3065 3066 /* Do not Store bad packets */ 3067 rctl &= ~E1000_RCTL_SBP; 3068 3069 /* Enable Long Packet receive */ 3070 if (adapter->netdev->mtu <= ETH_DATA_LEN) 3071 rctl &= ~E1000_RCTL_LPE; 3072 else 3073 rctl |= E1000_RCTL_LPE; 3074 3075 /* Some systems expect that the CRC is included in SMBUS traffic. The 3076 * hardware strips the CRC before sending to both SMBUS (BMC) and to 3077 * host memory when this is enabled 3078 */ 3079 if (adapter->flags2 & FLAG2_CRC_STRIPPING) 3080 rctl |= E1000_RCTL_SECRC; 3081 3082 /* Workaround Si errata on 82577 PHY - configure IPG for jumbos */ 3083 if ((hw->phy.type == e1000_phy_82577) && (rctl & E1000_RCTL_LPE)) { 3084 u16 phy_data; 3085 3086 e1e_rphy(hw, PHY_REG(770, 26), &phy_data); 3087 phy_data &= 0xfff8; 3088 phy_data |= BIT(2); 3089 e1e_wphy(hw, PHY_REG(770, 26), phy_data); 3090 3091 e1e_rphy(hw, 22, &phy_data); 3092 phy_data &= 0x0fff; 3093 phy_data |= BIT(14); 3094 e1e_wphy(hw, 0x10, 0x2823); 3095 e1e_wphy(hw, 0x11, 0x0003); 3096 e1e_wphy(hw, 22, phy_data); 3097 } 3098 3099 /* Setup buffer sizes */ 3100 rctl &= ~E1000_RCTL_SZ_4096; 3101 rctl |= E1000_RCTL_BSEX; 3102 switch (adapter->rx_buffer_len) { 3103 case 2048: 3104 default: 3105 rctl |= E1000_RCTL_SZ_2048; 3106 rctl &= ~E1000_RCTL_BSEX; 3107 break; 3108 case 4096: 3109 rctl |= E1000_RCTL_SZ_4096; 3110 break; 3111 case 8192: 3112 rctl |= E1000_RCTL_SZ_8192; 3113 break; 3114 case 16384: 3115 rctl |= E1000_RCTL_SZ_16384; 3116 break; 3117 } 3118 3119 /* Enable Extended Status in all Receive Descriptors */ 3120 rfctl = er32(RFCTL); 3121 rfctl |= E1000_RFCTL_EXTEN; 3122 ew32(RFCTL, rfctl); 3123 3124 /* 82571 and greater support packet-split where the protocol 3125 * header is placed in skb->data and the packet data is 3126 * placed in pages hanging off of skb_shinfo(skb)->nr_frags. 3127 * In the case of a non-split, skb->data is linearly filled, 3128 * followed by the page buffers. Therefore, skb->data is 3129 * sized to hold the largest protocol header. 3130 * 3131 * allocations using alloc_page take too long for regular MTU 3132 * so only enable packet split for jumbo frames 3133 * 3134 * Using pages when the page size is greater than 16k wastes 3135 * a lot of memory, since we allocate 3 pages at all times 3136 * per packet. 3137 */ 3138 pages = PAGE_USE_COUNT(adapter->netdev->mtu); 3139 if ((pages <= 3) && (PAGE_SIZE <= 16384) && (rctl & E1000_RCTL_LPE)) 3140 adapter->rx_ps_pages = pages; 3141 else 3142 adapter->rx_ps_pages = 0; 3143 3144 if (adapter->rx_ps_pages) { 3145 u32 psrctl = 0; 3146 3147 /* Enable Packet split descriptors */ 3148 rctl |= E1000_RCTL_DTYP_PS; 3149 3150 psrctl |= adapter->rx_ps_bsize0 >> E1000_PSRCTL_BSIZE0_SHIFT; 3151 3152 switch (adapter->rx_ps_pages) { 3153 case 3: 3154 psrctl |= PAGE_SIZE << E1000_PSRCTL_BSIZE3_SHIFT; 3155 fallthrough; 3156 case 2: 3157 psrctl |= PAGE_SIZE << E1000_PSRCTL_BSIZE2_SHIFT; 3158 fallthrough; 3159 case 1: 3160 psrctl |= PAGE_SIZE >> E1000_PSRCTL_BSIZE1_SHIFT; 3161 break; 3162 } 3163 3164 ew32(PSRCTL, psrctl); 3165 } 3166 3167 /* This is useful for sniffing bad packets. */ 3168 if (adapter->netdev->features & NETIF_F_RXALL) { 3169 /* UPE and MPE will be handled by normal PROMISC logic 3170 * in e1000e_set_rx_mode 3171 */ 3172 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */ 3173 E1000_RCTL_BAM | /* RX All Bcast Pkts */ 3174 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 3175 3176 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */ 3177 E1000_RCTL_DPF | /* Allow filtered pause */ 3178 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */ 3179 /* Do not mess with E1000_CTRL_VME, it affects transmit as well, 3180 * and that breaks VLANs. 3181 */ 3182 } 3183 3184 ew32(RCTL, rctl); 3185 /* just started the receive unit, no need to restart */ 3186 adapter->flags &= ~FLAG_RESTART_NOW; 3187 } 3188 3189 /** 3190 * e1000_configure_rx - Configure Receive Unit after Reset 3191 * @adapter: board private structure 3192 * 3193 * Configure the Rx unit of the MAC after a reset. 3194 **/ 3195 static void e1000_configure_rx(struct e1000_adapter *adapter) 3196 { 3197 struct e1000_hw *hw = &adapter->hw; 3198 struct e1000_ring *rx_ring = adapter->rx_ring; 3199 u64 rdba; 3200 u32 rdlen, rctl, rxcsum, ctrl_ext; 3201 3202 if (adapter->rx_ps_pages) { 3203 /* this is a 32 byte descriptor */ 3204 rdlen = rx_ring->count * 3205 sizeof(union e1000_rx_desc_packet_split); 3206 adapter->clean_rx = e1000_clean_rx_irq_ps; 3207 adapter->alloc_rx_buf = e1000_alloc_rx_buffers_ps; 3208 } else if (adapter->netdev->mtu > ETH_FRAME_LEN + ETH_FCS_LEN) { 3209 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended); 3210 adapter->clean_rx = e1000_clean_jumbo_rx_irq; 3211 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers; 3212 } else { 3213 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended); 3214 adapter->clean_rx = e1000_clean_rx_irq; 3215 adapter->alloc_rx_buf = e1000_alloc_rx_buffers; 3216 } 3217 3218 /* disable receives while setting up the descriptors */ 3219 rctl = er32(RCTL); 3220 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX)) 3221 ew32(RCTL, rctl & ~E1000_RCTL_EN); 3222 e1e_flush(); 3223 usleep_range(10000, 11000); 3224 3225 if (adapter->flags2 & FLAG2_DMA_BURST) { 3226 /* set the writeback threshold (only takes effect if the RDTR 3227 * is set). set GRAN=1 and write back up to 0x4 worth, and 3228 * enable prefetching of 0x20 Rx descriptors 3229 * granularity = 01 3230 * wthresh = 04, 3231 * hthresh = 04, 3232 * pthresh = 0x20 3233 */ 3234 ew32(RXDCTL(0), E1000_RXDCTL_DMA_BURST_ENABLE); 3235 ew32(RXDCTL(1), E1000_RXDCTL_DMA_BURST_ENABLE); 3236 } 3237 3238 /* set the Receive Delay Timer Register */ 3239 ew32(RDTR, adapter->rx_int_delay); 3240 3241 /* irq moderation */ 3242 ew32(RADV, adapter->rx_abs_int_delay); 3243 if ((adapter->itr_setting != 0) && (adapter->itr != 0)) 3244 e1000e_write_itr(adapter, adapter->itr); 3245 3246 ctrl_ext = er32(CTRL_EXT); 3247 /* Auto-Mask interrupts upon ICR access */ 3248 ctrl_ext |= E1000_CTRL_EXT_IAME; 3249 ew32(IAM, 0xffffffff); 3250 ew32(CTRL_EXT, ctrl_ext); 3251 e1e_flush(); 3252 3253 /* Setup the HW Rx Head and Tail Descriptor Pointers and 3254 * the Base and Length of the Rx Descriptor Ring 3255 */ 3256 rdba = rx_ring->dma; 3257 ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32))); 3258 ew32(RDBAH(0), (rdba >> 32)); 3259 ew32(RDLEN(0), rdlen); 3260 ew32(RDH(0), 0); 3261 ew32(RDT(0), 0); 3262 rx_ring->head = adapter->hw.hw_addr + E1000_RDH(0); 3263 rx_ring->tail = adapter->hw.hw_addr + E1000_RDT(0); 3264 3265 writel(0, rx_ring->head); 3266 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 3267 e1000e_update_rdt_wa(rx_ring, 0); 3268 else 3269 writel(0, rx_ring->tail); 3270 3271 /* Enable Receive Checksum Offload for TCP and UDP */ 3272 rxcsum = er32(RXCSUM); 3273 if (adapter->netdev->features & NETIF_F_RXCSUM) 3274 rxcsum |= E1000_RXCSUM_TUOFL; 3275 else 3276 rxcsum &= ~E1000_RXCSUM_TUOFL; 3277 ew32(RXCSUM, rxcsum); 3278 3279 /* With jumbo frames, excessive C-state transition latencies result 3280 * in dropped transactions. 3281 */ 3282 if (adapter->netdev->mtu > ETH_DATA_LEN) { 3283 u32 lat = 3284 ((er32(PBA) & E1000_PBA_RXA_MASK) * 1024 - 3285 adapter->max_frame_size) * 8 / 1000; 3286 3287 if (adapter->flags & FLAG_IS_ICH) { 3288 u32 rxdctl = er32(RXDCTL(0)); 3289 3290 ew32(RXDCTL(0), rxdctl | 0x3 | BIT(8)); 3291 } 3292 3293 dev_info(&adapter->pdev->dev, 3294 "Some CPU C-states have been disabled in order to enable jumbo frames\n"); 3295 cpu_latency_qos_update_request(&adapter->pm_qos_req, lat); 3296 } else { 3297 cpu_latency_qos_update_request(&adapter->pm_qos_req, 3298 PM_QOS_DEFAULT_VALUE); 3299 } 3300 3301 /* Enable Receives */ 3302 ew32(RCTL, rctl); 3303 } 3304 3305 /** 3306 * e1000e_write_mc_addr_list - write multicast addresses to MTA 3307 * @netdev: network interface device structure 3308 * 3309 * Writes multicast address list to the MTA hash table. 3310 * Returns: -ENOMEM on failure 3311 * 0 on no addresses written 3312 * X on writing X addresses to MTA 3313 */ 3314 static int e1000e_write_mc_addr_list(struct net_device *netdev) 3315 { 3316 struct e1000_adapter *adapter = netdev_priv(netdev); 3317 struct e1000_hw *hw = &adapter->hw; 3318 struct netdev_hw_addr *ha; 3319 u8 *mta_list; 3320 int i; 3321 3322 if (netdev_mc_empty(netdev)) { 3323 /* nothing to program, so clear mc list */ 3324 hw->mac.ops.update_mc_addr_list(hw, NULL, 0); 3325 return 0; 3326 } 3327 3328 mta_list = kcalloc(netdev_mc_count(netdev), ETH_ALEN, GFP_ATOMIC); 3329 if (!mta_list) 3330 return -ENOMEM; 3331 3332 /* update_mc_addr_list expects a packed array of only addresses. */ 3333 i = 0; 3334 netdev_for_each_mc_addr(ha, netdev) 3335 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 3336 3337 hw->mac.ops.update_mc_addr_list(hw, mta_list, i); 3338 kfree(mta_list); 3339 3340 return netdev_mc_count(netdev); 3341 } 3342 3343 /** 3344 * e1000e_write_uc_addr_list - write unicast addresses to RAR table 3345 * @netdev: network interface device structure 3346 * 3347 * Writes unicast address list to the RAR table. 3348 * Returns: -ENOMEM on failure/insufficient address space 3349 * 0 on no addresses written 3350 * X on writing X addresses to the RAR table 3351 **/ 3352 static int e1000e_write_uc_addr_list(struct net_device *netdev) 3353 { 3354 struct e1000_adapter *adapter = netdev_priv(netdev); 3355 struct e1000_hw *hw = &adapter->hw; 3356 unsigned int rar_entries; 3357 int count = 0; 3358 3359 rar_entries = hw->mac.ops.rar_get_count(hw); 3360 3361 /* save a rar entry for our hardware address */ 3362 rar_entries--; 3363 3364 /* save a rar entry for the LAA workaround */ 3365 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) 3366 rar_entries--; 3367 3368 /* return ENOMEM indicating insufficient memory for addresses */ 3369 if (netdev_uc_count(netdev) > rar_entries) 3370 return -ENOMEM; 3371 3372 if (!netdev_uc_empty(netdev) && rar_entries) { 3373 struct netdev_hw_addr *ha; 3374 3375 /* write the addresses in reverse order to avoid write 3376 * combining 3377 */ 3378 netdev_for_each_uc_addr(ha, netdev) { 3379 int ret_val; 3380 3381 if (!rar_entries) 3382 break; 3383 ret_val = hw->mac.ops.rar_set(hw, ha->addr, rar_entries--); 3384 if (ret_val < 0) 3385 return -ENOMEM; 3386 count++; 3387 } 3388 } 3389 3390 /* zero out the remaining RAR entries not used above */ 3391 for (; rar_entries > 0; rar_entries--) { 3392 ew32(RAH(rar_entries), 0); 3393 ew32(RAL(rar_entries), 0); 3394 } 3395 e1e_flush(); 3396 3397 return count; 3398 } 3399 3400 /** 3401 * e1000e_set_rx_mode - secondary unicast, Multicast and Promiscuous mode set 3402 * @netdev: network interface device structure 3403 * 3404 * The ndo_set_rx_mode entry point is called whenever the unicast or multicast 3405 * address list or the network interface flags are updated. This routine is 3406 * responsible for configuring the hardware for proper unicast, multicast, 3407 * promiscuous mode, and all-multi behavior. 3408 **/ 3409 static void e1000e_set_rx_mode(struct net_device *netdev) 3410 { 3411 struct e1000_adapter *adapter = netdev_priv(netdev); 3412 struct e1000_hw *hw = &adapter->hw; 3413 u32 rctl; 3414 3415 if (pm_runtime_suspended(netdev->dev.parent)) 3416 return; 3417 3418 /* Check for Promiscuous and All Multicast modes */ 3419 rctl = er32(RCTL); 3420 3421 /* clear the affected bits */ 3422 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE); 3423 3424 if (netdev->flags & IFF_PROMISC) { 3425 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 3426 /* Do not hardware filter VLANs in promisc mode */ 3427 e1000e_vlan_filter_disable(adapter); 3428 } else { 3429 int count; 3430 3431 if (netdev->flags & IFF_ALLMULTI) { 3432 rctl |= E1000_RCTL_MPE; 3433 } else { 3434 /* Write addresses to the MTA, if the attempt fails 3435 * then we should just turn on promiscuous mode so 3436 * that we can at least receive multicast traffic 3437 */ 3438 count = e1000e_write_mc_addr_list(netdev); 3439 if (count < 0) 3440 rctl |= E1000_RCTL_MPE; 3441 } 3442 e1000e_vlan_filter_enable(adapter); 3443 /* Write addresses to available RAR registers, if there is not 3444 * sufficient space to store all the addresses then enable 3445 * unicast promiscuous mode 3446 */ 3447 count = e1000e_write_uc_addr_list(netdev); 3448 if (count < 0) 3449 rctl |= E1000_RCTL_UPE; 3450 } 3451 3452 ew32(RCTL, rctl); 3453 3454 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 3455 e1000e_vlan_strip_enable(adapter); 3456 else 3457 e1000e_vlan_strip_disable(adapter); 3458 } 3459 3460 static void e1000e_setup_rss_hash(struct e1000_adapter *adapter) 3461 { 3462 struct e1000_hw *hw = &adapter->hw; 3463 u32 mrqc, rxcsum; 3464 u32 rss_key[10]; 3465 int i; 3466 3467 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 3468 for (i = 0; i < 10; i++) 3469 ew32(RSSRK(i), rss_key[i]); 3470 3471 /* Direct all traffic to queue 0 */ 3472 for (i = 0; i < 32; i++) 3473 ew32(RETA(i), 0); 3474 3475 /* Disable raw packet checksumming so that RSS hash is placed in 3476 * descriptor on writeback. 3477 */ 3478 rxcsum = er32(RXCSUM); 3479 rxcsum |= E1000_RXCSUM_PCSD; 3480 3481 ew32(RXCSUM, rxcsum); 3482 3483 mrqc = (E1000_MRQC_RSS_FIELD_IPV4 | 3484 E1000_MRQC_RSS_FIELD_IPV4_TCP | 3485 E1000_MRQC_RSS_FIELD_IPV6 | 3486 E1000_MRQC_RSS_FIELD_IPV6_TCP | 3487 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 3488 3489 ew32(MRQC, mrqc); 3490 } 3491 3492 /** 3493 * e1000e_get_base_timinca - get default SYSTIM time increment attributes 3494 * @adapter: board private structure 3495 * @timinca: pointer to returned time increment attributes 3496 * 3497 * Get attributes for incrementing the System Time Register SYSTIML/H at 3498 * the default base frequency, and set the cyclecounter shift value. 3499 **/ 3500 s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca) 3501 { 3502 struct e1000_hw *hw = &adapter->hw; 3503 u32 incvalue, incperiod, shift; 3504 3505 /* Make sure clock is enabled on I217/I218/I219 before checking 3506 * the frequency 3507 */ 3508 if ((hw->mac.type >= e1000_pch_lpt) && 3509 !(er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) && 3510 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_ENABLED)) { 3511 u32 fextnvm7 = er32(FEXTNVM7); 3512 3513 if (!(fextnvm7 & BIT(0))) { 3514 ew32(FEXTNVM7, fextnvm7 | BIT(0)); 3515 e1e_flush(); 3516 } 3517 } 3518 3519 switch (hw->mac.type) { 3520 case e1000_pch2lan: 3521 /* Stable 96MHz frequency */ 3522 incperiod = INCPERIOD_96MHZ; 3523 incvalue = INCVALUE_96MHZ; 3524 shift = INCVALUE_SHIFT_96MHZ; 3525 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHZ; 3526 break; 3527 case e1000_pch_lpt: 3528 if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) { 3529 /* Stable 96MHz frequency */ 3530 incperiod = INCPERIOD_96MHZ; 3531 incvalue = INCVALUE_96MHZ; 3532 shift = INCVALUE_SHIFT_96MHZ; 3533 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHZ; 3534 } else { 3535 /* Stable 25MHz frequency */ 3536 incperiod = INCPERIOD_25MHZ; 3537 incvalue = INCVALUE_25MHZ; 3538 shift = INCVALUE_SHIFT_25MHZ; 3539 adapter->cc.shift = shift; 3540 } 3541 break; 3542 case e1000_pch_spt: 3543 /* Stable 24MHz frequency */ 3544 incperiod = INCPERIOD_24MHZ; 3545 incvalue = INCVALUE_24MHZ; 3546 shift = INCVALUE_SHIFT_24MHZ; 3547 adapter->cc.shift = shift; 3548 break; 3549 case e1000_pch_cnp: 3550 case e1000_pch_tgp: 3551 case e1000_pch_adp: 3552 case e1000_pch_mtp: 3553 if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) { 3554 /* Stable 24MHz frequency */ 3555 incperiod = INCPERIOD_24MHZ; 3556 incvalue = INCVALUE_24MHZ; 3557 shift = INCVALUE_SHIFT_24MHZ; 3558 adapter->cc.shift = shift; 3559 } else { 3560 /* Stable 38400KHz frequency */ 3561 incperiod = INCPERIOD_38400KHZ; 3562 incvalue = INCVALUE_38400KHZ; 3563 shift = INCVALUE_SHIFT_38400KHZ; 3564 adapter->cc.shift = shift; 3565 } 3566 break; 3567 case e1000_82574: 3568 case e1000_82583: 3569 /* Stable 25MHz frequency */ 3570 incperiod = INCPERIOD_25MHZ; 3571 incvalue = INCVALUE_25MHZ; 3572 shift = INCVALUE_SHIFT_25MHZ; 3573 adapter->cc.shift = shift; 3574 break; 3575 default: 3576 return -EINVAL; 3577 } 3578 3579 *timinca = ((incperiod << E1000_TIMINCA_INCPERIOD_SHIFT) | 3580 ((incvalue << shift) & E1000_TIMINCA_INCVALUE_MASK)); 3581 3582 return 0; 3583 } 3584 3585 /** 3586 * e1000e_config_hwtstamp - configure the hwtstamp registers and enable/disable 3587 * @adapter: board private structure 3588 * @config: timestamp configuration 3589 * 3590 * Outgoing time stamping can be enabled and disabled. Play nice and 3591 * disable it when requested, although it shouldn't cause any overhead 3592 * when no packet needs it. At most one packet in the queue may be 3593 * marked for time stamping, otherwise it would be impossible to tell 3594 * for sure to which packet the hardware time stamp belongs. 3595 * 3596 * Incoming time stamping has to be configured via the hardware filters. 3597 * Not all combinations are supported, in particular event type has to be 3598 * specified. Matching the kind of event packet is not supported, with the 3599 * exception of "all V2 events regardless of level 2 or 4". 3600 **/ 3601 static int e1000e_config_hwtstamp(struct e1000_adapter *adapter, 3602 struct hwtstamp_config *config) 3603 { 3604 struct e1000_hw *hw = &adapter->hw; 3605 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED; 3606 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED; 3607 u32 rxmtrl = 0; 3608 u16 rxudp = 0; 3609 bool is_l4 = false; 3610 bool is_l2 = false; 3611 u32 regval; 3612 3613 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP)) 3614 return -EINVAL; 3615 3616 /* flags reserved for future extensions - must be zero */ 3617 if (config->flags) 3618 return -EINVAL; 3619 3620 switch (config->tx_type) { 3621 case HWTSTAMP_TX_OFF: 3622 tsync_tx_ctl = 0; 3623 break; 3624 case HWTSTAMP_TX_ON: 3625 break; 3626 default: 3627 return -ERANGE; 3628 } 3629 3630 switch (config->rx_filter) { 3631 case HWTSTAMP_FILTER_NONE: 3632 tsync_rx_ctl = 0; 3633 break; 3634 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 3635 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1; 3636 rxmtrl = E1000_RXMTRL_PTP_V1_SYNC_MESSAGE; 3637 is_l4 = true; 3638 break; 3639 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 3640 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1; 3641 rxmtrl = E1000_RXMTRL_PTP_V1_DELAY_REQ_MESSAGE; 3642 is_l4 = true; 3643 break; 3644 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 3645 /* Also time stamps V2 L2 Path Delay Request/Response */ 3646 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2; 3647 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE; 3648 is_l2 = true; 3649 break; 3650 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 3651 /* Also time stamps V2 L2 Path Delay Request/Response. */ 3652 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2; 3653 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE; 3654 is_l2 = true; 3655 break; 3656 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 3657 /* Hardware cannot filter just V2 L4 Sync messages */ 3658 fallthrough; 3659 case HWTSTAMP_FILTER_PTP_V2_SYNC: 3660 /* Also time stamps V2 Path Delay Request/Response. */ 3661 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2; 3662 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE; 3663 is_l2 = true; 3664 is_l4 = true; 3665 break; 3666 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 3667 /* Hardware cannot filter just V2 L4 Delay Request messages */ 3668 fallthrough; 3669 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 3670 /* Also time stamps V2 Path Delay Request/Response. */ 3671 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2; 3672 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE; 3673 is_l2 = true; 3674 is_l4 = true; 3675 break; 3676 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 3677 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 3678 /* Hardware cannot filter just V2 L4 or L2 Event messages */ 3679 fallthrough; 3680 case HWTSTAMP_FILTER_PTP_V2_EVENT: 3681 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2; 3682 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 3683 is_l2 = true; 3684 is_l4 = true; 3685 break; 3686 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 3687 /* For V1, the hardware can only filter Sync messages or 3688 * Delay Request messages but not both so fall-through to 3689 * time stamp all packets. 3690 */ 3691 fallthrough; 3692 case HWTSTAMP_FILTER_NTP_ALL: 3693 case HWTSTAMP_FILTER_ALL: 3694 is_l2 = true; 3695 is_l4 = true; 3696 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL; 3697 config->rx_filter = HWTSTAMP_FILTER_ALL; 3698 break; 3699 default: 3700 return -ERANGE; 3701 } 3702 3703 adapter->hwtstamp_config = *config; 3704 3705 /* enable/disable Tx h/w time stamping */ 3706 regval = er32(TSYNCTXCTL); 3707 regval &= ~E1000_TSYNCTXCTL_ENABLED; 3708 regval |= tsync_tx_ctl; 3709 ew32(TSYNCTXCTL, regval); 3710 if ((er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) != 3711 (regval & E1000_TSYNCTXCTL_ENABLED)) { 3712 e_err("Timesync Tx Control register not set as expected\n"); 3713 return -EAGAIN; 3714 } 3715 3716 /* enable/disable Rx h/w time stamping */ 3717 regval = er32(TSYNCRXCTL); 3718 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK); 3719 regval |= tsync_rx_ctl; 3720 ew32(TSYNCRXCTL, regval); 3721 if ((er32(TSYNCRXCTL) & (E1000_TSYNCRXCTL_ENABLED | 3722 E1000_TSYNCRXCTL_TYPE_MASK)) != 3723 (regval & (E1000_TSYNCRXCTL_ENABLED | 3724 E1000_TSYNCRXCTL_TYPE_MASK))) { 3725 e_err("Timesync Rx Control register not set as expected\n"); 3726 return -EAGAIN; 3727 } 3728 3729 /* L2: define ethertype filter for time stamped packets */ 3730 if (is_l2) 3731 rxmtrl |= ETH_P_1588; 3732 3733 /* define which PTP packets get time stamped */ 3734 ew32(RXMTRL, rxmtrl); 3735 3736 /* Filter by destination port */ 3737 if (is_l4) { 3738 rxudp = PTP_EV_PORT; 3739 cpu_to_be16s(&rxudp); 3740 } 3741 ew32(RXUDP, rxudp); 3742 3743 e1e_flush(); 3744 3745 /* Clear TSYNCRXCTL_VALID & TSYNCTXCTL_VALID bit */ 3746 er32(RXSTMPH); 3747 er32(TXSTMPH); 3748 3749 return 0; 3750 } 3751 3752 /** 3753 * e1000_configure - configure the hardware for Rx and Tx 3754 * @adapter: private board structure 3755 **/ 3756 static void e1000_configure(struct e1000_adapter *adapter) 3757 { 3758 struct e1000_ring *rx_ring = adapter->rx_ring; 3759 3760 e1000e_set_rx_mode(adapter->netdev); 3761 3762 e1000_restore_vlan(adapter); 3763 e1000_init_manageability_pt(adapter); 3764 3765 e1000_configure_tx(adapter); 3766 3767 if (adapter->netdev->features & NETIF_F_RXHASH) 3768 e1000e_setup_rss_hash(adapter); 3769 e1000_setup_rctl(adapter); 3770 e1000_configure_rx(adapter); 3771 adapter->alloc_rx_buf(rx_ring, e1000_desc_unused(rx_ring), GFP_KERNEL); 3772 } 3773 3774 /** 3775 * e1000e_power_up_phy - restore link in case the phy was powered down 3776 * @adapter: address of board private structure 3777 * 3778 * The phy may be powered down to save power and turn off link when the 3779 * driver is unloaded and wake on lan is not enabled (among others) 3780 * *** this routine MUST be followed by a call to e1000e_reset *** 3781 **/ 3782 void e1000e_power_up_phy(struct e1000_adapter *adapter) 3783 { 3784 if (adapter->hw.phy.ops.power_up) 3785 adapter->hw.phy.ops.power_up(&adapter->hw); 3786 3787 adapter->hw.mac.ops.setup_link(&adapter->hw); 3788 } 3789 3790 /** 3791 * e1000_power_down_phy - Power down the PHY 3792 * @adapter: board private structure 3793 * 3794 * Power down the PHY so no link is implied when interface is down. 3795 * The PHY cannot be powered down if management or WoL is active. 3796 */ 3797 static void e1000_power_down_phy(struct e1000_adapter *adapter) 3798 { 3799 if (adapter->hw.phy.ops.power_down) 3800 adapter->hw.phy.ops.power_down(&adapter->hw); 3801 } 3802 3803 /** 3804 * e1000_flush_tx_ring - remove all descriptors from the tx_ring 3805 * @adapter: board private structure 3806 * 3807 * We want to clear all pending descriptors from the TX ring. 3808 * zeroing happens when the HW reads the regs. We assign the ring itself as 3809 * the data of the next descriptor. We don't care about the data we are about 3810 * to reset the HW. 3811 */ 3812 static void e1000_flush_tx_ring(struct e1000_adapter *adapter) 3813 { 3814 struct e1000_hw *hw = &adapter->hw; 3815 struct e1000_ring *tx_ring = adapter->tx_ring; 3816 struct e1000_tx_desc *tx_desc = NULL; 3817 u32 tdt, tctl, txd_lower = E1000_TXD_CMD_IFCS; 3818 u16 size = 512; 3819 3820 tctl = er32(TCTL); 3821 ew32(TCTL, tctl | E1000_TCTL_EN); 3822 tdt = er32(TDT(0)); 3823 BUG_ON(tdt != tx_ring->next_to_use); 3824 tx_desc = E1000_TX_DESC(*tx_ring, tx_ring->next_to_use); 3825 tx_desc->buffer_addr = cpu_to_le64(tx_ring->dma); 3826 3827 tx_desc->lower.data = cpu_to_le32(txd_lower | size); 3828 tx_desc->upper.data = 0; 3829 /* flush descriptors to memory before notifying the HW */ 3830 wmb(); 3831 tx_ring->next_to_use++; 3832 if (tx_ring->next_to_use == tx_ring->count) 3833 tx_ring->next_to_use = 0; 3834 ew32(TDT(0), tx_ring->next_to_use); 3835 usleep_range(200, 250); 3836 } 3837 3838 /** 3839 * e1000_flush_rx_ring - remove all descriptors from the rx_ring 3840 * @adapter: board private structure 3841 * 3842 * Mark all descriptors in the RX ring as consumed and disable the rx ring 3843 */ 3844 static void e1000_flush_rx_ring(struct e1000_adapter *adapter) 3845 { 3846 u32 rctl, rxdctl; 3847 struct e1000_hw *hw = &adapter->hw; 3848 3849 rctl = er32(RCTL); 3850 ew32(RCTL, rctl & ~E1000_RCTL_EN); 3851 e1e_flush(); 3852 usleep_range(100, 150); 3853 3854 rxdctl = er32(RXDCTL(0)); 3855 /* zero the lower 14 bits (prefetch and host thresholds) */ 3856 rxdctl &= 0xffffc000; 3857 3858 /* update thresholds: prefetch threshold to 31, host threshold to 1 3859 * and make sure the granularity is "descriptors" and not "cache lines" 3860 */ 3861 rxdctl |= (0x1F | BIT(8) | E1000_RXDCTL_THRESH_UNIT_DESC); 3862 3863 ew32(RXDCTL(0), rxdctl); 3864 /* momentarily enable the RX ring for the changes to take effect */ 3865 ew32(RCTL, rctl | E1000_RCTL_EN); 3866 e1e_flush(); 3867 usleep_range(100, 150); 3868 ew32(RCTL, rctl & ~E1000_RCTL_EN); 3869 } 3870 3871 /** 3872 * e1000_flush_desc_rings - remove all descriptors from the descriptor rings 3873 * @adapter: board private structure 3874 * 3875 * In i219, the descriptor rings must be emptied before resetting the HW 3876 * or before changing the device state to D3 during runtime (runtime PM). 3877 * 3878 * Failure to do this will cause the HW to enter a unit hang state which can 3879 * only be released by PCI reset on the device 3880 * 3881 */ 3882 3883 static void e1000_flush_desc_rings(struct e1000_adapter *adapter) 3884 { 3885 u16 hang_state; 3886 u32 fext_nvm11, tdlen; 3887 struct e1000_hw *hw = &adapter->hw; 3888 3889 /* First, disable MULR fix in FEXTNVM11 */ 3890 fext_nvm11 = er32(FEXTNVM11); 3891 fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX; 3892 ew32(FEXTNVM11, fext_nvm11); 3893 /* do nothing if we're not in faulty state, or if the queue is empty */ 3894 tdlen = er32(TDLEN(0)); 3895 pci_read_config_word(adapter->pdev, PCICFG_DESC_RING_STATUS, 3896 &hang_state); 3897 if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen) 3898 return; 3899 e1000_flush_tx_ring(adapter); 3900 /* recheck, maybe the fault is caused by the rx ring */ 3901 pci_read_config_word(adapter->pdev, PCICFG_DESC_RING_STATUS, 3902 &hang_state); 3903 if (hang_state & FLUSH_DESC_REQUIRED) 3904 e1000_flush_rx_ring(adapter); 3905 } 3906 3907 /** 3908 * e1000e_systim_reset - reset the timesync registers after a hardware reset 3909 * @adapter: board private structure 3910 * 3911 * When the MAC is reset, all hardware bits for timesync will be reset to the 3912 * default values. This function will restore the settings last in place. 3913 * Since the clock SYSTIME registers are reset, we will simply restore the 3914 * cyclecounter to the kernel real clock time. 3915 **/ 3916 static void e1000e_systim_reset(struct e1000_adapter *adapter) 3917 { 3918 struct ptp_clock_info *info = &adapter->ptp_clock_info; 3919 struct e1000_hw *hw = &adapter->hw; 3920 unsigned long flags; 3921 u32 timinca; 3922 s32 ret_val; 3923 3924 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP)) 3925 return; 3926 3927 if (info->adjfreq) { 3928 /* restore the previous ptp frequency delta */ 3929 ret_val = info->adjfreq(info, adapter->ptp_delta); 3930 } else { 3931 /* set the default base frequency if no adjustment possible */ 3932 ret_val = e1000e_get_base_timinca(adapter, &timinca); 3933 if (!ret_val) 3934 ew32(TIMINCA, timinca); 3935 } 3936 3937 if (ret_val) { 3938 dev_warn(&adapter->pdev->dev, 3939 "Failed to restore TIMINCA clock rate delta: %d\n", 3940 ret_val); 3941 return; 3942 } 3943 3944 /* reset the systim ns time counter */ 3945 spin_lock_irqsave(&adapter->systim_lock, flags); 3946 timecounter_init(&adapter->tc, &adapter->cc, 3947 ktime_to_ns(ktime_get_real())); 3948 spin_unlock_irqrestore(&adapter->systim_lock, flags); 3949 3950 /* restore the previous hwtstamp configuration settings */ 3951 e1000e_config_hwtstamp(adapter, &adapter->hwtstamp_config); 3952 } 3953 3954 /** 3955 * e1000e_reset - bring the hardware into a known good state 3956 * @adapter: board private structure 3957 * 3958 * This function boots the hardware and enables some settings that 3959 * require a configuration cycle of the hardware - those cannot be 3960 * set/changed during runtime. After reset the device needs to be 3961 * properly configured for Rx, Tx etc. 3962 */ 3963 void e1000e_reset(struct e1000_adapter *adapter) 3964 { 3965 struct e1000_mac_info *mac = &adapter->hw.mac; 3966 struct e1000_fc_info *fc = &adapter->hw.fc; 3967 struct e1000_hw *hw = &adapter->hw; 3968 u32 tx_space, min_tx_space, min_rx_space; 3969 u32 pba = adapter->pba; 3970 u16 hwm; 3971 3972 /* reset Packet Buffer Allocation to default */ 3973 ew32(PBA, pba); 3974 3975 if (adapter->max_frame_size > (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)) { 3976 /* To maintain wire speed transmits, the Tx FIFO should be 3977 * large enough to accommodate two full transmit packets, 3978 * rounded up to the next 1KB and expressed in KB. Likewise, 3979 * the Rx FIFO should be large enough to accommodate at least 3980 * one full receive packet and is similarly rounded up and 3981 * expressed in KB. 3982 */ 3983 pba = er32(PBA); 3984 /* upper 16 bits has Tx packet buffer allocation size in KB */ 3985 tx_space = pba >> 16; 3986 /* lower 16 bits has Rx packet buffer allocation size in KB */ 3987 pba &= 0xffff; 3988 /* the Tx fifo also stores 16 bytes of information about the Tx 3989 * but don't include ethernet FCS because hardware appends it 3990 */ 3991 min_tx_space = (adapter->max_frame_size + 3992 sizeof(struct e1000_tx_desc) - ETH_FCS_LEN) * 2; 3993 min_tx_space = ALIGN(min_tx_space, 1024); 3994 min_tx_space >>= 10; 3995 /* software strips receive CRC, so leave room for it */ 3996 min_rx_space = adapter->max_frame_size; 3997 min_rx_space = ALIGN(min_rx_space, 1024); 3998 min_rx_space >>= 10; 3999 4000 /* If current Tx allocation is less than the min Tx FIFO size, 4001 * and the min Tx FIFO size is less than the current Rx FIFO 4002 * allocation, take space away from current Rx allocation 4003 */ 4004 if ((tx_space < min_tx_space) && 4005 ((min_tx_space - tx_space) < pba)) { 4006 pba -= min_tx_space - tx_space; 4007 4008 /* if short on Rx space, Rx wins and must trump Tx 4009 * adjustment 4010 */ 4011 if (pba < min_rx_space) 4012 pba = min_rx_space; 4013 } 4014 4015 ew32(PBA, pba); 4016 } 4017 4018 /* flow control settings 4019 * 4020 * The high water mark must be low enough to fit one full frame 4021 * (or the size used for early receive) above it in the Rx FIFO. 4022 * Set it to the lower of: 4023 * - 90% of the Rx FIFO size, and 4024 * - the full Rx FIFO size minus one full frame 4025 */ 4026 if (adapter->flags & FLAG_DISABLE_FC_PAUSE_TIME) 4027 fc->pause_time = 0xFFFF; 4028 else 4029 fc->pause_time = E1000_FC_PAUSE_TIME; 4030 fc->send_xon = true; 4031 fc->current_mode = fc->requested_mode; 4032 4033 switch (hw->mac.type) { 4034 case e1000_ich9lan: 4035 case e1000_ich10lan: 4036 if (adapter->netdev->mtu > ETH_DATA_LEN) { 4037 pba = 14; 4038 ew32(PBA, pba); 4039 fc->high_water = 0x2800; 4040 fc->low_water = fc->high_water - 8; 4041 break; 4042 } 4043 fallthrough; 4044 default: 4045 hwm = min(((pba << 10) * 9 / 10), 4046 ((pba << 10) - adapter->max_frame_size)); 4047 4048 fc->high_water = hwm & E1000_FCRTH_RTH; /* 8-byte granularity */ 4049 fc->low_water = fc->high_water - 8; 4050 break; 4051 case e1000_pchlan: 4052 /* Workaround PCH LOM adapter hangs with certain network 4053 * loads. If hangs persist, try disabling Tx flow control. 4054 */ 4055 if (adapter->netdev->mtu > ETH_DATA_LEN) { 4056 fc->high_water = 0x3500; 4057 fc->low_water = 0x1500; 4058 } else { 4059 fc->high_water = 0x5000; 4060 fc->low_water = 0x3000; 4061 } 4062 fc->refresh_time = 0x1000; 4063 break; 4064 case e1000_pch2lan: 4065 case e1000_pch_lpt: 4066 case e1000_pch_spt: 4067 case e1000_pch_cnp: 4068 case e1000_pch_tgp: 4069 case e1000_pch_adp: 4070 case e1000_pch_mtp: 4071 fc->refresh_time = 0xFFFF; 4072 fc->pause_time = 0xFFFF; 4073 4074 if (adapter->netdev->mtu <= ETH_DATA_LEN) { 4075 fc->high_water = 0x05C20; 4076 fc->low_water = 0x05048; 4077 break; 4078 } 4079 4080 pba = 14; 4081 ew32(PBA, pba); 4082 fc->high_water = ((pba << 10) * 9 / 10) & E1000_FCRTH_RTH; 4083 fc->low_water = ((pba << 10) * 8 / 10) & E1000_FCRTL_RTL; 4084 break; 4085 } 4086 4087 /* Alignment of Tx data is on an arbitrary byte boundary with the 4088 * maximum size per Tx descriptor limited only to the transmit 4089 * allocation of the packet buffer minus 96 bytes with an upper 4090 * limit of 24KB due to receive synchronization limitations. 4091 */ 4092 adapter->tx_fifo_limit = min_t(u32, ((er32(PBA) >> 16) << 10) - 96, 4093 24 << 10); 4094 4095 /* Disable Adaptive Interrupt Moderation if 2 full packets cannot 4096 * fit in receive buffer. 4097 */ 4098 if (adapter->itr_setting & 0x3) { 4099 if ((adapter->max_frame_size * 2) > (pba << 10)) { 4100 if (!(adapter->flags2 & FLAG2_DISABLE_AIM)) { 4101 dev_info(&adapter->pdev->dev, 4102 "Interrupt Throttle Rate off\n"); 4103 adapter->flags2 |= FLAG2_DISABLE_AIM; 4104 e1000e_write_itr(adapter, 0); 4105 } 4106 } else if (adapter->flags2 & FLAG2_DISABLE_AIM) { 4107 dev_info(&adapter->pdev->dev, 4108 "Interrupt Throttle Rate on\n"); 4109 adapter->flags2 &= ~FLAG2_DISABLE_AIM; 4110 adapter->itr = 20000; 4111 e1000e_write_itr(adapter, adapter->itr); 4112 } 4113 } 4114 4115 if (hw->mac.type >= e1000_pch_spt) 4116 e1000_flush_desc_rings(adapter); 4117 /* Allow time for pending master requests to run */ 4118 mac->ops.reset_hw(hw); 4119 4120 /* For parts with AMT enabled, let the firmware know 4121 * that the network interface is in control 4122 */ 4123 if (adapter->flags & FLAG_HAS_AMT) 4124 e1000e_get_hw_control(adapter); 4125 4126 ew32(WUC, 0); 4127 4128 if (mac->ops.init_hw(hw)) 4129 e_err("Hardware Error\n"); 4130 4131 e1000_update_mng_vlan(adapter); 4132 4133 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */ 4134 ew32(VET, ETH_P_8021Q); 4135 4136 e1000e_reset_adaptive(hw); 4137 4138 /* restore systim and hwtstamp settings */ 4139 e1000e_systim_reset(adapter); 4140 4141 /* Set EEE advertisement as appropriate */ 4142 if (adapter->flags2 & FLAG2_HAS_EEE) { 4143 s32 ret_val; 4144 u16 adv_addr; 4145 4146 switch (hw->phy.type) { 4147 case e1000_phy_82579: 4148 adv_addr = I82579_EEE_ADVERTISEMENT; 4149 break; 4150 case e1000_phy_i217: 4151 adv_addr = I217_EEE_ADVERTISEMENT; 4152 break; 4153 default: 4154 dev_err(&adapter->pdev->dev, 4155 "Invalid PHY type setting EEE advertisement\n"); 4156 return; 4157 } 4158 4159 ret_val = hw->phy.ops.acquire(hw); 4160 if (ret_val) { 4161 dev_err(&adapter->pdev->dev, 4162 "EEE advertisement - unable to acquire PHY\n"); 4163 return; 4164 } 4165 4166 e1000_write_emi_reg_locked(hw, adv_addr, 4167 hw->dev_spec.ich8lan.eee_disable ? 4168 0 : adapter->eee_advert); 4169 4170 hw->phy.ops.release(hw); 4171 } 4172 4173 if (!netif_running(adapter->netdev) && 4174 !test_bit(__E1000_TESTING, &adapter->state)) 4175 e1000_power_down_phy(adapter); 4176 4177 e1000_get_phy_info(hw); 4178 4179 if ((adapter->flags & FLAG_HAS_SMART_POWER_DOWN) && 4180 !(adapter->flags & FLAG_SMART_POWER_DOWN)) { 4181 u16 phy_data = 0; 4182 /* speed up time to link by disabling smart power down, ignore 4183 * the return value of this function because there is nothing 4184 * different we would do if it failed 4185 */ 4186 e1e_rphy(hw, IGP02E1000_PHY_POWER_MGMT, &phy_data); 4187 phy_data &= ~IGP02E1000_PM_SPD; 4188 e1e_wphy(hw, IGP02E1000_PHY_POWER_MGMT, phy_data); 4189 } 4190 if (hw->mac.type >= e1000_pch_spt && adapter->int_mode == 0) { 4191 u32 reg; 4192 4193 /* Fextnvm7 @ 0xe4[2] = 1 */ 4194 reg = er32(FEXTNVM7); 4195 reg |= E1000_FEXTNVM7_SIDE_CLK_UNGATE; 4196 ew32(FEXTNVM7, reg); 4197 /* Fextnvm9 @ 0x5bb4[13:12] = 11 */ 4198 reg = er32(FEXTNVM9); 4199 reg |= E1000_FEXTNVM9_IOSFSB_CLKGATE_DIS | 4200 E1000_FEXTNVM9_IOSFSB_CLKREQ_DIS; 4201 ew32(FEXTNVM9, reg); 4202 } 4203 4204 } 4205 4206 /** 4207 * e1000e_trigger_lsc - trigger an LSC interrupt 4208 * @adapter: 4209 * 4210 * Fire a link status change interrupt to start the watchdog. 4211 **/ 4212 static void e1000e_trigger_lsc(struct e1000_adapter *adapter) 4213 { 4214 struct e1000_hw *hw = &adapter->hw; 4215 4216 if (adapter->msix_entries) 4217 ew32(ICS, E1000_ICS_LSC | E1000_ICS_OTHER); 4218 else 4219 ew32(ICS, E1000_ICS_LSC); 4220 } 4221 4222 void e1000e_up(struct e1000_adapter *adapter) 4223 { 4224 /* hardware has been reset, we need to reload some things */ 4225 e1000_configure(adapter); 4226 4227 clear_bit(__E1000_DOWN, &adapter->state); 4228 4229 if (adapter->msix_entries) 4230 e1000_configure_msix(adapter); 4231 e1000_irq_enable(adapter); 4232 4233 /* Tx queue started by watchdog timer when link is up */ 4234 4235 e1000e_trigger_lsc(adapter); 4236 } 4237 4238 static void e1000e_flush_descriptors(struct e1000_adapter *adapter) 4239 { 4240 struct e1000_hw *hw = &adapter->hw; 4241 4242 if (!(adapter->flags2 & FLAG2_DMA_BURST)) 4243 return; 4244 4245 /* flush pending descriptor writebacks to memory */ 4246 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 4247 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD); 4248 4249 /* execute the writes immediately */ 4250 e1e_flush(); 4251 4252 /* due to rare timing issues, write to TIDV/RDTR again to ensure the 4253 * write is successful 4254 */ 4255 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 4256 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD); 4257 4258 /* execute the writes immediately */ 4259 e1e_flush(); 4260 } 4261 4262 static void e1000e_update_stats(struct e1000_adapter *adapter); 4263 4264 /** 4265 * e1000e_down - quiesce the device and optionally reset the hardware 4266 * @adapter: board private structure 4267 * @reset: boolean flag to reset the hardware or not 4268 */ 4269 void e1000e_down(struct e1000_adapter *adapter, bool reset) 4270 { 4271 struct net_device *netdev = adapter->netdev; 4272 struct e1000_hw *hw = &adapter->hw; 4273 u32 tctl, rctl; 4274 4275 /* signal that we're down so the interrupt handler does not 4276 * reschedule our watchdog timer 4277 */ 4278 set_bit(__E1000_DOWN, &adapter->state); 4279 4280 netif_carrier_off(netdev); 4281 4282 /* disable receives in the hardware */ 4283 rctl = er32(RCTL); 4284 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX)) 4285 ew32(RCTL, rctl & ~E1000_RCTL_EN); 4286 /* flush and sleep below */ 4287 4288 netif_stop_queue(netdev); 4289 4290 /* disable transmits in the hardware */ 4291 tctl = er32(TCTL); 4292 tctl &= ~E1000_TCTL_EN; 4293 ew32(TCTL, tctl); 4294 4295 /* flush both disables and wait for them to finish */ 4296 e1e_flush(); 4297 usleep_range(10000, 11000); 4298 4299 e1000_irq_disable(adapter); 4300 4301 napi_synchronize(&adapter->napi); 4302 4303 del_timer_sync(&adapter->watchdog_timer); 4304 del_timer_sync(&adapter->phy_info_timer); 4305 4306 spin_lock(&adapter->stats64_lock); 4307 e1000e_update_stats(adapter); 4308 spin_unlock(&adapter->stats64_lock); 4309 4310 e1000e_flush_descriptors(adapter); 4311 4312 adapter->link_speed = 0; 4313 adapter->link_duplex = 0; 4314 4315 /* Disable Si errata workaround on PCHx for jumbo frame flow */ 4316 if ((hw->mac.type >= e1000_pch2lan) && 4317 (adapter->netdev->mtu > ETH_DATA_LEN) && 4318 e1000_lv_jumbo_workaround_ich8lan(hw, false)) 4319 e_dbg("failed to disable jumbo frame workaround mode\n"); 4320 4321 if (!pci_channel_offline(adapter->pdev)) { 4322 if (reset) 4323 e1000e_reset(adapter); 4324 else if (hw->mac.type >= e1000_pch_spt) 4325 e1000_flush_desc_rings(adapter); 4326 } 4327 e1000_clean_tx_ring(adapter->tx_ring); 4328 e1000_clean_rx_ring(adapter->rx_ring); 4329 } 4330 4331 void e1000e_reinit_locked(struct e1000_adapter *adapter) 4332 { 4333 might_sleep(); 4334 while (test_and_set_bit(__E1000_RESETTING, &adapter->state)) 4335 usleep_range(1000, 1100); 4336 e1000e_down(adapter, true); 4337 e1000e_up(adapter); 4338 clear_bit(__E1000_RESETTING, &adapter->state); 4339 } 4340 4341 /** 4342 * e1000e_sanitize_systim - sanitize raw cycle counter reads 4343 * @hw: pointer to the HW structure 4344 * @systim: PHC time value read, sanitized and returned 4345 * @sts: structure to hold system time before and after reading SYSTIML, 4346 * may be NULL 4347 * 4348 * Errata for 82574/82583 possible bad bits read from SYSTIMH/L: 4349 * check to see that the time is incrementing at a reasonable 4350 * rate and is a multiple of incvalue. 4351 **/ 4352 static u64 e1000e_sanitize_systim(struct e1000_hw *hw, u64 systim, 4353 struct ptp_system_timestamp *sts) 4354 { 4355 u64 time_delta, rem, temp; 4356 u64 systim_next; 4357 u32 incvalue; 4358 int i; 4359 4360 incvalue = er32(TIMINCA) & E1000_TIMINCA_INCVALUE_MASK; 4361 for (i = 0; i < E1000_MAX_82574_SYSTIM_REREADS; i++) { 4362 /* latch SYSTIMH on read of SYSTIML */ 4363 ptp_read_system_prets(sts); 4364 systim_next = (u64)er32(SYSTIML); 4365 ptp_read_system_postts(sts); 4366 systim_next |= (u64)er32(SYSTIMH) << 32; 4367 4368 time_delta = systim_next - systim; 4369 temp = time_delta; 4370 /* VMWare users have seen incvalue of zero, don't div / 0 */ 4371 rem = incvalue ? do_div(temp, incvalue) : (time_delta != 0); 4372 4373 systim = systim_next; 4374 4375 if ((time_delta < E1000_82574_SYSTIM_EPSILON) && (rem == 0)) 4376 break; 4377 } 4378 4379 return systim; 4380 } 4381 4382 /** 4383 * e1000e_read_systim - read SYSTIM register 4384 * @adapter: board private structure 4385 * @sts: structure which will contain system time before and after reading 4386 * SYSTIML, may be NULL 4387 **/ 4388 u64 e1000e_read_systim(struct e1000_adapter *adapter, 4389 struct ptp_system_timestamp *sts) 4390 { 4391 struct e1000_hw *hw = &adapter->hw; 4392 u32 systimel, systimel_2, systimeh; 4393 u64 systim; 4394 /* SYSTIMH latching upon SYSTIML read does not work well. 4395 * This means that if SYSTIML overflows after we read it but before 4396 * we read SYSTIMH, the value of SYSTIMH has been incremented and we 4397 * will experience a huge non linear increment in the systime value 4398 * to fix that we test for overflow and if true, we re-read systime. 4399 */ 4400 ptp_read_system_prets(sts); 4401 systimel = er32(SYSTIML); 4402 ptp_read_system_postts(sts); 4403 systimeh = er32(SYSTIMH); 4404 /* Is systimel is so large that overflow is possible? */ 4405 if (systimel >= (u32)0xffffffff - E1000_TIMINCA_INCVALUE_MASK) { 4406 ptp_read_system_prets(sts); 4407 systimel_2 = er32(SYSTIML); 4408 ptp_read_system_postts(sts); 4409 if (systimel > systimel_2) { 4410 /* There was an overflow, read again SYSTIMH, and use 4411 * systimel_2 4412 */ 4413 systimeh = er32(SYSTIMH); 4414 systimel = systimel_2; 4415 } 4416 } 4417 systim = (u64)systimel; 4418 systim |= (u64)systimeh << 32; 4419 4420 if (adapter->flags2 & FLAG2_CHECK_SYSTIM_OVERFLOW) 4421 systim = e1000e_sanitize_systim(hw, systim, sts); 4422 4423 return systim; 4424 } 4425 4426 /** 4427 * e1000e_cyclecounter_read - read raw cycle counter (used by time counter) 4428 * @cc: cyclecounter structure 4429 **/ 4430 static u64 e1000e_cyclecounter_read(const struct cyclecounter *cc) 4431 { 4432 struct e1000_adapter *adapter = container_of(cc, struct e1000_adapter, 4433 cc); 4434 4435 return e1000e_read_systim(adapter, NULL); 4436 } 4437 4438 /** 4439 * e1000_sw_init - Initialize general software structures (struct e1000_adapter) 4440 * @adapter: board private structure to initialize 4441 * 4442 * e1000_sw_init initializes the Adapter private data structure. 4443 * Fields are initialized based on PCI device information and 4444 * OS network device settings (MTU size). 4445 **/ 4446 static int e1000_sw_init(struct e1000_adapter *adapter) 4447 { 4448 struct net_device *netdev = adapter->netdev; 4449 4450 adapter->rx_buffer_len = VLAN_ETH_FRAME_LEN + ETH_FCS_LEN; 4451 adapter->rx_ps_bsize0 = 128; 4452 adapter->max_frame_size = netdev->mtu + VLAN_ETH_HLEN + ETH_FCS_LEN; 4453 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 4454 adapter->tx_ring_count = E1000_DEFAULT_TXD; 4455 adapter->rx_ring_count = E1000_DEFAULT_RXD; 4456 4457 spin_lock_init(&adapter->stats64_lock); 4458 4459 e1000e_set_interrupt_capability(adapter); 4460 4461 if (e1000_alloc_queues(adapter)) 4462 return -ENOMEM; 4463 4464 /* Setup hardware time stamping cyclecounter */ 4465 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) { 4466 adapter->cc.read = e1000e_cyclecounter_read; 4467 adapter->cc.mask = CYCLECOUNTER_MASK(64); 4468 adapter->cc.mult = 1; 4469 /* cc.shift set in e1000e_get_base_tininca() */ 4470 4471 spin_lock_init(&adapter->systim_lock); 4472 INIT_WORK(&adapter->tx_hwtstamp_work, e1000e_tx_hwtstamp_work); 4473 } 4474 4475 /* Explicitly disable IRQ since the NIC can be in any state. */ 4476 e1000_irq_disable(adapter); 4477 4478 set_bit(__E1000_DOWN, &adapter->state); 4479 return 0; 4480 } 4481 4482 /** 4483 * e1000_intr_msi_test - Interrupt Handler 4484 * @irq: interrupt number 4485 * @data: pointer to a network interface device structure 4486 **/ 4487 static irqreturn_t e1000_intr_msi_test(int __always_unused irq, void *data) 4488 { 4489 struct net_device *netdev = data; 4490 struct e1000_adapter *adapter = netdev_priv(netdev); 4491 struct e1000_hw *hw = &adapter->hw; 4492 u32 icr = er32(ICR); 4493 4494 e_dbg("icr is %08X\n", icr); 4495 if (icr & E1000_ICR_RXSEQ) { 4496 adapter->flags &= ~FLAG_MSI_TEST_FAILED; 4497 /* Force memory writes to complete before acknowledging the 4498 * interrupt is handled. 4499 */ 4500 wmb(); 4501 } 4502 4503 return IRQ_HANDLED; 4504 } 4505 4506 /** 4507 * e1000_test_msi_interrupt - Returns 0 for successful test 4508 * @adapter: board private struct 4509 * 4510 * code flow taken from tg3.c 4511 **/ 4512 static int e1000_test_msi_interrupt(struct e1000_adapter *adapter) 4513 { 4514 struct net_device *netdev = adapter->netdev; 4515 struct e1000_hw *hw = &adapter->hw; 4516 int err; 4517 4518 /* poll_enable hasn't been called yet, so don't need disable */ 4519 /* clear any pending events */ 4520 er32(ICR); 4521 4522 /* free the real vector and request a test handler */ 4523 e1000_free_irq(adapter); 4524 e1000e_reset_interrupt_capability(adapter); 4525 4526 /* Assume that the test fails, if it succeeds then the test 4527 * MSI irq handler will unset this flag 4528 */ 4529 adapter->flags |= FLAG_MSI_TEST_FAILED; 4530 4531 err = pci_enable_msi(adapter->pdev); 4532 if (err) 4533 goto msi_test_failed; 4534 4535 err = request_irq(adapter->pdev->irq, e1000_intr_msi_test, 0, 4536 netdev->name, netdev); 4537 if (err) { 4538 pci_disable_msi(adapter->pdev); 4539 goto msi_test_failed; 4540 } 4541 4542 /* Force memory writes to complete before enabling and firing an 4543 * interrupt. 4544 */ 4545 wmb(); 4546 4547 e1000_irq_enable(adapter); 4548 4549 /* fire an unusual interrupt on the test handler */ 4550 ew32(ICS, E1000_ICS_RXSEQ); 4551 e1e_flush(); 4552 msleep(100); 4553 4554 e1000_irq_disable(adapter); 4555 4556 rmb(); /* read flags after interrupt has been fired */ 4557 4558 if (adapter->flags & FLAG_MSI_TEST_FAILED) { 4559 adapter->int_mode = E1000E_INT_MODE_LEGACY; 4560 e_info("MSI interrupt test failed, using legacy interrupt.\n"); 4561 } else { 4562 e_dbg("MSI interrupt test succeeded!\n"); 4563 } 4564 4565 free_irq(adapter->pdev->irq, netdev); 4566 pci_disable_msi(adapter->pdev); 4567 4568 msi_test_failed: 4569 e1000e_set_interrupt_capability(adapter); 4570 return e1000_request_irq(adapter); 4571 } 4572 4573 /** 4574 * e1000_test_msi - Returns 0 if MSI test succeeds or INTx mode is restored 4575 * @adapter: board private struct 4576 * 4577 * code flow taken from tg3.c, called with e1000 interrupts disabled. 4578 **/ 4579 static int e1000_test_msi(struct e1000_adapter *adapter) 4580 { 4581 int err; 4582 u16 pci_cmd; 4583 4584 if (!(adapter->flags & FLAG_MSI_ENABLED)) 4585 return 0; 4586 4587 /* disable SERR in case the MSI write causes a master abort */ 4588 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd); 4589 if (pci_cmd & PCI_COMMAND_SERR) 4590 pci_write_config_word(adapter->pdev, PCI_COMMAND, 4591 pci_cmd & ~PCI_COMMAND_SERR); 4592 4593 err = e1000_test_msi_interrupt(adapter); 4594 4595 /* re-enable SERR */ 4596 if (pci_cmd & PCI_COMMAND_SERR) { 4597 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd); 4598 pci_cmd |= PCI_COMMAND_SERR; 4599 pci_write_config_word(adapter->pdev, PCI_COMMAND, pci_cmd); 4600 } 4601 4602 return err; 4603 } 4604 4605 /** 4606 * e1000e_open - Called when a network interface is made active 4607 * @netdev: network interface device structure 4608 * 4609 * Returns 0 on success, negative value on failure 4610 * 4611 * The open entry point is called when a network interface is made 4612 * active by the system (IFF_UP). At this point all resources needed 4613 * for transmit and receive operations are allocated, the interrupt 4614 * handler is registered with the OS, the watchdog timer is started, 4615 * and the stack is notified that the interface is ready. 4616 **/ 4617 int e1000e_open(struct net_device *netdev) 4618 { 4619 struct e1000_adapter *adapter = netdev_priv(netdev); 4620 struct e1000_hw *hw = &adapter->hw; 4621 struct pci_dev *pdev = adapter->pdev; 4622 int err; 4623 4624 /* disallow open during test */ 4625 if (test_bit(__E1000_TESTING, &adapter->state)) 4626 return -EBUSY; 4627 4628 pm_runtime_get_sync(&pdev->dev); 4629 4630 netif_carrier_off(netdev); 4631 netif_stop_queue(netdev); 4632 4633 /* allocate transmit descriptors */ 4634 err = e1000e_setup_tx_resources(adapter->tx_ring); 4635 if (err) 4636 goto err_setup_tx; 4637 4638 /* allocate receive descriptors */ 4639 err = e1000e_setup_rx_resources(adapter->rx_ring); 4640 if (err) 4641 goto err_setup_rx; 4642 4643 /* If AMT is enabled, let the firmware know that the network 4644 * interface is now open and reset the part to a known state. 4645 */ 4646 if (adapter->flags & FLAG_HAS_AMT) { 4647 e1000e_get_hw_control(adapter); 4648 e1000e_reset(adapter); 4649 } 4650 4651 e1000e_power_up_phy(adapter); 4652 4653 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE; 4654 if ((adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN)) 4655 e1000_update_mng_vlan(adapter); 4656 4657 /* DMA latency requirement to workaround jumbo issue */ 4658 cpu_latency_qos_add_request(&adapter->pm_qos_req, PM_QOS_DEFAULT_VALUE); 4659 4660 /* before we allocate an interrupt, we must be ready to handle it. 4661 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt 4662 * as soon as we call pci_request_irq, so we have to setup our 4663 * clean_rx handler before we do so. 4664 */ 4665 e1000_configure(adapter); 4666 4667 err = e1000_request_irq(adapter); 4668 if (err) 4669 goto err_req_irq; 4670 4671 /* Work around PCIe errata with MSI interrupts causing some chipsets to 4672 * ignore e1000e MSI messages, which means we need to test our MSI 4673 * interrupt now 4674 */ 4675 if (adapter->int_mode != E1000E_INT_MODE_LEGACY) { 4676 err = e1000_test_msi(adapter); 4677 if (err) { 4678 e_err("Interrupt allocation failed\n"); 4679 goto err_req_irq; 4680 } 4681 } 4682 4683 /* From here on the code is the same as e1000e_up() */ 4684 clear_bit(__E1000_DOWN, &adapter->state); 4685 4686 napi_enable(&adapter->napi); 4687 4688 e1000_irq_enable(adapter); 4689 4690 adapter->tx_hang_recheck = false; 4691 4692 hw->mac.get_link_status = true; 4693 pm_runtime_put(&pdev->dev); 4694 4695 e1000e_trigger_lsc(adapter); 4696 4697 return 0; 4698 4699 err_req_irq: 4700 cpu_latency_qos_remove_request(&adapter->pm_qos_req); 4701 e1000e_release_hw_control(adapter); 4702 e1000_power_down_phy(adapter); 4703 e1000e_free_rx_resources(adapter->rx_ring); 4704 err_setup_rx: 4705 e1000e_free_tx_resources(adapter->tx_ring); 4706 err_setup_tx: 4707 e1000e_reset(adapter); 4708 pm_runtime_put_sync(&pdev->dev); 4709 4710 return err; 4711 } 4712 4713 /** 4714 * e1000e_close - Disables a network interface 4715 * @netdev: network interface device structure 4716 * 4717 * Returns 0, this is not allowed to fail 4718 * 4719 * The close entry point is called when an interface is de-activated 4720 * by the OS. The hardware is still under the drivers control, but 4721 * needs to be disabled. A global MAC reset is issued to stop the 4722 * hardware, and all transmit and receive resources are freed. 4723 **/ 4724 int e1000e_close(struct net_device *netdev) 4725 { 4726 struct e1000_adapter *adapter = netdev_priv(netdev); 4727 struct pci_dev *pdev = adapter->pdev; 4728 int count = E1000_CHECK_RESET_COUNT; 4729 4730 while (test_bit(__E1000_RESETTING, &adapter->state) && count--) 4731 usleep_range(10000, 11000); 4732 4733 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state)); 4734 4735 pm_runtime_get_sync(&pdev->dev); 4736 4737 if (netif_device_present(netdev)) { 4738 e1000e_down(adapter, true); 4739 e1000_free_irq(adapter); 4740 4741 /* Link status message must follow this format */ 4742 netdev_info(netdev, "NIC Link is Down\n"); 4743 } 4744 4745 napi_disable(&adapter->napi); 4746 4747 e1000e_free_tx_resources(adapter->tx_ring); 4748 e1000e_free_rx_resources(adapter->rx_ring); 4749 4750 /* kill manageability vlan ID if supported, but not if a vlan with 4751 * the same ID is registered on the host OS (let 8021q kill it) 4752 */ 4753 if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) 4754 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), 4755 adapter->mng_vlan_id); 4756 4757 /* If AMT is enabled, let the firmware know that the network 4758 * interface is now closed 4759 */ 4760 if ((adapter->flags & FLAG_HAS_AMT) && 4761 !test_bit(__E1000_TESTING, &adapter->state)) 4762 e1000e_release_hw_control(adapter); 4763 4764 cpu_latency_qos_remove_request(&adapter->pm_qos_req); 4765 4766 pm_runtime_put_sync(&pdev->dev); 4767 4768 return 0; 4769 } 4770 4771 /** 4772 * e1000_set_mac - Change the Ethernet Address of the NIC 4773 * @netdev: network interface device structure 4774 * @p: pointer to an address structure 4775 * 4776 * Returns 0 on success, negative on failure 4777 **/ 4778 static int e1000_set_mac(struct net_device *netdev, void *p) 4779 { 4780 struct e1000_adapter *adapter = netdev_priv(netdev); 4781 struct e1000_hw *hw = &adapter->hw; 4782 struct sockaddr *addr = p; 4783 4784 if (!is_valid_ether_addr(addr->sa_data)) 4785 return -EADDRNOTAVAIL; 4786 4787 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 4788 memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len); 4789 4790 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 4791 4792 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) { 4793 /* activate the work around */ 4794 e1000e_set_laa_state_82571(&adapter->hw, 1); 4795 4796 /* Hold a copy of the LAA in RAR[14] This is done so that 4797 * between the time RAR[0] gets clobbered and the time it 4798 * gets fixed (in e1000_watchdog), the actual LAA is in one 4799 * of the RARs and no incoming packets directed to this port 4800 * are dropped. Eventually the LAA will be in RAR[0] and 4801 * RAR[14] 4802 */ 4803 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 4804 adapter->hw.mac.rar_entry_count - 1); 4805 } 4806 4807 return 0; 4808 } 4809 4810 /** 4811 * e1000e_update_phy_task - work thread to update phy 4812 * @work: pointer to our work struct 4813 * 4814 * this worker thread exists because we must acquire a 4815 * semaphore to read the phy, which we could msleep while 4816 * waiting for it, and we can't msleep in a timer. 4817 **/ 4818 static void e1000e_update_phy_task(struct work_struct *work) 4819 { 4820 struct e1000_adapter *adapter = container_of(work, 4821 struct e1000_adapter, 4822 update_phy_task); 4823 struct e1000_hw *hw = &adapter->hw; 4824 4825 if (test_bit(__E1000_DOWN, &adapter->state)) 4826 return; 4827 4828 e1000_get_phy_info(hw); 4829 4830 /* Enable EEE on 82579 after link up */ 4831 if (hw->phy.type >= e1000_phy_82579) 4832 e1000_set_eee_pchlan(hw); 4833 } 4834 4835 /** 4836 * e1000_update_phy_info - timre call-back to update PHY info 4837 * @t: pointer to timer_list containing private info adapter 4838 * 4839 * Need to wait a few seconds after link up to get diagnostic information from 4840 * the phy 4841 **/ 4842 static void e1000_update_phy_info(struct timer_list *t) 4843 { 4844 struct e1000_adapter *adapter = from_timer(adapter, t, phy_info_timer); 4845 4846 if (test_bit(__E1000_DOWN, &adapter->state)) 4847 return; 4848 4849 schedule_work(&adapter->update_phy_task); 4850 } 4851 4852 /** 4853 * e1000e_update_phy_stats - Update the PHY statistics counters 4854 * @adapter: board private structure 4855 * 4856 * Read/clear the upper 16-bit PHY registers and read/accumulate lower 4857 **/ 4858 static void e1000e_update_phy_stats(struct e1000_adapter *adapter) 4859 { 4860 struct e1000_hw *hw = &adapter->hw; 4861 s32 ret_val; 4862 u16 phy_data; 4863 4864 ret_val = hw->phy.ops.acquire(hw); 4865 if (ret_val) 4866 return; 4867 4868 /* A page set is expensive so check if already on desired page. 4869 * If not, set to the page with the PHY status registers. 4870 */ 4871 hw->phy.addr = 1; 4872 ret_val = e1000e_read_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, 4873 &phy_data); 4874 if (ret_val) 4875 goto release; 4876 if (phy_data != (HV_STATS_PAGE << IGP_PAGE_SHIFT)) { 4877 ret_val = hw->phy.ops.set_page(hw, 4878 HV_STATS_PAGE << IGP_PAGE_SHIFT); 4879 if (ret_val) 4880 goto release; 4881 } 4882 4883 /* Single Collision Count */ 4884 hw->phy.ops.read_reg_page(hw, HV_SCC_UPPER, &phy_data); 4885 ret_val = hw->phy.ops.read_reg_page(hw, HV_SCC_LOWER, &phy_data); 4886 if (!ret_val) 4887 adapter->stats.scc += phy_data; 4888 4889 /* Excessive Collision Count */ 4890 hw->phy.ops.read_reg_page(hw, HV_ECOL_UPPER, &phy_data); 4891 ret_val = hw->phy.ops.read_reg_page(hw, HV_ECOL_LOWER, &phy_data); 4892 if (!ret_val) 4893 adapter->stats.ecol += phy_data; 4894 4895 /* Multiple Collision Count */ 4896 hw->phy.ops.read_reg_page(hw, HV_MCC_UPPER, &phy_data); 4897 ret_val = hw->phy.ops.read_reg_page(hw, HV_MCC_LOWER, &phy_data); 4898 if (!ret_val) 4899 adapter->stats.mcc += phy_data; 4900 4901 /* Late Collision Count */ 4902 hw->phy.ops.read_reg_page(hw, HV_LATECOL_UPPER, &phy_data); 4903 ret_val = hw->phy.ops.read_reg_page(hw, HV_LATECOL_LOWER, &phy_data); 4904 if (!ret_val) 4905 adapter->stats.latecol += phy_data; 4906 4907 /* Collision Count - also used for adaptive IFS */ 4908 hw->phy.ops.read_reg_page(hw, HV_COLC_UPPER, &phy_data); 4909 ret_val = hw->phy.ops.read_reg_page(hw, HV_COLC_LOWER, &phy_data); 4910 if (!ret_val) 4911 hw->mac.collision_delta = phy_data; 4912 4913 /* Defer Count */ 4914 hw->phy.ops.read_reg_page(hw, HV_DC_UPPER, &phy_data); 4915 ret_val = hw->phy.ops.read_reg_page(hw, HV_DC_LOWER, &phy_data); 4916 if (!ret_val) 4917 adapter->stats.dc += phy_data; 4918 4919 /* Transmit with no CRS */ 4920 hw->phy.ops.read_reg_page(hw, HV_TNCRS_UPPER, &phy_data); 4921 ret_val = hw->phy.ops.read_reg_page(hw, HV_TNCRS_LOWER, &phy_data); 4922 if (!ret_val) 4923 adapter->stats.tncrs += phy_data; 4924 4925 release: 4926 hw->phy.ops.release(hw); 4927 } 4928 4929 /** 4930 * e1000e_update_stats - Update the board statistics counters 4931 * @adapter: board private structure 4932 **/ 4933 static void e1000e_update_stats(struct e1000_adapter *adapter) 4934 { 4935 struct net_device *netdev = adapter->netdev; 4936 struct e1000_hw *hw = &adapter->hw; 4937 struct pci_dev *pdev = adapter->pdev; 4938 4939 /* Prevent stats update while adapter is being reset, or if the pci 4940 * connection is down. 4941 */ 4942 if (adapter->link_speed == 0) 4943 return; 4944 if (pci_channel_offline(pdev)) 4945 return; 4946 4947 adapter->stats.crcerrs += er32(CRCERRS); 4948 adapter->stats.gprc += er32(GPRC); 4949 adapter->stats.gorc += er32(GORCL); 4950 er32(GORCH); /* Clear gorc */ 4951 adapter->stats.bprc += er32(BPRC); 4952 adapter->stats.mprc += er32(MPRC); 4953 adapter->stats.roc += er32(ROC); 4954 4955 adapter->stats.mpc += er32(MPC); 4956 4957 /* Half-duplex statistics */ 4958 if (adapter->link_duplex == HALF_DUPLEX) { 4959 if (adapter->flags2 & FLAG2_HAS_PHY_STATS) { 4960 e1000e_update_phy_stats(adapter); 4961 } else { 4962 adapter->stats.scc += er32(SCC); 4963 adapter->stats.ecol += er32(ECOL); 4964 adapter->stats.mcc += er32(MCC); 4965 adapter->stats.latecol += er32(LATECOL); 4966 adapter->stats.dc += er32(DC); 4967 4968 hw->mac.collision_delta = er32(COLC); 4969 4970 if ((hw->mac.type != e1000_82574) && 4971 (hw->mac.type != e1000_82583)) 4972 adapter->stats.tncrs += er32(TNCRS); 4973 } 4974 adapter->stats.colc += hw->mac.collision_delta; 4975 } 4976 4977 adapter->stats.xonrxc += er32(XONRXC); 4978 adapter->stats.xontxc += er32(XONTXC); 4979 adapter->stats.xoffrxc += er32(XOFFRXC); 4980 adapter->stats.xofftxc += er32(XOFFTXC); 4981 adapter->stats.gptc += er32(GPTC); 4982 adapter->stats.gotc += er32(GOTCL); 4983 er32(GOTCH); /* Clear gotc */ 4984 adapter->stats.rnbc += er32(RNBC); 4985 adapter->stats.ruc += er32(RUC); 4986 4987 adapter->stats.mptc += er32(MPTC); 4988 adapter->stats.bptc += er32(BPTC); 4989 4990 /* used for adaptive IFS */ 4991 4992 hw->mac.tx_packet_delta = er32(TPT); 4993 adapter->stats.tpt += hw->mac.tx_packet_delta; 4994 4995 adapter->stats.algnerrc += er32(ALGNERRC); 4996 adapter->stats.rxerrc += er32(RXERRC); 4997 adapter->stats.cexterr += er32(CEXTERR); 4998 adapter->stats.tsctc += er32(TSCTC); 4999 adapter->stats.tsctfc += er32(TSCTFC); 5000 5001 /* Fill out the OS statistics structure */ 5002 netdev->stats.multicast = adapter->stats.mprc; 5003 netdev->stats.collisions = adapter->stats.colc; 5004 5005 /* Rx Errors */ 5006 5007 /* RLEC on some newer hardware can be incorrect so build 5008 * our own version based on RUC and ROC 5009 */ 5010 netdev->stats.rx_errors = adapter->stats.rxerrc + 5011 adapter->stats.crcerrs + adapter->stats.algnerrc + 5012 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr; 5013 netdev->stats.rx_length_errors = adapter->stats.ruc + 5014 adapter->stats.roc; 5015 netdev->stats.rx_crc_errors = adapter->stats.crcerrs; 5016 netdev->stats.rx_frame_errors = adapter->stats.algnerrc; 5017 netdev->stats.rx_missed_errors = adapter->stats.mpc; 5018 5019 /* Tx Errors */ 5020 netdev->stats.tx_errors = adapter->stats.ecol + adapter->stats.latecol; 5021 netdev->stats.tx_aborted_errors = adapter->stats.ecol; 5022 netdev->stats.tx_window_errors = adapter->stats.latecol; 5023 netdev->stats.tx_carrier_errors = adapter->stats.tncrs; 5024 5025 /* Tx Dropped needs to be maintained elsewhere */ 5026 5027 /* Management Stats */ 5028 adapter->stats.mgptc += er32(MGTPTC); 5029 adapter->stats.mgprc += er32(MGTPRC); 5030 adapter->stats.mgpdc += er32(MGTPDC); 5031 5032 /* Correctable ECC Errors */ 5033 if (hw->mac.type >= e1000_pch_lpt) { 5034 u32 pbeccsts = er32(PBECCSTS); 5035 5036 adapter->corr_errors += 5037 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 5038 adapter->uncorr_errors += 5039 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >> 5040 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT; 5041 } 5042 } 5043 5044 /** 5045 * e1000_phy_read_status - Update the PHY register status snapshot 5046 * @adapter: board private structure 5047 **/ 5048 static void e1000_phy_read_status(struct e1000_adapter *adapter) 5049 { 5050 struct e1000_hw *hw = &adapter->hw; 5051 struct e1000_phy_regs *phy = &adapter->phy_regs; 5052 5053 if (!pm_runtime_suspended((&adapter->pdev->dev)->parent) && 5054 (er32(STATUS) & E1000_STATUS_LU) && 5055 (adapter->hw.phy.media_type == e1000_media_type_copper)) { 5056 int ret_val; 5057 5058 ret_val = e1e_rphy(hw, MII_BMCR, &phy->bmcr); 5059 ret_val |= e1e_rphy(hw, MII_BMSR, &phy->bmsr); 5060 ret_val |= e1e_rphy(hw, MII_ADVERTISE, &phy->advertise); 5061 ret_val |= e1e_rphy(hw, MII_LPA, &phy->lpa); 5062 ret_val |= e1e_rphy(hw, MII_EXPANSION, &phy->expansion); 5063 ret_val |= e1e_rphy(hw, MII_CTRL1000, &phy->ctrl1000); 5064 ret_val |= e1e_rphy(hw, MII_STAT1000, &phy->stat1000); 5065 ret_val |= e1e_rphy(hw, MII_ESTATUS, &phy->estatus); 5066 if (ret_val) 5067 e_warn("Error reading PHY register\n"); 5068 } else { 5069 /* Do not read PHY registers if link is not up 5070 * Set values to typical power-on defaults 5071 */ 5072 phy->bmcr = (BMCR_SPEED1000 | BMCR_ANENABLE | BMCR_FULLDPLX); 5073 phy->bmsr = (BMSR_100FULL | BMSR_100HALF | BMSR_10FULL | 5074 BMSR_10HALF | BMSR_ESTATEN | BMSR_ANEGCAPABLE | 5075 BMSR_ERCAP); 5076 phy->advertise = (ADVERTISE_PAUSE_ASYM | ADVERTISE_PAUSE_CAP | 5077 ADVERTISE_ALL | ADVERTISE_CSMA); 5078 phy->lpa = 0; 5079 phy->expansion = EXPANSION_ENABLENPAGE; 5080 phy->ctrl1000 = ADVERTISE_1000FULL; 5081 phy->stat1000 = 0; 5082 phy->estatus = (ESTATUS_1000_TFULL | ESTATUS_1000_THALF); 5083 } 5084 } 5085 5086 static void e1000_print_link_info(struct e1000_adapter *adapter) 5087 { 5088 struct e1000_hw *hw = &adapter->hw; 5089 u32 ctrl = er32(CTRL); 5090 5091 /* Link status message must follow this format for user tools */ 5092 netdev_info(adapter->netdev, 5093 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 5094 adapter->link_speed, 5095 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half", 5096 (ctrl & E1000_CTRL_TFCE) && (ctrl & E1000_CTRL_RFCE) ? "Rx/Tx" : 5097 (ctrl & E1000_CTRL_RFCE) ? "Rx" : 5098 (ctrl & E1000_CTRL_TFCE) ? "Tx" : "None"); 5099 } 5100 5101 static bool e1000e_has_link(struct e1000_adapter *adapter) 5102 { 5103 struct e1000_hw *hw = &adapter->hw; 5104 bool link_active = false; 5105 s32 ret_val = 0; 5106 5107 /* get_link_status is set on LSC (link status) interrupt or 5108 * Rx sequence error interrupt. get_link_status will stay 5109 * true until the check_for_link establishes link 5110 * for copper adapters ONLY 5111 */ 5112 switch (hw->phy.media_type) { 5113 case e1000_media_type_copper: 5114 if (hw->mac.get_link_status) { 5115 ret_val = hw->mac.ops.check_for_link(hw); 5116 link_active = !hw->mac.get_link_status; 5117 } else { 5118 link_active = true; 5119 } 5120 break; 5121 case e1000_media_type_fiber: 5122 ret_val = hw->mac.ops.check_for_link(hw); 5123 link_active = !!(er32(STATUS) & E1000_STATUS_LU); 5124 break; 5125 case e1000_media_type_internal_serdes: 5126 ret_val = hw->mac.ops.check_for_link(hw); 5127 link_active = hw->mac.serdes_has_link; 5128 break; 5129 default: 5130 case e1000_media_type_unknown: 5131 break; 5132 } 5133 5134 if ((ret_val == -E1000_ERR_PHY) && (hw->phy.type == e1000_phy_igp_3) && 5135 (er32(CTRL) & E1000_PHY_CTRL_GBE_DISABLE)) { 5136 /* See e1000_kmrn_lock_loss_workaround_ich8lan() */ 5137 e_info("Gigabit has been disabled, downgrading speed\n"); 5138 } 5139 5140 return link_active; 5141 } 5142 5143 static void e1000e_enable_receives(struct e1000_adapter *adapter) 5144 { 5145 /* make sure the receive unit is started */ 5146 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) && 5147 (adapter->flags & FLAG_RESTART_NOW)) { 5148 struct e1000_hw *hw = &adapter->hw; 5149 u32 rctl = er32(RCTL); 5150 5151 ew32(RCTL, rctl | E1000_RCTL_EN); 5152 adapter->flags &= ~FLAG_RESTART_NOW; 5153 } 5154 } 5155 5156 static void e1000e_check_82574_phy_workaround(struct e1000_adapter *adapter) 5157 { 5158 struct e1000_hw *hw = &adapter->hw; 5159 5160 /* With 82574 controllers, PHY needs to be checked periodically 5161 * for hung state and reset, if two calls return true 5162 */ 5163 if (e1000_check_phy_82574(hw)) 5164 adapter->phy_hang_count++; 5165 else 5166 adapter->phy_hang_count = 0; 5167 5168 if (adapter->phy_hang_count > 1) { 5169 adapter->phy_hang_count = 0; 5170 e_dbg("PHY appears hung - resetting\n"); 5171 schedule_work(&adapter->reset_task); 5172 } 5173 } 5174 5175 /** 5176 * e1000_watchdog - Timer Call-back 5177 * @t: pointer to timer_list containing private info adapter 5178 **/ 5179 static void e1000_watchdog(struct timer_list *t) 5180 { 5181 struct e1000_adapter *adapter = from_timer(adapter, t, watchdog_timer); 5182 5183 /* Do the rest outside of interrupt context */ 5184 schedule_work(&adapter->watchdog_task); 5185 5186 /* TODO: make this use queue_delayed_work() */ 5187 } 5188 5189 static void e1000_watchdog_task(struct work_struct *work) 5190 { 5191 struct e1000_adapter *adapter = container_of(work, 5192 struct e1000_adapter, 5193 watchdog_task); 5194 struct net_device *netdev = adapter->netdev; 5195 struct e1000_mac_info *mac = &adapter->hw.mac; 5196 struct e1000_phy_info *phy = &adapter->hw.phy; 5197 struct e1000_ring *tx_ring = adapter->tx_ring; 5198 u32 dmoff_exit_timeout = 100, tries = 0; 5199 struct e1000_hw *hw = &adapter->hw; 5200 u32 link, tctl, pcim_state; 5201 5202 if (test_bit(__E1000_DOWN, &adapter->state)) 5203 return; 5204 5205 link = e1000e_has_link(adapter); 5206 if ((netif_carrier_ok(netdev)) && link) { 5207 /* Cancel scheduled suspend requests. */ 5208 pm_runtime_resume(netdev->dev.parent); 5209 5210 e1000e_enable_receives(adapter); 5211 goto link_up; 5212 } 5213 5214 if ((e1000e_enable_tx_pkt_filtering(hw)) && 5215 (adapter->mng_vlan_id != adapter->hw.mng_cookie.vlan_id)) 5216 e1000_update_mng_vlan(adapter); 5217 5218 if (link) { 5219 if (!netif_carrier_ok(netdev)) { 5220 bool txb2b = true; 5221 5222 /* Cancel scheduled suspend requests. */ 5223 pm_runtime_resume(netdev->dev.parent); 5224 5225 /* Checking if MAC is in DMoff state*/ 5226 pcim_state = er32(STATUS); 5227 while (pcim_state & E1000_STATUS_PCIM_STATE) { 5228 if (tries++ == dmoff_exit_timeout) { 5229 e_dbg("Error in exiting dmoff\n"); 5230 break; 5231 } 5232 usleep_range(10000, 20000); 5233 pcim_state = er32(STATUS); 5234 5235 /* Checking if MAC exited DMoff state */ 5236 if (!(pcim_state & E1000_STATUS_PCIM_STATE)) 5237 e1000_phy_hw_reset(&adapter->hw); 5238 } 5239 5240 /* update snapshot of PHY registers on LSC */ 5241 e1000_phy_read_status(adapter); 5242 mac->ops.get_link_up_info(&adapter->hw, 5243 &adapter->link_speed, 5244 &adapter->link_duplex); 5245 e1000_print_link_info(adapter); 5246 5247 /* check if SmartSpeed worked */ 5248 e1000e_check_downshift(hw); 5249 if (phy->speed_downgraded) 5250 netdev_warn(netdev, 5251 "Link Speed was downgraded by SmartSpeed\n"); 5252 5253 /* On supported PHYs, check for duplex mismatch only 5254 * if link has autonegotiated at 10/100 half 5255 */ 5256 if ((hw->phy.type == e1000_phy_igp_3 || 5257 hw->phy.type == e1000_phy_bm) && 5258 hw->mac.autoneg && 5259 (adapter->link_speed == SPEED_10 || 5260 adapter->link_speed == SPEED_100) && 5261 (adapter->link_duplex == HALF_DUPLEX)) { 5262 u16 autoneg_exp; 5263 5264 e1e_rphy(hw, MII_EXPANSION, &autoneg_exp); 5265 5266 if (!(autoneg_exp & EXPANSION_NWAY)) 5267 e_info("Autonegotiated half duplex but link partner cannot autoneg. Try forcing full duplex if link gets many collisions.\n"); 5268 } 5269 5270 /* adjust timeout factor according to speed/duplex */ 5271 adapter->tx_timeout_factor = 1; 5272 switch (adapter->link_speed) { 5273 case SPEED_10: 5274 txb2b = false; 5275 adapter->tx_timeout_factor = 16; 5276 break; 5277 case SPEED_100: 5278 txb2b = false; 5279 adapter->tx_timeout_factor = 10; 5280 break; 5281 } 5282 5283 /* workaround: re-program speed mode bit after 5284 * link-up event 5285 */ 5286 if ((adapter->flags & FLAG_TARC_SPEED_MODE_BIT) && 5287 !txb2b) { 5288 u32 tarc0; 5289 5290 tarc0 = er32(TARC(0)); 5291 tarc0 &= ~SPEED_MODE_BIT; 5292 ew32(TARC(0), tarc0); 5293 } 5294 5295 /* disable TSO for pcie and 10/100 speeds, to avoid 5296 * some hardware issues 5297 */ 5298 if (!(adapter->flags & FLAG_TSO_FORCE)) { 5299 switch (adapter->link_speed) { 5300 case SPEED_10: 5301 case SPEED_100: 5302 e_info("10/100 speed: disabling TSO\n"); 5303 netdev->features &= ~NETIF_F_TSO; 5304 netdev->features &= ~NETIF_F_TSO6; 5305 break; 5306 case SPEED_1000: 5307 netdev->features |= NETIF_F_TSO; 5308 netdev->features |= NETIF_F_TSO6; 5309 break; 5310 default: 5311 /* oops */ 5312 break; 5313 } 5314 if (hw->mac.type == e1000_pch_spt) { 5315 netdev->features &= ~NETIF_F_TSO; 5316 netdev->features &= ~NETIF_F_TSO6; 5317 } 5318 } 5319 5320 /* enable transmits in the hardware, need to do this 5321 * after setting TARC(0) 5322 */ 5323 tctl = er32(TCTL); 5324 tctl |= E1000_TCTL_EN; 5325 ew32(TCTL, tctl); 5326 5327 /* Perform any post-link-up configuration before 5328 * reporting link up. 5329 */ 5330 if (phy->ops.cfg_on_link_up) 5331 phy->ops.cfg_on_link_up(hw); 5332 5333 netif_wake_queue(netdev); 5334 netif_carrier_on(netdev); 5335 5336 if (!test_bit(__E1000_DOWN, &adapter->state)) 5337 mod_timer(&adapter->phy_info_timer, 5338 round_jiffies(jiffies + 2 * HZ)); 5339 } 5340 } else { 5341 if (netif_carrier_ok(netdev)) { 5342 adapter->link_speed = 0; 5343 adapter->link_duplex = 0; 5344 /* Link status message must follow this format */ 5345 netdev_info(netdev, "NIC Link is Down\n"); 5346 netif_carrier_off(netdev); 5347 netif_stop_queue(netdev); 5348 if (!test_bit(__E1000_DOWN, &adapter->state)) 5349 mod_timer(&adapter->phy_info_timer, 5350 round_jiffies(jiffies + 2 * HZ)); 5351 5352 /* 8000ES2LAN requires a Rx packet buffer work-around 5353 * on link down event; reset the controller to flush 5354 * the Rx packet buffer. 5355 */ 5356 if (adapter->flags & FLAG_RX_NEEDS_RESTART) 5357 adapter->flags |= FLAG_RESTART_NOW; 5358 else 5359 pm_schedule_suspend(netdev->dev.parent, 5360 LINK_TIMEOUT); 5361 } 5362 } 5363 5364 link_up: 5365 spin_lock(&adapter->stats64_lock); 5366 e1000e_update_stats(adapter); 5367 5368 mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old; 5369 adapter->tpt_old = adapter->stats.tpt; 5370 mac->collision_delta = adapter->stats.colc - adapter->colc_old; 5371 adapter->colc_old = adapter->stats.colc; 5372 5373 adapter->gorc = adapter->stats.gorc - adapter->gorc_old; 5374 adapter->gorc_old = adapter->stats.gorc; 5375 adapter->gotc = adapter->stats.gotc - adapter->gotc_old; 5376 adapter->gotc_old = adapter->stats.gotc; 5377 spin_unlock(&adapter->stats64_lock); 5378 5379 /* If the link is lost the controller stops DMA, but 5380 * if there is queued Tx work it cannot be done. So 5381 * reset the controller to flush the Tx packet buffers. 5382 */ 5383 if (!netif_carrier_ok(netdev) && 5384 (e1000_desc_unused(tx_ring) + 1 < tx_ring->count)) 5385 adapter->flags |= FLAG_RESTART_NOW; 5386 5387 /* If reset is necessary, do it outside of interrupt context. */ 5388 if (adapter->flags & FLAG_RESTART_NOW) { 5389 schedule_work(&adapter->reset_task); 5390 /* return immediately since reset is imminent */ 5391 return; 5392 } 5393 5394 e1000e_update_adaptive(&adapter->hw); 5395 5396 /* Simple mode for Interrupt Throttle Rate (ITR) */ 5397 if (adapter->itr_setting == 4) { 5398 /* Symmetric Tx/Rx gets a reduced ITR=2000; 5399 * Total asymmetrical Tx or Rx gets ITR=8000; 5400 * everyone else is between 2000-8000. 5401 */ 5402 u32 goc = (adapter->gotc + adapter->gorc) / 10000; 5403 u32 dif = (adapter->gotc > adapter->gorc ? 5404 adapter->gotc - adapter->gorc : 5405 adapter->gorc - adapter->gotc) / 10000; 5406 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000; 5407 5408 e1000e_write_itr(adapter, itr); 5409 } 5410 5411 /* Cause software interrupt to ensure Rx ring is cleaned */ 5412 if (adapter->msix_entries) 5413 ew32(ICS, adapter->rx_ring->ims_val); 5414 else 5415 ew32(ICS, E1000_ICS_RXDMT0); 5416 5417 /* flush pending descriptors to memory before detecting Tx hang */ 5418 e1000e_flush_descriptors(adapter); 5419 5420 /* Force detection of hung controller every watchdog period */ 5421 adapter->detect_tx_hung = true; 5422 5423 /* With 82571 controllers, LAA may be overwritten due to controller 5424 * reset from the other port. Set the appropriate LAA in RAR[0] 5425 */ 5426 if (e1000e_get_laa_state_82571(hw)) 5427 hw->mac.ops.rar_set(hw, adapter->hw.mac.addr, 0); 5428 5429 if (adapter->flags2 & FLAG2_CHECK_PHY_HANG) 5430 e1000e_check_82574_phy_workaround(adapter); 5431 5432 /* Clear valid timestamp stuck in RXSTMPL/H due to a Rx error */ 5433 if (adapter->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE) { 5434 if ((adapter->flags2 & FLAG2_CHECK_RX_HWTSTAMP) && 5435 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) { 5436 er32(RXSTMPH); 5437 adapter->rx_hwtstamp_cleared++; 5438 } else { 5439 adapter->flags2 |= FLAG2_CHECK_RX_HWTSTAMP; 5440 } 5441 } 5442 5443 /* Reset the timer */ 5444 if (!test_bit(__E1000_DOWN, &adapter->state)) 5445 mod_timer(&adapter->watchdog_timer, 5446 round_jiffies(jiffies + 2 * HZ)); 5447 } 5448 5449 #define E1000_TX_FLAGS_CSUM 0x00000001 5450 #define E1000_TX_FLAGS_VLAN 0x00000002 5451 #define E1000_TX_FLAGS_TSO 0x00000004 5452 #define E1000_TX_FLAGS_IPV4 0x00000008 5453 #define E1000_TX_FLAGS_NO_FCS 0x00000010 5454 #define E1000_TX_FLAGS_HWTSTAMP 0x00000020 5455 #define E1000_TX_FLAGS_VLAN_MASK 0xffff0000 5456 #define E1000_TX_FLAGS_VLAN_SHIFT 16 5457 5458 static int e1000_tso(struct e1000_ring *tx_ring, struct sk_buff *skb, 5459 __be16 protocol) 5460 { 5461 struct e1000_context_desc *context_desc; 5462 struct e1000_buffer *buffer_info; 5463 unsigned int i; 5464 u32 cmd_length = 0; 5465 u16 ipcse = 0, mss; 5466 u8 ipcss, ipcso, tucss, tucso, hdr_len; 5467 int err; 5468 5469 if (!skb_is_gso(skb)) 5470 return 0; 5471 5472 err = skb_cow_head(skb, 0); 5473 if (err < 0) 5474 return err; 5475 5476 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 5477 mss = skb_shinfo(skb)->gso_size; 5478 if (protocol == htons(ETH_P_IP)) { 5479 struct iphdr *iph = ip_hdr(skb); 5480 iph->tot_len = 0; 5481 iph->check = 0; 5482 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 5483 0, IPPROTO_TCP, 0); 5484 cmd_length = E1000_TXD_CMD_IP; 5485 ipcse = skb_transport_offset(skb) - 1; 5486 } else if (skb_is_gso_v6(skb)) { 5487 tcp_v6_gso_csum_prep(skb); 5488 ipcse = 0; 5489 } 5490 ipcss = skb_network_offset(skb); 5491 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data; 5492 tucss = skb_transport_offset(skb); 5493 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data; 5494 5495 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE | 5496 E1000_TXD_CMD_TCP | (skb->len - (hdr_len))); 5497 5498 i = tx_ring->next_to_use; 5499 context_desc = E1000_CONTEXT_DESC(*tx_ring, i); 5500 buffer_info = &tx_ring->buffer_info[i]; 5501 5502 context_desc->lower_setup.ip_fields.ipcss = ipcss; 5503 context_desc->lower_setup.ip_fields.ipcso = ipcso; 5504 context_desc->lower_setup.ip_fields.ipcse = cpu_to_le16(ipcse); 5505 context_desc->upper_setup.tcp_fields.tucss = tucss; 5506 context_desc->upper_setup.tcp_fields.tucso = tucso; 5507 context_desc->upper_setup.tcp_fields.tucse = 0; 5508 context_desc->tcp_seg_setup.fields.mss = cpu_to_le16(mss); 5509 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len; 5510 context_desc->cmd_and_length = cpu_to_le32(cmd_length); 5511 5512 buffer_info->time_stamp = jiffies; 5513 buffer_info->next_to_watch = i; 5514 5515 i++; 5516 if (i == tx_ring->count) 5517 i = 0; 5518 tx_ring->next_to_use = i; 5519 5520 return 1; 5521 } 5522 5523 static bool e1000_tx_csum(struct e1000_ring *tx_ring, struct sk_buff *skb, 5524 __be16 protocol) 5525 { 5526 struct e1000_adapter *adapter = tx_ring->adapter; 5527 struct e1000_context_desc *context_desc; 5528 struct e1000_buffer *buffer_info; 5529 unsigned int i; 5530 u8 css; 5531 u32 cmd_len = E1000_TXD_CMD_DEXT; 5532 5533 if (skb->ip_summed != CHECKSUM_PARTIAL) 5534 return false; 5535 5536 switch (protocol) { 5537 case cpu_to_be16(ETH_P_IP): 5538 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 5539 cmd_len |= E1000_TXD_CMD_TCP; 5540 break; 5541 case cpu_to_be16(ETH_P_IPV6): 5542 /* XXX not handling all IPV6 headers */ 5543 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 5544 cmd_len |= E1000_TXD_CMD_TCP; 5545 break; 5546 default: 5547 if (unlikely(net_ratelimit())) 5548 e_warn("checksum_partial proto=%x!\n", 5549 be16_to_cpu(protocol)); 5550 break; 5551 } 5552 5553 css = skb_checksum_start_offset(skb); 5554 5555 i = tx_ring->next_to_use; 5556 buffer_info = &tx_ring->buffer_info[i]; 5557 context_desc = E1000_CONTEXT_DESC(*tx_ring, i); 5558 5559 context_desc->lower_setup.ip_config = 0; 5560 context_desc->upper_setup.tcp_fields.tucss = css; 5561 context_desc->upper_setup.tcp_fields.tucso = css + skb->csum_offset; 5562 context_desc->upper_setup.tcp_fields.tucse = 0; 5563 context_desc->tcp_seg_setup.data = 0; 5564 context_desc->cmd_and_length = cpu_to_le32(cmd_len); 5565 5566 buffer_info->time_stamp = jiffies; 5567 buffer_info->next_to_watch = i; 5568 5569 i++; 5570 if (i == tx_ring->count) 5571 i = 0; 5572 tx_ring->next_to_use = i; 5573 5574 return true; 5575 } 5576 5577 static int e1000_tx_map(struct e1000_ring *tx_ring, struct sk_buff *skb, 5578 unsigned int first, unsigned int max_per_txd, 5579 unsigned int nr_frags) 5580 { 5581 struct e1000_adapter *adapter = tx_ring->adapter; 5582 struct pci_dev *pdev = adapter->pdev; 5583 struct e1000_buffer *buffer_info; 5584 unsigned int len = skb_headlen(skb); 5585 unsigned int offset = 0, size, count = 0, i; 5586 unsigned int f, bytecount, segs; 5587 5588 i = tx_ring->next_to_use; 5589 5590 while (len) { 5591 buffer_info = &tx_ring->buffer_info[i]; 5592 size = min(len, max_per_txd); 5593 5594 buffer_info->length = size; 5595 buffer_info->time_stamp = jiffies; 5596 buffer_info->next_to_watch = i; 5597 buffer_info->dma = dma_map_single(&pdev->dev, 5598 skb->data + offset, 5599 size, DMA_TO_DEVICE); 5600 buffer_info->mapped_as_page = false; 5601 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) 5602 goto dma_error; 5603 5604 len -= size; 5605 offset += size; 5606 count++; 5607 5608 if (len) { 5609 i++; 5610 if (i == tx_ring->count) 5611 i = 0; 5612 } 5613 } 5614 5615 for (f = 0; f < nr_frags; f++) { 5616 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 5617 5618 len = skb_frag_size(frag); 5619 offset = 0; 5620 5621 while (len) { 5622 i++; 5623 if (i == tx_ring->count) 5624 i = 0; 5625 5626 buffer_info = &tx_ring->buffer_info[i]; 5627 size = min(len, max_per_txd); 5628 5629 buffer_info->length = size; 5630 buffer_info->time_stamp = jiffies; 5631 buffer_info->next_to_watch = i; 5632 buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag, 5633 offset, size, 5634 DMA_TO_DEVICE); 5635 buffer_info->mapped_as_page = true; 5636 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) 5637 goto dma_error; 5638 5639 len -= size; 5640 offset += size; 5641 count++; 5642 } 5643 } 5644 5645 segs = skb_shinfo(skb)->gso_segs ? : 1; 5646 /* multiply data chunks by size of headers */ 5647 bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len; 5648 5649 tx_ring->buffer_info[i].skb = skb; 5650 tx_ring->buffer_info[i].segs = segs; 5651 tx_ring->buffer_info[i].bytecount = bytecount; 5652 tx_ring->buffer_info[first].next_to_watch = i; 5653 5654 return count; 5655 5656 dma_error: 5657 dev_err(&pdev->dev, "Tx DMA map failed\n"); 5658 buffer_info->dma = 0; 5659 if (count) 5660 count--; 5661 5662 while (count--) { 5663 if (i == 0) 5664 i += tx_ring->count; 5665 i--; 5666 buffer_info = &tx_ring->buffer_info[i]; 5667 e1000_put_txbuf(tx_ring, buffer_info, true); 5668 } 5669 5670 return 0; 5671 } 5672 5673 static void e1000_tx_queue(struct e1000_ring *tx_ring, int tx_flags, int count) 5674 { 5675 struct e1000_adapter *adapter = tx_ring->adapter; 5676 struct e1000_tx_desc *tx_desc = NULL; 5677 struct e1000_buffer *buffer_info; 5678 u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS; 5679 unsigned int i; 5680 5681 if (tx_flags & E1000_TX_FLAGS_TSO) { 5682 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D | 5683 E1000_TXD_CMD_TSE; 5684 txd_upper |= E1000_TXD_POPTS_TXSM << 8; 5685 5686 if (tx_flags & E1000_TX_FLAGS_IPV4) 5687 txd_upper |= E1000_TXD_POPTS_IXSM << 8; 5688 } 5689 5690 if (tx_flags & E1000_TX_FLAGS_CSUM) { 5691 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 5692 txd_upper |= E1000_TXD_POPTS_TXSM << 8; 5693 } 5694 5695 if (tx_flags & E1000_TX_FLAGS_VLAN) { 5696 txd_lower |= E1000_TXD_CMD_VLE; 5697 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK); 5698 } 5699 5700 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS)) 5701 txd_lower &= ~(E1000_TXD_CMD_IFCS); 5702 5703 if (unlikely(tx_flags & E1000_TX_FLAGS_HWTSTAMP)) { 5704 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 5705 txd_upper |= E1000_TXD_EXTCMD_TSTAMP; 5706 } 5707 5708 i = tx_ring->next_to_use; 5709 5710 do { 5711 buffer_info = &tx_ring->buffer_info[i]; 5712 tx_desc = E1000_TX_DESC(*tx_ring, i); 5713 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma); 5714 tx_desc->lower.data = cpu_to_le32(txd_lower | 5715 buffer_info->length); 5716 tx_desc->upper.data = cpu_to_le32(txd_upper); 5717 5718 i++; 5719 if (i == tx_ring->count) 5720 i = 0; 5721 } while (--count > 0); 5722 5723 tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd); 5724 5725 /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */ 5726 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS)) 5727 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS)); 5728 5729 /* Force memory writes to complete before letting h/w 5730 * know there are new descriptors to fetch. (Only 5731 * applicable for weak-ordered memory model archs, 5732 * such as IA-64). 5733 */ 5734 wmb(); 5735 5736 tx_ring->next_to_use = i; 5737 } 5738 5739 #define MINIMUM_DHCP_PACKET_SIZE 282 5740 static int e1000_transfer_dhcp_info(struct e1000_adapter *adapter, 5741 struct sk_buff *skb) 5742 { 5743 struct e1000_hw *hw = &adapter->hw; 5744 u16 length, offset; 5745 5746 if (skb_vlan_tag_present(skb) && 5747 !((skb_vlan_tag_get(skb) == adapter->hw.mng_cookie.vlan_id) && 5748 (adapter->hw.mng_cookie.status & 5749 E1000_MNG_DHCP_COOKIE_STATUS_VLAN))) 5750 return 0; 5751 5752 if (skb->len <= MINIMUM_DHCP_PACKET_SIZE) 5753 return 0; 5754 5755 if (((struct ethhdr *)skb->data)->h_proto != htons(ETH_P_IP)) 5756 return 0; 5757 5758 { 5759 const struct iphdr *ip = (struct iphdr *)((u8 *)skb->data + 14); 5760 struct udphdr *udp; 5761 5762 if (ip->protocol != IPPROTO_UDP) 5763 return 0; 5764 5765 udp = (struct udphdr *)((u8 *)ip + (ip->ihl << 2)); 5766 if (ntohs(udp->dest) != 67) 5767 return 0; 5768 5769 offset = (u8 *)udp + 8 - skb->data; 5770 length = skb->len - offset; 5771 return e1000e_mng_write_dhcp_info(hw, (u8 *)udp + 8, length); 5772 } 5773 5774 return 0; 5775 } 5776 5777 static int __e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size) 5778 { 5779 struct e1000_adapter *adapter = tx_ring->adapter; 5780 5781 netif_stop_queue(adapter->netdev); 5782 /* Herbert's original patch had: 5783 * smp_mb__after_netif_stop_queue(); 5784 * but since that doesn't exist yet, just open code it. 5785 */ 5786 smp_mb(); 5787 5788 /* We need to check again in a case another CPU has just 5789 * made room available. 5790 */ 5791 if (e1000_desc_unused(tx_ring) < size) 5792 return -EBUSY; 5793 5794 /* A reprieve! */ 5795 netif_start_queue(adapter->netdev); 5796 ++adapter->restart_queue; 5797 return 0; 5798 } 5799 5800 static int e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size) 5801 { 5802 BUG_ON(size > tx_ring->count); 5803 5804 if (e1000_desc_unused(tx_ring) >= size) 5805 return 0; 5806 return __e1000_maybe_stop_tx(tx_ring, size); 5807 } 5808 5809 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, 5810 struct net_device *netdev) 5811 { 5812 struct e1000_adapter *adapter = netdev_priv(netdev); 5813 struct e1000_ring *tx_ring = adapter->tx_ring; 5814 unsigned int first; 5815 unsigned int tx_flags = 0; 5816 unsigned int len = skb_headlen(skb); 5817 unsigned int nr_frags; 5818 unsigned int mss; 5819 int count = 0; 5820 int tso; 5821 unsigned int f; 5822 __be16 protocol = vlan_get_protocol(skb); 5823 5824 if (test_bit(__E1000_DOWN, &adapter->state)) { 5825 dev_kfree_skb_any(skb); 5826 return NETDEV_TX_OK; 5827 } 5828 5829 if (skb->len <= 0) { 5830 dev_kfree_skb_any(skb); 5831 return NETDEV_TX_OK; 5832 } 5833 5834 /* The minimum packet size with TCTL.PSP set is 17 bytes so 5835 * pad skb in order to meet this minimum size requirement 5836 */ 5837 if (skb_put_padto(skb, 17)) 5838 return NETDEV_TX_OK; 5839 5840 mss = skb_shinfo(skb)->gso_size; 5841 if (mss) { 5842 u8 hdr_len; 5843 5844 /* TSO Workaround for 82571/2/3 Controllers -- if skb->data 5845 * points to just header, pull a few bytes of payload from 5846 * frags into skb->data 5847 */ 5848 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 5849 /* we do this workaround for ES2LAN, but it is un-necessary, 5850 * avoiding it could save a lot of cycles 5851 */ 5852 if (skb->data_len && (hdr_len == len)) { 5853 unsigned int pull_size; 5854 5855 pull_size = min_t(unsigned int, 4, skb->data_len); 5856 if (!__pskb_pull_tail(skb, pull_size)) { 5857 e_err("__pskb_pull_tail failed.\n"); 5858 dev_kfree_skb_any(skb); 5859 return NETDEV_TX_OK; 5860 } 5861 len = skb_headlen(skb); 5862 } 5863 } 5864 5865 /* reserve a descriptor for the offload context */ 5866 if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL)) 5867 count++; 5868 count++; 5869 5870 count += DIV_ROUND_UP(len, adapter->tx_fifo_limit); 5871 5872 nr_frags = skb_shinfo(skb)->nr_frags; 5873 for (f = 0; f < nr_frags; f++) 5874 count += DIV_ROUND_UP(skb_frag_size(&skb_shinfo(skb)->frags[f]), 5875 adapter->tx_fifo_limit); 5876 5877 if (adapter->hw.mac.tx_pkt_filtering) 5878 e1000_transfer_dhcp_info(adapter, skb); 5879 5880 /* need: count + 2 desc gap to keep tail from touching 5881 * head, otherwise try next time 5882 */ 5883 if (e1000_maybe_stop_tx(tx_ring, count + 2)) 5884 return NETDEV_TX_BUSY; 5885 5886 if (skb_vlan_tag_present(skb)) { 5887 tx_flags |= E1000_TX_FLAGS_VLAN; 5888 tx_flags |= (skb_vlan_tag_get(skb) << 5889 E1000_TX_FLAGS_VLAN_SHIFT); 5890 } 5891 5892 first = tx_ring->next_to_use; 5893 5894 tso = e1000_tso(tx_ring, skb, protocol); 5895 if (tso < 0) { 5896 dev_kfree_skb_any(skb); 5897 return NETDEV_TX_OK; 5898 } 5899 5900 if (tso) 5901 tx_flags |= E1000_TX_FLAGS_TSO; 5902 else if (e1000_tx_csum(tx_ring, skb, protocol)) 5903 tx_flags |= E1000_TX_FLAGS_CSUM; 5904 5905 /* Old method was to assume IPv4 packet by default if TSO was enabled. 5906 * 82571 hardware supports TSO capabilities for IPv6 as well... 5907 * no longer assume, we must. 5908 */ 5909 if (protocol == htons(ETH_P_IP)) 5910 tx_flags |= E1000_TX_FLAGS_IPV4; 5911 5912 if (unlikely(skb->no_fcs)) 5913 tx_flags |= E1000_TX_FLAGS_NO_FCS; 5914 5915 /* if count is 0 then mapping error has occurred */ 5916 count = e1000_tx_map(tx_ring, skb, first, adapter->tx_fifo_limit, 5917 nr_frags); 5918 if (count) { 5919 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 5920 (adapter->flags & FLAG_HAS_HW_TIMESTAMP)) { 5921 if (!adapter->tx_hwtstamp_skb) { 5922 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 5923 tx_flags |= E1000_TX_FLAGS_HWTSTAMP; 5924 adapter->tx_hwtstamp_skb = skb_get(skb); 5925 adapter->tx_hwtstamp_start = jiffies; 5926 schedule_work(&adapter->tx_hwtstamp_work); 5927 } else { 5928 adapter->tx_hwtstamp_skipped++; 5929 } 5930 } 5931 5932 skb_tx_timestamp(skb); 5933 5934 netdev_sent_queue(netdev, skb->len); 5935 e1000_tx_queue(tx_ring, tx_flags, count); 5936 /* Make sure there is space in the ring for the next send. */ 5937 e1000_maybe_stop_tx(tx_ring, 5938 (MAX_SKB_FRAGS * 5939 DIV_ROUND_UP(PAGE_SIZE, 5940 adapter->tx_fifo_limit) + 2)); 5941 5942 if (!netdev_xmit_more() || 5943 netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) { 5944 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 5945 e1000e_update_tdt_wa(tx_ring, 5946 tx_ring->next_to_use); 5947 else 5948 writel(tx_ring->next_to_use, tx_ring->tail); 5949 } 5950 } else { 5951 dev_kfree_skb_any(skb); 5952 tx_ring->buffer_info[first].time_stamp = 0; 5953 tx_ring->next_to_use = first; 5954 } 5955 5956 return NETDEV_TX_OK; 5957 } 5958 5959 /** 5960 * e1000_tx_timeout - Respond to a Tx Hang 5961 * @netdev: network interface device structure 5962 * @txqueue: index of the hung queue (unused) 5963 **/ 5964 static void e1000_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue) 5965 { 5966 struct e1000_adapter *adapter = netdev_priv(netdev); 5967 5968 /* Do the reset outside of interrupt context */ 5969 adapter->tx_timeout_count++; 5970 schedule_work(&adapter->reset_task); 5971 } 5972 5973 static void e1000_reset_task(struct work_struct *work) 5974 { 5975 struct e1000_adapter *adapter; 5976 adapter = container_of(work, struct e1000_adapter, reset_task); 5977 5978 rtnl_lock(); 5979 /* don't run the task if already down */ 5980 if (test_bit(__E1000_DOWN, &adapter->state)) { 5981 rtnl_unlock(); 5982 return; 5983 } 5984 5985 if (!(adapter->flags & FLAG_RESTART_NOW)) { 5986 e1000e_dump(adapter); 5987 e_err("Reset adapter unexpectedly\n"); 5988 } 5989 e1000e_reinit_locked(adapter); 5990 rtnl_unlock(); 5991 } 5992 5993 /** 5994 * e1000e_get_stats64 - Get System Network Statistics 5995 * @netdev: network interface device structure 5996 * @stats: rtnl_link_stats64 pointer 5997 * 5998 * Returns the address of the device statistics structure. 5999 **/ 6000 void e1000e_get_stats64(struct net_device *netdev, 6001 struct rtnl_link_stats64 *stats) 6002 { 6003 struct e1000_adapter *adapter = netdev_priv(netdev); 6004 6005 spin_lock(&adapter->stats64_lock); 6006 e1000e_update_stats(adapter); 6007 /* Fill out the OS statistics structure */ 6008 stats->rx_bytes = adapter->stats.gorc; 6009 stats->rx_packets = adapter->stats.gprc; 6010 stats->tx_bytes = adapter->stats.gotc; 6011 stats->tx_packets = adapter->stats.gptc; 6012 stats->multicast = adapter->stats.mprc; 6013 stats->collisions = adapter->stats.colc; 6014 6015 /* Rx Errors */ 6016 6017 /* RLEC on some newer hardware can be incorrect so build 6018 * our own version based on RUC and ROC 6019 */ 6020 stats->rx_errors = adapter->stats.rxerrc + 6021 adapter->stats.crcerrs + adapter->stats.algnerrc + 6022 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr; 6023 stats->rx_length_errors = adapter->stats.ruc + adapter->stats.roc; 6024 stats->rx_crc_errors = adapter->stats.crcerrs; 6025 stats->rx_frame_errors = adapter->stats.algnerrc; 6026 stats->rx_missed_errors = adapter->stats.mpc; 6027 6028 /* Tx Errors */ 6029 stats->tx_errors = adapter->stats.ecol + adapter->stats.latecol; 6030 stats->tx_aborted_errors = adapter->stats.ecol; 6031 stats->tx_window_errors = adapter->stats.latecol; 6032 stats->tx_carrier_errors = adapter->stats.tncrs; 6033 6034 /* Tx Dropped needs to be maintained elsewhere */ 6035 6036 spin_unlock(&adapter->stats64_lock); 6037 } 6038 6039 /** 6040 * e1000_change_mtu - Change the Maximum Transfer Unit 6041 * @netdev: network interface device structure 6042 * @new_mtu: new value for maximum frame size 6043 * 6044 * Returns 0 on success, negative on failure 6045 **/ 6046 static int e1000_change_mtu(struct net_device *netdev, int new_mtu) 6047 { 6048 struct e1000_adapter *adapter = netdev_priv(netdev); 6049 int max_frame = new_mtu + VLAN_ETH_HLEN + ETH_FCS_LEN; 6050 6051 /* Jumbo frame support */ 6052 if ((new_mtu > ETH_DATA_LEN) && 6053 !(adapter->flags & FLAG_HAS_JUMBO_FRAMES)) { 6054 e_err("Jumbo Frames not supported.\n"); 6055 return -EINVAL; 6056 } 6057 6058 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */ 6059 if ((adapter->hw.mac.type >= e1000_pch2lan) && 6060 !(adapter->flags2 & FLAG2_CRC_STRIPPING) && 6061 (new_mtu > ETH_DATA_LEN)) { 6062 e_err("Jumbo Frames not supported on this device when CRC stripping is disabled.\n"); 6063 return -EINVAL; 6064 } 6065 6066 while (test_and_set_bit(__E1000_RESETTING, &adapter->state)) 6067 usleep_range(1000, 1100); 6068 /* e1000e_down -> e1000e_reset dependent on max_frame_size & mtu */ 6069 adapter->max_frame_size = max_frame; 6070 netdev_dbg(netdev, "changing MTU from %d to %d\n", 6071 netdev->mtu, new_mtu); 6072 netdev->mtu = new_mtu; 6073 6074 pm_runtime_get_sync(netdev->dev.parent); 6075 6076 if (netif_running(netdev)) 6077 e1000e_down(adapter, true); 6078 6079 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN 6080 * means we reserve 2 more, this pushes us to allocate from the next 6081 * larger slab size. 6082 * i.e. RXBUFFER_2048 --> size-4096 slab 6083 * However with the new *_jumbo_rx* routines, jumbo receives will use 6084 * fragmented skbs 6085 */ 6086 6087 if (max_frame <= 2048) 6088 adapter->rx_buffer_len = 2048; 6089 else 6090 adapter->rx_buffer_len = 4096; 6091 6092 /* adjust allocation if LPE protects us, and we aren't using SBP */ 6093 if (max_frame <= (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)) 6094 adapter->rx_buffer_len = VLAN_ETH_FRAME_LEN + ETH_FCS_LEN; 6095 6096 if (netif_running(netdev)) 6097 e1000e_up(adapter); 6098 else 6099 e1000e_reset(adapter); 6100 6101 pm_runtime_put_sync(netdev->dev.parent); 6102 6103 clear_bit(__E1000_RESETTING, &adapter->state); 6104 6105 return 0; 6106 } 6107 6108 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, 6109 int cmd) 6110 { 6111 struct e1000_adapter *adapter = netdev_priv(netdev); 6112 struct mii_ioctl_data *data = if_mii(ifr); 6113 6114 if (adapter->hw.phy.media_type != e1000_media_type_copper) 6115 return -EOPNOTSUPP; 6116 6117 switch (cmd) { 6118 case SIOCGMIIPHY: 6119 data->phy_id = adapter->hw.phy.addr; 6120 break; 6121 case SIOCGMIIREG: 6122 e1000_phy_read_status(adapter); 6123 6124 switch (data->reg_num & 0x1F) { 6125 case MII_BMCR: 6126 data->val_out = adapter->phy_regs.bmcr; 6127 break; 6128 case MII_BMSR: 6129 data->val_out = adapter->phy_regs.bmsr; 6130 break; 6131 case MII_PHYSID1: 6132 data->val_out = (adapter->hw.phy.id >> 16); 6133 break; 6134 case MII_PHYSID2: 6135 data->val_out = (adapter->hw.phy.id & 0xFFFF); 6136 break; 6137 case MII_ADVERTISE: 6138 data->val_out = adapter->phy_regs.advertise; 6139 break; 6140 case MII_LPA: 6141 data->val_out = adapter->phy_regs.lpa; 6142 break; 6143 case MII_EXPANSION: 6144 data->val_out = adapter->phy_regs.expansion; 6145 break; 6146 case MII_CTRL1000: 6147 data->val_out = adapter->phy_regs.ctrl1000; 6148 break; 6149 case MII_STAT1000: 6150 data->val_out = adapter->phy_regs.stat1000; 6151 break; 6152 case MII_ESTATUS: 6153 data->val_out = adapter->phy_regs.estatus; 6154 break; 6155 default: 6156 return -EIO; 6157 } 6158 break; 6159 case SIOCSMIIREG: 6160 default: 6161 return -EOPNOTSUPP; 6162 } 6163 return 0; 6164 } 6165 6166 /** 6167 * e1000e_hwtstamp_set - control hardware time stamping 6168 * @netdev: network interface device structure 6169 * @ifr: interface request 6170 * 6171 * Outgoing time stamping can be enabled and disabled. Play nice and 6172 * disable it when requested, although it shouldn't cause any overhead 6173 * when no packet needs it. At most one packet in the queue may be 6174 * marked for time stamping, otherwise it would be impossible to tell 6175 * for sure to which packet the hardware time stamp belongs. 6176 * 6177 * Incoming time stamping has to be configured via the hardware filters. 6178 * Not all combinations are supported, in particular event type has to be 6179 * specified. Matching the kind of event packet is not supported, with the 6180 * exception of "all V2 events regardless of level 2 or 4". 6181 **/ 6182 static int e1000e_hwtstamp_set(struct net_device *netdev, struct ifreq *ifr) 6183 { 6184 struct e1000_adapter *adapter = netdev_priv(netdev); 6185 struct hwtstamp_config config; 6186 int ret_val; 6187 6188 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 6189 return -EFAULT; 6190 6191 ret_val = e1000e_config_hwtstamp(adapter, &config); 6192 if (ret_val) 6193 return ret_val; 6194 6195 switch (config.rx_filter) { 6196 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 6197 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 6198 case HWTSTAMP_FILTER_PTP_V2_SYNC: 6199 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 6200 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 6201 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 6202 /* With V2 type filters which specify a Sync or Delay Request, 6203 * Path Delay Request/Response messages are also time stamped 6204 * by hardware so notify the caller the requested packets plus 6205 * some others are time stamped. 6206 */ 6207 config.rx_filter = HWTSTAMP_FILTER_SOME; 6208 break; 6209 default: 6210 break; 6211 } 6212 6213 return copy_to_user(ifr->ifr_data, &config, 6214 sizeof(config)) ? -EFAULT : 0; 6215 } 6216 6217 static int e1000e_hwtstamp_get(struct net_device *netdev, struct ifreq *ifr) 6218 { 6219 struct e1000_adapter *adapter = netdev_priv(netdev); 6220 6221 return copy_to_user(ifr->ifr_data, &adapter->hwtstamp_config, 6222 sizeof(adapter->hwtstamp_config)) ? -EFAULT : 0; 6223 } 6224 6225 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 6226 { 6227 switch (cmd) { 6228 case SIOCGMIIPHY: 6229 case SIOCGMIIREG: 6230 case SIOCSMIIREG: 6231 return e1000_mii_ioctl(netdev, ifr, cmd); 6232 case SIOCSHWTSTAMP: 6233 return e1000e_hwtstamp_set(netdev, ifr); 6234 case SIOCGHWTSTAMP: 6235 return e1000e_hwtstamp_get(netdev, ifr); 6236 default: 6237 return -EOPNOTSUPP; 6238 } 6239 } 6240 6241 static int e1000_init_phy_wakeup(struct e1000_adapter *adapter, u32 wufc) 6242 { 6243 struct e1000_hw *hw = &adapter->hw; 6244 u32 i, mac_reg, wuc; 6245 u16 phy_reg, wuc_enable; 6246 int retval; 6247 6248 /* copy MAC RARs to PHY RARs */ 6249 e1000_copy_rx_addrs_to_phy_ich8lan(hw); 6250 6251 retval = hw->phy.ops.acquire(hw); 6252 if (retval) { 6253 e_err("Could not acquire PHY\n"); 6254 return retval; 6255 } 6256 6257 /* Enable access to wakeup registers on and set page to BM_WUC_PAGE */ 6258 retval = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 6259 if (retval) 6260 goto release; 6261 6262 /* copy MAC MTA to PHY MTA - only needed for pchlan */ 6263 for (i = 0; i < adapter->hw.mac.mta_reg_count; i++) { 6264 mac_reg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 6265 hw->phy.ops.write_reg_page(hw, BM_MTA(i), 6266 (u16)(mac_reg & 0xFFFF)); 6267 hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1, 6268 (u16)((mac_reg >> 16) & 0xFFFF)); 6269 } 6270 6271 /* configure PHY Rx Control register */ 6272 hw->phy.ops.read_reg_page(&adapter->hw, BM_RCTL, &phy_reg); 6273 mac_reg = er32(RCTL); 6274 if (mac_reg & E1000_RCTL_UPE) 6275 phy_reg |= BM_RCTL_UPE; 6276 if (mac_reg & E1000_RCTL_MPE) 6277 phy_reg |= BM_RCTL_MPE; 6278 phy_reg &= ~(BM_RCTL_MO_MASK); 6279 if (mac_reg & E1000_RCTL_MO_3) 6280 phy_reg |= (((mac_reg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) 6281 << BM_RCTL_MO_SHIFT); 6282 if (mac_reg & E1000_RCTL_BAM) 6283 phy_reg |= BM_RCTL_BAM; 6284 if (mac_reg & E1000_RCTL_PMCF) 6285 phy_reg |= BM_RCTL_PMCF; 6286 mac_reg = er32(CTRL); 6287 if (mac_reg & E1000_CTRL_RFCE) 6288 phy_reg |= BM_RCTL_RFCE; 6289 hw->phy.ops.write_reg_page(&adapter->hw, BM_RCTL, phy_reg); 6290 6291 wuc = E1000_WUC_PME_EN; 6292 if (wufc & (E1000_WUFC_MAG | E1000_WUFC_LNKC)) 6293 wuc |= E1000_WUC_APME; 6294 6295 /* enable PHY wakeup in MAC register */ 6296 ew32(WUFC, wufc); 6297 ew32(WUC, (E1000_WUC_PHY_WAKE | E1000_WUC_APMPME | 6298 E1000_WUC_PME_STATUS | wuc)); 6299 6300 /* configure and enable PHY wakeup in PHY registers */ 6301 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUFC, wufc); 6302 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUC, wuc); 6303 6304 /* activate PHY wakeup */ 6305 wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 6306 retval = e1000_disable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 6307 if (retval) 6308 e_err("Could not set PHY Host Wakeup bit\n"); 6309 release: 6310 hw->phy.ops.release(hw); 6311 6312 return retval; 6313 } 6314 6315 static void e1000e_flush_lpic(struct pci_dev *pdev) 6316 { 6317 struct net_device *netdev = pci_get_drvdata(pdev); 6318 struct e1000_adapter *adapter = netdev_priv(netdev); 6319 struct e1000_hw *hw = &adapter->hw; 6320 u32 ret_val; 6321 6322 pm_runtime_get_sync(netdev->dev.parent); 6323 6324 ret_val = hw->phy.ops.acquire(hw); 6325 if (ret_val) 6326 goto fl_out; 6327 6328 pr_info("EEE TX LPI TIMER: %08X\n", 6329 er32(LPIC) >> E1000_LPIC_LPIET_SHIFT); 6330 6331 hw->phy.ops.release(hw); 6332 6333 fl_out: 6334 pm_runtime_put_sync(netdev->dev.parent); 6335 } 6336 6337 /* S0ix implementation */ 6338 static void e1000e_s0ix_entry_flow(struct e1000_adapter *adapter) 6339 { 6340 struct e1000_hw *hw = &adapter->hw; 6341 u32 mac_data; 6342 u16 phy_data; 6343 6344 /* Disable the periodic inband message, 6345 * don't request PCIe clock in K1 page770_17[10:9] = 10b 6346 */ 6347 e1e_rphy(hw, HV_PM_CTRL, &phy_data); 6348 phy_data &= ~HV_PM_CTRL_K1_CLK_REQ; 6349 phy_data |= BIT(10); 6350 e1e_wphy(hw, HV_PM_CTRL, phy_data); 6351 6352 /* Make sure we don't exit K1 every time a new packet arrives 6353 * 772_29[5] = 1 CS_Mode_Stay_In_K1 6354 */ 6355 e1e_rphy(hw, I217_CGFREG, &phy_data); 6356 phy_data |= BIT(5); 6357 e1e_wphy(hw, I217_CGFREG, phy_data); 6358 6359 /* Change the MAC/PHY interface to SMBus 6360 * Force the SMBus in PHY page769_23[0] = 1 6361 * Force the SMBus in MAC CTRL_EXT[11] = 1 6362 */ 6363 e1e_rphy(hw, CV_SMB_CTRL, &phy_data); 6364 phy_data |= CV_SMB_CTRL_FORCE_SMBUS; 6365 e1e_wphy(hw, CV_SMB_CTRL, phy_data); 6366 mac_data = er32(CTRL_EXT); 6367 mac_data |= E1000_CTRL_EXT_FORCE_SMBUS; 6368 ew32(CTRL_EXT, mac_data); 6369 6370 /* DFT control: PHY bit: page769_20[0] = 1 6371 * Gate PPW via EXTCNF_CTRL - set 0x0F00[7] = 1 6372 */ 6373 e1e_rphy(hw, I82579_DFT_CTRL, &phy_data); 6374 phy_data |= BIT(0); 6375 e1e_wphy(hw, I82579_DFT_CTRL, phy_data); 6376 6377 mac_data = er32(EXTCNF_CTRL); 6378 mac_data |= E1000_EXTCNF_CTRL_GATE_PHY_CFG; 6379 ew32(EXTCNF_CTRL, mac_data); 6380 6381 /* Check MAC Tx/Rx packet buffer pointers. 6382 * Reset MAC Tx/Rx packet buffer pointers to suppress any 6383 * pending traffic indication that would prevent power gating. 6384 */ 6385 mac_data = er32(TDFH); 6386 if (mac_data) 6387 ew32(TDFH, 0); 6388 mac_data = er32(TDFT); 6389 if (mac_data) 6390 ew32(TDFT, 0); 6391 mac_data = er32(TDFHS); 6392 if (mac_data) 6393 ew32(TDFHS, 0); 6394 mac_data = er32(TDFTS); 6395 if (mac_data) 6396 ew32(TDFTS, 0); 6397 mac_data = er32(TDFPC); 6398 if (mac_data) 6399 ew32(TDFPC, 0); 6400 mac_data = er32(RDFH); 6401 if (mac_data) 6402 ew32(RDFH, 0); 6403 mac_data = er32(RDFT); 6404 if (mac_data) 6405 ew32(RDFT, 0); 6406 mac_data = er32(RDFHS); 6407 if (mac_data) 6408 ew32(RDFHS, 0); 6409 mac_data = er32(RDFTS); 6410 if (mac_data) 6411 ew32(RDFTS, 0); 6412 mac_data = er32(RDFPC); 6413 if (mac_data) 6414 ew32(RDFPC, 0); 6415 6416 /* Enable the Dynamic Power Gating in the MAC */ 6417 mac_data = er32(FEXTNVM7); 6418 mac_data |= BIT(22); 6419 ew32(FEXTNVM7, mac_data); 6420 6421 /* Disable the time synchronization clock */ 6422 mac_data = er32(FEXTNVM7); 6423 mac_data |= BIT(31); 6424 mac_data &= ~BIT(0); 6425 ew32(FEXTNVM7, mac_data); 6426 6427 /* Dynamic Power Gating Enable */ 6428 mac_data = er32(CTRL_EXT); 6429 mac_data |= BIT(3); 6430 ew32(CTRL_EXT, mac_data); 6431 6432 /* Disable disconnected cable conditioning for Power Gating */ 6433 mac_data = er32(DPGFR); 6434 mac_data |= BIT(2); 6435 ew32(DPGFR, mac_data); 6436 6437 /* Don't wake from dynamic Power Gating with clock request */ 6438 mac_data = er32(FEXTNVM12); 6439 mac_data |= BIT(12); 6440 ew32(FEXTNVM12, mac_data); 6441 6442 /* Ungate PGCB clock */ 6443 mac_data = er32(FEXTNVM9); 6444 mac_data &= ~BIT(28); 6445 ew32(FEXTNVM9, mac_data); 6446 6447 /* Enable K1 off to enable mPHY Power Gating */ 6448 mac_data = er32(FEXTNVM6); 6449 mac_data |= BIT(31); 6450 ew32(FEXTNVM6, mac_data); 6451 6452 /* Enable mPHY power gating for any link and speed */ 6453 mac_data = er32(FEXTNVM8); 6454 mac_data |= BIT(9); 6455 ew32(FEXTNVM8, mac_data); 6456 6457 /* Enable the Dynamic Clock Gating in the DMA and MAC */ 6458 mac_data = er32(CTRL_EXT); 6459 mac_data |= E1000_CTRL_EXT_DMA_DYN_CLK_EN; 6460 ew32(CTRL_EXT, mac_data); 6461 6462 /* No MAC DPG gating SLP_S0 in modern standby 6463 * Switch the logic of the lanphypc to use PMC counter 6464 */ 6465 mac_data = er32(FEXTNVM5); 6466 mac_data |= BIT(7); 6467 ew32(FEXTNVM5, mac_data); 6468 } 6469 6470 static void e1000e_s0ix_exit_flow(struct e1000_adapter *adapter) 6471 { 6472 struct e1000_hw *hw = &adapter->hw; 6473 u32 mac_data; 6474 u16 phy_data; 6475 6476 /* Disable the Dynamic Power Gating in the MAC */ 6477 mac_data = er32(FEXTNVM7); 6478 mac_data &= 0xFFBFFFFF; 6479 ew32(FEXTNVM7, mac_data); 6480 6481 /* Enable the time synchronization clock */ 6482 mac_data = er32(FEXTNVM7); 6483 mac_data |= BIT(0); 6484 ew32(FEXTNVM7, mac_data); 6485 6486 /* Disable mPHY power gating for any link and speed */ 6487 mac_data = er32(FEXTNVM8); 6488 mac_data &= ~BIT(9); 6489 ew32(FEXTNVM8, mac_data); 6490 6491 /* Disable K1 off */ 6492 mac_data = er32(FEXTNVM6); 6493 mac_data &= ~BIT(31); 6494 ew32(FEXTNVM6, mac_data); 6495 6496 /* Disable Ungate PGCB clock */ 6497 mac_data = er32(FEXTNVM9); 6498 mac_data |= BIT(28); 6499 ew32(FEXTNVM9, mac_data); 6500 6501 /* Cancel not waking from dynamic 6502 * Power Gating with clock request 6503 */ 6504 mac_data = er32(FEXTNVM12); 6505 mac_data &= ~BIT(12); 6506 ew32(FEXTNVM12, mac_data); 6507 6508 /* Cancel disable disconnected cable conditioning 6509 * for Power Gating 6510 */ 6511 mac_data = er32(DPGFR); 6512 mac_data &= ~BIT(2); 6513 ew32(DPGFR, mac_data); 6514 6515 /* Disable Dynamic Power Gating */ 6516 mac_data = er32(CTRL_EXT); 6517 mac_data &= 0xFFFFFFF7; 6518 ew32(CTRL_EXT, mac_data); 6519 6520 /* Disable the Dynamic Clock Gating in the DMA and MAC */ 6521 mac_data = er32(CTRL_EXT); 6522 mac_data &= 0xFFF7FFFF; 6523 ew32(CTRL_EXT, mac_data); 6524 6525 /* Revert the lanphypc logic to use the internal Gbe counter 6526 * and not the PMC counter 6527 */ 6528 mac_data = er32(FEXTNVM5); 6529 mac_data &= 0xFFFFFF7F; 6530 ew32(FEXTNVM5, mac_data); 6531 6532 /* Enable the periodic inband message, 6533 * Request PCIe clock in K1 page770_17[10:9] =01b 6534 */ 6535 e1e_rphy(hw, HV_PM_CTRL, &phy_data); 6536 phy_data &= 0xFBFF; 6537 phy_data |= HV_PM_CTRL_K1_CLK_REQ; 6538 e1e_wphy(hw, HV_PM_CTRL, phy_data); 6539 6540 /* Return back configuration 6541 * 772_29[5] = 0 CS_Mode_Stay_In_K1 6542 */ 6543 e1e_rphy(hw, I217_CGFREG, &phy_data); 6544 phy_data &= 0xFFDF; 6545 e1e_wphy(hw, I217_CGFREG, phy_data); 6546 6547 /* Change the MAC/PHY interface to Kumeran 6548 * Unforce the SMBus in PHY page769_23[0] = 0 6549 * Unforce the SMBus in MAC CTRL_EXT[11] = 0 6550 */ 6551 e1e_rphy(hw, CV_SMB_CTRL, &phy_data); 6552 phy_data &= ~CV_SMB_CTRL_FORCE_SMBUS; 6553 e1e_wphy(hw, CV_SMB_CTRL, phy_data); 6554 mac_data = er32(CTRL_EXT); 6555 mac_data &= ~E1000_CTRL_EXT_FORCE_SMBUS; 6556 ew32(CTRL_EXT, mac_data); 6557 } 6558 6559 static int e1000e_pm_freeze(struct device *dev) 6560 { 6561 struct net_device *netdev = dev_get_drvdata(dev); 6562 struct e1000_adapter *adapter = netdev_priv(netdev); 6563 bool present; 6564 6565 rtnl_lock(); 6566 6567 present = netif_device_present(netdev); 6568 netif_device_detach(netdev); 6569 6570 if (present && netif_running(netdev)) { 6571 int count = E1000_CHECK_RESET_COUNT; 6572 6573 while (test_bit(__E1000_RESETTING, &adapter->state) && count--) 6574 usleep_range(10000, 11000); 6575 6576 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state)); 6577 6578 /* Quiesce the device without resetting the hardware */ 6579 e1000e_down(adapter, false); 6580 e1000_free_irq(adapter); 6581 } 6582 rtnl_unlock(); 6583 6584 e1000e_reset_interrupt_capability(adapter); 6585 6586 /* Allow time for pending master requests to run */ 6587 e1000e_disable_pcie_master(&adapter->hw); 6588 6589 return 0; 6590 } 6591 6592 static int __e1000_shutdown(struct pci_dev *pdev, bool runtime) 6593 { 6594 struct net_device *netdev = pci_get_drvdata(pdev); 6595 struct e1000_adapter *adapter = netdev_priv(netdev); 6596 struct e1000_hw *hw = &adapter->hw; 6597 u32 ctrl, ctrl_ext, rctl, status, wufc; 6598 int retval = 0; 6599 6600 /* Runtime suspend should only enable wakeup for link changes */ 6601 if (runtime) 6602 wufc = E1000_WUFC_LNKC; 6603 else if (device_may_wakeup(&pdev->dev)) 6604 wufc = adapter->wol; 6605 else 6606 wufc = 0; 6607 6608 status = er32(STATUS); 6609 if (status & E1000_STATUS_LU) 6610 wufc &= ~E1000_WUFC_LNKC; 6611 6612 if (wufc) { 6613 e1000_setup_rctl(adapter); 6614 e1000e_set_rx_mode(netdev); 6615 6616 /* turn on all-multi mode if wake on multicast is enabled */ 6617 if (wufc & E1000_WUFC_MC) { 6618 rctl = er32(RCTL); 6619 rctl |= E1000_RCTL_MPE; 6620 ew32(RCTL, rctl); 6621 } 6622 6623 ctrl = er32(CTRL); 6624 ctrl |= E1000_CTRL_ADVD3WUC; 6625 if (!(adapter->flags2 & FLAG2_HAS_PHY_WAKEUP)) 6626 ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT; 6627 ew32(CTRL, ctrl); 6628 6629 if (adapter->hw.phy.media_type == e1000_media_type_fiber || 6630 adapter->hw.phy.media_type == 6631 e1000_media_type_internal_serdes) { 6632 /* keep the laser running in D3 */ 6633 ctrl_ext = er32(CTRL_EXT); 6634 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 6635 ew32(CTRL_EXT, ctrl_ext); 6636 } 6637 6638 if (!runtime) 6639 e1000e_power_up_phy(adapter); 6640 6641 if (adapter->flags & FLAG_IS_ICH) 6642 e1000_suspend_workarounds_ich8lan(&adapter->hw); 6643 6644 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) { 6645 /* enable wakeup by the PHY */ 6646 retval = e1000_init_phy_wakeup(adapter, wufc); 6647 if (retval) 6648 return retval; 6649 } else { 6650 /* enable wakeup by the MAC */ 6651 ew32(WUFC, wufc); 6652 ew32(WUC, E1000_WUC_PME_EN); 6653 } 6654 } else { 6655 ew32(WUC, 0); 6656 ew32(WUFC, 0); 6657 6658 e1000_power_down_phy(adapter); 6659 } 6660 6661 if (adapter->hw.phy.type == e1000_phy_igp_3) { 6662 e1000e_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw); 6663 } else if (hw->mac.type >= e1000_pch_lpt) { 6664 if (wufc && !(wufc & (E1000_WUFC_EX | E1000_WUFC_MC | E1000_WUFC_BC))) 6665 /* ULP does not support wake from unicast, multicast 6666 * or broadcast. 6667 */ 6668 retval = e1000_enable_ulp_lpt_lp(hw, !runtime); 6669 6670 if (retval) 6671 return retval; 6672 } 6673 6674 /* Ensure that the appropriate bits are set in LPI_CTRL 6675 * for EEE in Sx 6676 */ 6677 if ((hw->phy.type >= e1000_phy_i217) && 6678 adapter->eee_advert && hw->dev_spec.ich8lan.eee_lp_ability) { 6679 u16 lpi_ctrl = 0; 6680 6681 retval = hw->phy.ops.acquire(hw); 6682 if (!retval) { 6683 retval = e1e_rphy_locked(hw, I82579_LPI_CTRL, 6684 &lpi_ctrl); 6685 if (!retval) { 6686 if (adapter->eee_advert & 6687 hw->dev_spec.ich8lan.eee_lp_ability & 6688 I82579_EEE_100_SUPPORTED) 6689 lpi_ctrl |= I82579_LPI_CTRL_100_ENABLE; 6690 if (adapter->eee_advert & 6691 hw->dev_spec.ich8lan.eee_lp_ability & 6692 I82579_EEE_1000_SUPPORTED) 6693 lpi_ctrl |= I82579_LPI_CTRL_1000_ENABLE; 6694 6695 retval = e1e_wphy_locked(hw, I82579_LPI_CTRL, 6696 lpi_ctrl); 6697 } 6698 } 6699 hw->phy.ops.release(hw); 6700 } 6701 6702 /* Release control of h/w to f/w. If f/w is AMT enabled, this 6703 * would have already happened in close and is redundant. 6704 */ 6705 e1000e_release_hw_control(adapter); 6706 6707 pci_clear_master(pdev); 6708 6709 /* The pci-e switch on some quad port adapters will report a 6710 * correctable error when the MAC transitions from D0 to D3. To 6711 * prevent this we need to mask off the correctable errors on the 6712 * downstream port of the pci-e switch. 6713 * 6714 * We don't have the associated upstream bridge while assigning 6715 * the PCI device into guest. For example, the KVM on power is 6716 * one of the cases. 6717 */ 6718 if (adapter->flags & FLAG_IS_QUAD_PORT) { 6719 struct pci_dev *us_dev = pdev->bus->self; 6720 u16 devctl; 6721 6722 if (!us_dev) 6723 return 0; 6724 6725 pcie_capability_read_word(us_dev, PCI_EXP_DEVCTL, &devctl); 6726 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, 6727 (devctl & ~PCI_EXP_DEVCTL_CERE)); 6728 6729 pci_save_state(pdev); 6730 pci_prepare_to_sleep(pdev); 6731 6732 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, devctl); 6733 } 6734 6735 return 0; 6736 } 6737 6738 /** 6739 * __e1000e_disable_aspm - Disable ASPM states 6740 * @pdev: pointer to PCI device struct 6741 * @state: bit-mask of ASPM states to disable 6742 * @locked: indication if this context holds pci_bus_sem locked. 6743 * 6744 * Some devices *must* have certain ASPM states disabled per hardware errata. 6745 **/ 6746 static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state, int locked) 6747 { 6748 struct pci_dev *parent = pdev->bus->self; 6749 u16 aspm_dis_mask = 0; 6750 u16 pdev_aspmc, parent_aspmc; 6751 6752 switch (state) { 6753 case PCIE_LINK_STATE_L0S: 6754 case PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1: 6755 aspm_dis_mask |= PCI_EXP_LNKCTL_ASPM_L0S; 6756 fallthrough; /* can't have L1 without L0s */ 6757 case PCIE_LINK_STATE_L1: 6758 aspm_dis_mask |= PCI_EXP_LNKCTL_ASPM_L1; 6759 break; 6760 default: 6761 return; 6762 } 6763 6764 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &pdev_aspmc); 6765 pdev_aspmc &= PCI_EXP_LNKCTL_ASPMC; 6766 6767 if (parent) { 6768 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, 6769 &parent_aspmc); 6770 parent_aspmc &= PCI_EXP_LNKCTL_ASPMC; 6771 } 6772 6773 /* Nothing to do if the ASPM states to be disabled already are */ 6774 if (!(pdev_aspmc & aspm_dis_mask) && 6775 (!parent || !(parent_aspmc & aspm_dis_mask))) 6776 return; 6777 6778 dev_info(&pdev->dev, "Disabling ASPM %s %s\n", 6779 (aspm_dis_mask & pdev_aspmc & PCI_EXP_LNKCTL_ASPM_L0S) ? 6780 "L0s" : "", 6781 (aspm_dis_mask & pdev_aspmc & PCI_EXP_LNKCTL_ASPM_L1) ? 6782 "L1" : ""); 6783 6784 #ifdef CONFIG_PCIEASPM 6785 if (locked) 6786 pci_disable_link_state_locked(pdev, state); 6787 else 6788 pci_disable_link_state(pdev, state); 6789 6790 /* Double-check ASPM control. If not disabled by the above, the 6791 * BIOS is preventing that from happening (or CONFIG_PCIEASPM is 6792 * not enabled); override by writing PCI config space directly. 6793 */ 6794 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &pdev_aspmc); 6795 pdev_aspmc &= PCI_EXP_LNKCTL_ASPMC; 6796 6797 if (!(aspm_dis_mask & pdev_aspmc)) 6798 return; 6799 #endif 6800 6801 /* Both device and parent should have the same ASPM setting. 6802 * Disable ASPM in downstream component first and then upstream. 6803 */ 6804 pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, aspm_dis_mask); 6805 6806 if (parent) 6807 pcie_capability_clear_word(parent, PCI_EXP_LNKCTL, 6808 aspm_dis_mask); 6809 } 6810 6811 /** 6812 * e1000e_disable_aspm - Disable ASPM states. 6813 * @pdev: pointer to PCI device struct 6814 * @state: bit-mask of ASPM states to disable 6815 * 6816 * This function acquires the pci_bus_sem! 6817 * Some devices *must* have certain ASPM states disabled per hardware errata. 6818 **/ 6819 static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state) 6820 { 6821 __e1000e_disable_aspm(pdev, state, 0); 6822 } 6823 6824 /** 6825 * e1000e_disable_aspm_locked - Disable ASPM states. 6826 * @pdev: pointer to PCI device struct 6827 * @state: bit-mask of ASPM states to disable 6828 * 6829 * This function must be called with pci_bus_sem acquired! 6830 * Some devices *must* have certain ASPM states disabled per hardware errata. 6831 **/ 6832 static void e1000e_disable_aspm_locked(struct pci_dev *pdev, u16 state) 6833 { 6834 __e1000e_disable_aspm(pdev, state, 1); 6835 } 6836 6837 static int e1000e_pm_thaw(struct device *dev) 6838 { 6839 struct net_device *netdev = dev_get_drvdata(dev); 6840 struct e1000_adapter *adapter = netdev_priv(netdev); 6841 int rc = 0; 6842 6843 e1000e_set_interrupt_capability(adapter); 6844 6845 rtnl_lock(); 6846 if (netif_running(netdev)) { 6847 rc = e1000_request_irq(adapter); 6848 if (rc) 6849 goto err_irq; 6850 6851 e1000e_up(adapter); 6852 } 6853 6854 netif_device_attach(netdev); 6855 err_irq: 6856 rtnl_unlock(); 6857 6858 return rc; 6859 } 6860 6861 static int __e1000_resume(struct pci_dev *pdev) 6862 { 6863 struct net_device *netdev = pci_get_drvdata(pdev); 6864 struct e1000_adapter *adapter = netdev_priv(netdev); 6865 struct e1000_hw *hw = &adapter->hw; 6866 u16 aspm_disable_flag = 0; 6867 6868 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S) 6869 aspm_disable_flag = PCIE_LINK_STATE_L0S; 6870 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1) 6871 aspm_disable_flag |= PCIE_LINK_STATE_L1; 6872 if (aspm_disable_flag) 6873 e1000e_disable_aspm(pdev, aspm_disable_flag); 6874 6875 pci_set_master(pdev); 6876 6877 if (hw->mac.type >= e1000_pch2lan) 6878 e1000_resume_workarounds_pchlan(&adapter->hw); 6879 6880 e1000e_power_up_phy(adapter); 6881 6882 /* report the system wakeup cause from S3/S4 */ 6883 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) { 6884 u16 phy_data; 6885 6886 e1e_rphy(&adapter->hw, BM_WUS, &phy_data); 6887 if (phy_data) { 6888 e_info("PHY Wakeup cause - %s\n", 6889 phy_data & E1000_WUS_EX ? "Unicast Packet" : 6890 phy_data & E1000_WUS_MC ? "Multicast Packet" : 6891 phy_data & E1000_WUS_BC ? "Broadcast Packet" : 6892 phy_data & E1000_WUS_MAG ? "Magic Packet" : 6893 phy_data & E1000_WUS_LNKC ? 6894 "Link Status Change" : "other"); 6895 } 6896 e1e_wphy(&adapter->hw, BM_WUS, ~0); 6897 } else { 6898 u32 wus = er32(WUS); 6899 6900 if (wus) { 6901 e_info("MAC Wakeup cause - %s\n", 6902 wus & E1000_WUS_EX ? "Unicast Packet" : 6903 wus & E1000_WUS_MC ? "Multicast Packet" : 6904 wus & E1000_WUS_BC ? "Broadcast Packet" : 6905 wus & E1000_WUS_MAG ? "Magic Packet" : 6906 wus & E1000_WUS_LNKC ? "Link Status Change" : 6907 "other"); 6908 } 6909 ew32(WUS, ~0); 6910 } 6911 6912 e1000e_reset(adapter); 6913 6914 e1000_init_manageability_pt(adapter); 6915 6916 /* If the controller has AMT, do not set DRV_LOAD until the interface 6917 * is up. For all other cases, let the f/w know that the h/w is now 6918 * under the control of the driver. 6919 */ 6920 if (!(adapter->flags & FLAG_HAS_AMT)) 6921 e1000e_get_hw_control(adapter); 6922 6923 return 0; 6924 } 6925 6926 static __maybe_unused int e1000e_pm_prepare(struct device *dev) 6927 { 6928 return pm_runtime_suspended(dev) && 6929 pm_suspend_via_firmware(); 6930 } 6931 6932 static __maybe_unused int e1000e_pm_suspend(struct device *dev) 6933 { 6934 struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev)); 6935 struct e1000_adapter *adapter = netdev_priv(netdev); 6936 struct pci_dev *pdev = to_pci_dev(dev); 6937 int rc; 6938 6939 e1000e_flush_lpic(pdev); 6940 6941 e1000e_pm_freeze(dev); 6942 6943 rc = __e1000_shutdown(pdev, false); 6944 if (rc) { 6945 e1000e_pm_thaw(dev); 6946 } else { 6947 /* Introduce S0ix implementation */ 6948 if (adapter->flags2 & FLAG2_ENABLE_S0IX_FLOWS) 6949 e1000e_s0ix_entry_flow(adapter); 6950 } 6951 6952 return rc; 6953 } 6954 6955 static __maybe_unused int e1000e_pm_resume(struct device *dev) 6956 { 6957 struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev)); 6958 struct e1000_adapter *adapter = netdev_priv(netdev); 6959 struct pci_dev *pdev = to_pci_dev(dev); 6960 int rc; 6961 6962 /* Introduce S0ix implementation */ 6963 if (adapter->flags2 & FLAG2_ENABLE_S0IX_FLOWS) 6964 e1000e_s0ix_exit_flow(adapter); 6965 6966 rc = __e1000_resume(pdev); 6967 if (rc) 6968 return rc; 6969 6970 return e1000e_pm_thaw(dev); 6971 } 6972 6973 static __maybe_unused int e1000e_pm_runtime_idle(struct device *dev) 6974 { 6975 struct net_device *netdev = dev_get_drvdata(dev); 6976 struct e1000_adapter *adapter = netdev_priv(netdev); 6977 u16 eee_lp; 6978 6979 eee_lp = adapter->hw.dev_spec.ich8lan.eee_lp_ability; 6980 6981 if (!e1000e_has_link(adapter)) { 6982 adapter->hw.dev_spec.ich8lan.eee_lp_ability = eee_lp; 6983 pm_schedule_suspend(dev, 5 * MSEC_PER_SEC); 6984 } 6985 6986 return -EBUSY; 6987 } 6988 6989 static __maybe_unused int e1000e_pm_runtime_resume(struct device *dev) 6990 { 6991 struct pci_dev *pdev = to_pci_dev(dev); 6992 struct net_device *netdev = pci_get_drvdata(pdev); 6993 struct e1000_adapter *adapter = netdev_priv(netdev); 6994 int rc; 6995 6996 rc = __e1000_resume(pdev); 6997 if (rc) 6998 return rc; 6999 7000 if (netdev->flags & IFF_UP) 7001 e1000e_up(adapter); 7002 7003 return rc; 7004 } 7005 7006 static __maybe_unused int e1000e_pm_runtime_suspend(struct device *dev) 7007 { 7008 struct pci_dev *pdev = to_pci_dev(dev); 7009 struct net_device *netdev = pci_get_drvdata(pdev); 7010 struct e1000_adapter *adapter = netdev_priv(netdev); 7011 7012 if (netdev->flags & IFF_UP) { 7013 int count = E1000_CHECK_RESET_COUNT; 7014 7015 while (test_bit(__E1000_RESETTING, &adapter->state) && count--) 7016 usleep_range(10000, 11000); 7017 7018 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state)); 7019 7020 /* Down the device without resetting the hardware */ 7021 e1000e_down(adapter, false); 7022 } 7023 7024 if (__e1000_shutdown(pdev, true)) { 7025 e1000e_pm_runtime_resume(dev); 7026 return -EBUSY; 7027 } 7028 7029 return 0; 7030 } 7031 7032 static void e1000_shutdown(struct pci_dev *pdev) 7033 { 7034 e1000e_flush_lpic(pdev); 7035 7036 e1000e_pm_freeze(&pdev->dev); 7037 7038 __e1000_shutdown(pdev, false); 7039 } 7040 7041 #ifdef CONFIG_NET_POLL_CONTROLLER 7042 7043 static irqreturn_t e1000_intr_msix(int __always_unused irq, void *data) 7044 { 7045 struct net_device *netdev = data; 7046 struct e1000_adapter *adapter = netdev_priv(netdev); 7047 7048 if (adapter->msix_entries) { 7049 int vector, msix_irq; 7050 7051 vector = 0; 7052 msix_irq = adapter->msix_entries[vector].vector; 7053 if (disable_hardirq(msix_irq)) 7054 e1000_intr_msix_rx(msix_irq, netdev); 7055 enable_irq(msix_irq); 7056 7057 vector++; 7058 msix_irq = adapter->msix_entries[vector].vector; 7059 if (disable_hardirq(msix_irq)) 7060 e1000_intr_msix_tx(msix_irq, netdev); 7061 enable_irq(msix_irq); 7062 7063 vector++; 7064 msix_irq = adapter->msix_entries[vector].vector; 7065 if (disable_hardirq(msix_irq)) 7066 e1000_msix_other(msix_irq, netdev); 7067 enable_irq(msix_irq); 7068 } 7069 7070 return IRQ_HANDLED; 7071 } 7072 7073 /** 7074 * e1000_netpoll 7075 * @netdev: network interface device structure 7076 * 7077 * Polling 'interrupt' - used by things like netconsole to send skbs 7078 * without having to re-enable interrupts. It's not called while 7079 * the interrupt routine is executing. 7080 */ 7081 static void e1000_netpoll(struct net_device *netdev) 7082 { 7083 struct e1000_adapter *adapter = netdev_priv(netdev); 7084 7085 switch (adapter->int_mode) { 7086 case E1000E_INT_MODE_MSIX: 7087 e1000_intr_msix(adapter->pdev->irq, netdev); 7088 break; 7089 case E1000E_INT_MODE_MSI: 7090 if (disable_hardirq(adapter->pdev->irq)) 7091 e1000_intr_msi(adapter->pdev->irq, netdev); 7092 enable_irq(adapter->pdev->irq); 7093 break; 7094 default: /* E1000E_INT_MODE_LEGACY */ 7095 if (disable_hardirq(adapter->pdev->irq)) 7096 e1000_intr(adapter->pdev->irq, netdev); 7097 enable_irq(adapter->pdev->irq); 7098 break; 7099 } 7100 } 7101 #endif 7102 7103 /** 7104 * e1000_io_error_detected - called when PCI error is detected 7105 * @pdev: Pointer to PCI device 7106 * @state: The current pci connection state 7107 * 7108 * This function is called after a PCI bus error affecting 7109 * this device has been detected. 7110 */ 7111 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev, 7112 pci_channel_state_t state) 7113 { 7114 e1000e_pm_freeze(&pdev->dev); 7115 7116 if (state == pci_channel_io_perm_failure) 7117 return PCI_ERS_RESULT_DISCONNECT; 7118 7119 pci_disable_device(pdev); 7120 7121 /* Request a slot slot reset. */ 7122 return PCI_ERS_RESULT_NEED_RESET; 7123 } 7124 7125 /** 7126 * e1000_io_slot_reset - called after the pci bus has been reset. 7127 * @pdev: Pointer to PCI device 7128 * 7129 * Restart the card from scratch, as if from a cold-boot. Implementation 7130 * resembles the first-half of the e1000e_pm_resume routine. 7131 */ 7132 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev) 7133 { 7134 struct net_device *netdev = pci_get_drvdata(pdev); 7135 struct e1000_adapter *adapter = netdev_priv(netdev); 7136 struct e1000_hw *hw = &adapter->hw; 7137 u16 aspm_disable_flag = 0; 7138 int err; 7139 pci_ers_result_t result; 7140 7141 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S) 7142 aspm_disable_flag = PCIE_LINK_STATE_L0S; 7143 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1) 7144 aspm_disable_flag |= PCIE_LINK_STATE_L1; 7145 if (aspm_disable_flag) 7146 e1000e_disable_aspm_locked(pdev, aspm_disable_flag); 7147 7148 err = pci_enable_device_mem(pdev); 7149 if (err) { 7150 dev_err(&pdev->dev, 7151 "Cannot re-enable PCI device after reset.\n"); 7152 result = PCI_ERS_RESULT_DISCONNECT; 7153 } else { 7154 pdev->state_saved = true; 7155 pci_restore_state(pdev); 7156 pci_set_master(pdev); 7157 7158 pci_enable_wake(pdev, PCI_D3hot, 0); 7159 pci_enable_wake(pdev, PCI_D3cold, 0); 7160 7161 e1000e_reset(adapter); 7162 ew32(WUS, ~0); 7163 result = PCI_ERS_RESULT_RECOVERED; 7164 } 7165 7166 return result; 7167 } 7168 7169 /** 7170 * e1000_io_resume - called when traffic can start flowing again. 7171 * @pdev: Pointer to PCI device 7172 * 7173 * This callback is called when the error recovery driver tells us that 7174 * its OK to resume normal operation. Implementation resembles the 7175 * second-half of the e1000e_pm_resume routine. 7176 */ 7177 static void e1000_io_resume(struct pci_dev *pdev) 7178 { 7179 struct net_device *netdev = pci_get_drvdata(pdev); 7180 struct e1000_adapter *adapter = netdev_priv(netdev); 7181 7182 e1000_init_manageability_pt(adapter); 7183 7184 e1000e_pm_thaw(&pdev->dev); 7185 7186 /* If the controller has AMT, do not set DRV_LOAD until the interface 7187 * is up. For all other cases, let the f/w know that the h/w is now 7188 * under the control of the driver. 7189 */ 7190 if (!(adapter->flags & FLAG_HAS_AMT)) 7191 e1000e_get_hw_control(adapter); 7192 } 7193 7194 static void e1000_print_device_info(struct e1000_adapter *adapter) 7195 { 7196 struct e1000_hw *hw = &adapter->hw; 7197 struct net_device *netdev = adapter->netdev; 7198 u32 ret_val; 7199 u8 pba_str[E1000_PBANUM_LENGTH]; 7200 7201 /* print bus type/speed/width info */ 7202 e_info("(PCI Express:2.5GT/s:%s) %pM\n", 7203 /* bus width */ 7204 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" : 7205 "Width x1"), 7206 /* MAC address */ 7207 netdev->dev_addr); 7208 e_info("Intel(R) PRO/%s Network Connection\n", 7209 (hw->phy.type == e1000_phy_ife) ? "10/100" : "1000"); 7210 ret_val = e1000_read_pba_string_generic(hw, pba_str, 7211 E1000_PBANUM_LENGTH); 7212 if (ret_val) 7213 strlcpy((char *)pba_str, "Unknown", sizeof(pba_str)); 7214 e_info("MAC: %d, PHY: %d, PBA No: %s\n", 7215 hw->mac.type, hw->phy.type, pba_str); 7216 } 7217 7218 static void e1000_eeprom_checks(struct e1000_adapter *adapter) 7219 { 7220 struct e1000_hw *hw = &adapter->hw; 7221 int ret_val; 7222 u16 buf = 0; 7223 7224 if (hw->mac.type != e1000_82573) 7225 return; 7226 7227 ret_val = e1000_read_nvm(hw, NVM_INIT_CONTROL2_REG, 1, &buf); 7228 le16_to_cpus(&buf); 7229 if (!ret_val && (!(buf & BIT(0)))) { 7230 /* Deep Smart Power Down (DSPD) */ 7231 dev_warn(&adapter->pdev->dev, 7232 "Warning: detected DSPD enabled in EEPROM\n"); 7233 } 7234 } 7235 7236 static netdev_features_t e1000_fix_features(struct net_device *netdev, 7237 netdev_features_t features) 7238 { 7239 struct e1000_adapter *adapter = netdev_priv(netdev); 7240 struct e1000_hw *hw = &adapter->hw; 7241 7242 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */ 7243 if ((hw->mac.type >= e1000_pch2lan) && (netdev->mtu > ETH_DATA_LEN)) 7244 features &= ~NETIF_F_RXFCS; 7245 7246 /* Since there is no support for separate Rx/Tx vlan accel 7247 * enable/disable make sure Tx flag is always in same state as Rx. 7248 */ 7249 if (features & NETIF_F_HW_VLAN_CTAG_RX) 7250 features |= NETIF_F_HW_VLAN_CTAG_TX; 7251 else 7252 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 7253 7254 return features; 7255 } 7256 7257 static int e1000_set_features(struct net_device *netdev, 7258 netdev_features_t features) 7259 { 7260 struct e1000_adapter *adapter = netdev_priv(netdev); 7261 netdev_features_t changed = features ^ netdev->features; 7262 7263 if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) 7264 adapter->flags |= FLAG_TSO_FORCE; 7265 7266 if (!(changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX | 7267 NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_RXFCS | 7268 NETIF_F_RXALL))) 7269 return 0; 7270 7271 if (changed & NETIF_F_RXFCS) { 7272 if (features & NETIF_F_RXFCS) { 7273 adapter->flags2 &= ~FLAG2_CRC_STRIPPING; 7274 } else { 7275 /* We need to take it back to defaults, which might mean 7276 * stripping is still disabled at the adapter level. 7277 */ 7278 if (adapter->flags2 & FLAG2_DFLT_CRC_STRIPPING) 7279 adapter->flags2 |= FLAG2_CRC_STRIPPING; 7280 else 7281 adapter->flags2 &= ~FLAG2_CRC_STRIPPING; 7282 } 7283 } 7284 7285 netdev->features = features; 7286 7287 if (netif_running(netdev)) 7288 e1000e_reinit_locked(adapter); 7289 else 7290 e1000e_reset(adapter); 7291 7292 return 1; 7293 } 7294 7295 static const struct net_device_ops e1000e_netdev_ops = { 7296 .ndo_open = e1000e_open, 7297 .ndo_stop = e1000e_close, 7298 .ndo_start_xmit = e1000_xmit_frame, 7299 .ndo_get_stats64 = e1000e_get_stats64, 7300 .ndo_set_rx_mode = e1000e_set_rx_mode, 7301 .ndo_set_mac_address = e1000_set_mac, 7302 .ndo_change_mtu = e1000_change_mtu, 7303 .ndo_do_ioctl = e1000_ioctl, 7304 .ndo_tx_timeout = e1000_tx_timeout, 7305 .ndo_validate_addr = eth_validate_addr, 7306 7307 .ndo_vlan_rx_add_vid = e1000_vlan_rx_add_vid, 7308 .ndo_vlan_rx_kill_vid = e1000_vlan_rx_kill_vid, 7309 #ifdef CONFIG_NET_POLL_CONTROLLER 7310 .ndo_poll_controller = e1000_netpoll, 7311 #endif 7312 .ndo_set_features = e1000_set_features, 7313 .ndo_fix_features = e1000_fix_features, 7314 .ndo_features_check = passthru_features_check, 7315 }; 7316 7317 /** 7318 * e1000_probe - Device Initialization Routine 7319 * @pdev: PCI device information struct 7320 * @ent: entry in e1000_pci_tbl 7321 * 7322 * Returns 0 on success, negative on failure 7323 * 7324 * e1000_probe initializes an adapter identified by a pci_dev structure. 7325 * The OS initialization, configuring of the adapter private structure, 7326 * and a hardware reset occur. 7327 **/ 7328 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 7329 { 7330 struct net_device *netdev; 7331 struct e1000_adapter *adapter; 7332 struct e1000_hw *hw; 7333 const struct e1000_info *ei = e1000_info_tbl[ent->driver_data]; 7334 resource_size_t mmio_start, mmio_len; 7335 resource_size_t flash_start, flash_len; 7336 static int cards_found; 7337 u16 aspm_disable_flag = 0; 7338 int bars, i, err, pci_using_dac; 7339 u16 eeprom_data = 0; 7340 u16 eeprom_apme_mask = E1000_EEPROM_APME; 7341 s32 ret_val = 0; 7342 7343 if (ei->flags2 & FLAG2_DISABLE_ASPM_L0S) 7344 aspm_disable_flag = PCIE_LINK_STATE_L0S; 7345 if (ei->flags2 & FLAG2_DISABLE_ASPM_L1) 7346 aspm_disable_flag |= PCIE_LINK_STATE_L1; 7347 if (aspm_disable_flag) 7348 e1000e_disable_aspm(pdev, aspm_disable_flag); 7349 7350 err = pci_enable_device_mem(pdev); 7351 if (err) 7352 return err; 7353 7354 pci_using_dac = 0; 7355 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 7356 if (!err) { 7357 pci_using_dac = 1; 7358 } else { 7359 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 7360 if (err) { 7361 dev_err(&pdev->dev, 7362 "No usable DMA configuration, aborting\n"); 7363 goto err_dma; 7364 } 7365 } 7366 7367 bars = pci_select_bars(pdev, IORESOURCE_MEM); 7368 err = pci_request_selected_regions_exclusive(pdev, bars, 7369 e1000e_driver_name); 7370 if (err) 7371 goto err_pci_reg; 7372 7373 /* AER (Advanced Error Reporting) hooks */ 7374 pci_enable_pcie_error_reporting(pdev); 7375 7376 pci_set_master(pdev); 7377 /* PCI config space info */ 7378 err = pci_save_state(pdev); 7379 if (err) 7380 goto err_alloc_etherdev; 7381 7382 err = -ENOMEM; 7383 netdev = alloc_etherdev(sizeof(struct e1000_adapter)); 7384 if (!netdev) 7385 goto err_alloc_etherdev; 7386 7387 SET_NETDEV_DEV(netdev, &pdev->dev); 7388 7389 netdev->irq = pdev->irq; 7390 7391 pci_set_drvdata(pdev, netdev); 7392 adapter = netdev_priv(netdev); 7393 hw = &adapter->hw; 7394 adapter->netdev = netdev; 7395 adapter->pdev = pdev; 7396 adapter->ei = ei; 7397 adapter->pba = ei->pba; 7398 adapter->flags = ei->flags; 7399 adapter->flags2 = ei->flags2; 7400 adapter->hw.adapter = adapter; 7401 adapter->hw.mac.type = ei->mac; 7402 adapter->max_hw_frame_size = ei->max_hw_frame_size; 7403 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 7404 7405 mmio_start = pci_resource_start(pdev, 0); 7406 mmio_len = pci_resource_len(pdev, 0); 7407 7408 err = -EIO; 7409 adapter->hw.hw_addr = ioremap(mmio_start, mmio_len); 7410 if (!adapter->hw.hw_addr) 7411 goto err_ioremap; 7412 7413 if ((adapter->flags & FLAG_HAS_FLASH) && 7414 (pci_resource_flags(pdev, 1) & IORESOURCE_MEM) && 7415 (hw->mac.type < e1000_pch_spt)) { 7416 flash_start = pci_resource_start(pdev, 1); 7417 flash_len = pci_resource_len(pdev, 1); 7418 adapter->hw.flash_address = ioremap(flash_start, flash_len); 7419 if (!adapter->hw.flash_address) 7420 goto err_flashmap; 7421 } 7422 7423 /* Set default EEE advertisement */ 7424 if (adapter->flags2 & FLAG2_HAS_EEE) 7425 adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T; 7426 7427 /* construct the net_device struct */ 7428 netdev->netdev_ops = &e1000e_netdev_ops; 7429 e1000e_set_ethtool_ops(netdev); 7430 netdev->watchdog_timeo = 5 * HZ; 7431 netif_napi_add(netdev, &adapter->napi, e1000e_poll, 64); 7432 strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name)); 7433 7434 netdev->mem_start = mmio_start; 7435 netdev->mem_end = mmio_start + mmio_len; 7436 7437 adapter->bd_number = cards_found++; 7438 7439 e1000e_check_options(adapter); 7440 7441 /* setup adapter struct */ 7442 err = e1000_sw_init(adapter); 7443 if (err) 7444 goto err_sw_init; 7445 7446 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 7447 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops)); 7448 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 7449 7450 err = ei->get_variants(adapter); 7451 if (err) 7452 goto err_hw_init; 7453 7454 if ((adapter->flags & FLAG_IS_ICH) && 7455 (adapter->flags & FLAG_READ_ONLY_NVM) && 7456 (hw->mac.type < e1000_pch_spt)) 7457 e1000e_write_protect_nvm_ich8lan(&adapter->hw); 7458 7459 hw->mac.ops.get_bus_info(&adapter->hw); 7460 7461 adapter->hw.phy.autoneg_wait_to_complete = 0; 7462 7463 /* Copper options */ 7464 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 7465 adapter->hw.phy.mdix = AUTO_ALL_MODES; 7466 adapter->hw.phy.disable_polarity_correction = 0; 7467 adapter->hw.phy.ms_type = e1000_ms_hw_default; 7468 } 7469 7470 if (hw->phy.ops.check_reset_block && hw->phy.ops.check_reset_block(hw)) 7471 dev_info(&pdev->dev, 7472 "PHY reset is blocked due to SOL/IDER session.\n"); 7473 7474 /* Set initial default active device features */ 7475 netdev->features = (NETIF_F_SG | 7476 NETIF_F_HW_VLAN_CTAG_RX | 7477 NETIF_F_HW_VLAN_CTAG_TX | 7478 NETIF_F_TSO | 7479 NETIF_F_TSO6 | 7480 NETIF_F_RXHASH | 7481 NETIF_F_RXCSUM | 7482 NETIF_F_HW_CSUM); 7483 7484 /* Set user-changeable features (subset of all device features) */ 7485 netdev->hw_features = netdev->features; 7486 netdev->hw_features |= NETIF_F_RXFCS; 7487 netdev->priv_flags |= IFF_SUPP_NOFCS; 7488 netdev->hw_features |= NETIF_F_RXALL; 7489 7490 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) 7491 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7492 7493 netdev->vlan_features |= (NETIF_F_SG | 7494 NETIF_F_TSO | 7495 NETIF_F_TSO6 | 7496 NETIF_F_HW_CSUM); 7497 7498 netdev->priv_flags |= IFF_UNICAST_FLT; 7499 7500 if (pci_using_dac) { 7501 netdev->features |= NETIF_F_HIGHDMA; 7502 netdev->vlan_features |= NETIF_F_HIGHDMA; 7503 } 7504 7505 /* MTU range: 68 - max_hw_frame_size */ 7506 netdev->min_mtu = ETH_MIN_MTU; 7507 netdev->max_mtu = adapter->max_hw_frame_size - 7508 (VLAN_ETH_HLEN + ETH_FCS_LEN); 7509 7510 if (e1000e_enable_mng_pass_thru(&adapter->hw)) 7511 adapter->flags |= FLAG_MNG_PT_ENABLED; 7512 7513 /* before reading the NVM, reset the controller to 7514 * put the device in a known good starting state 7515 */ 7516 adapter->hw.mac.ops.reset_hw(&adapter->hw); 7517 7518 /* systems with ASPM and others may see the checksum fail on the first 7519 * attempt. Let's give it a few tries 7520 */ 7521 for (i = 0;; i++) { 7522 if (e1000_validate_nvm_checksum(&adapter->hw) >= 0) 7523 break; 7524 if (i == 2) { 7525 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 7526 err = -EIO; 7527 goto err_eeprom; 7528 } 7529 } 7530 7531 e1000_eeprom_checks(adapter); 7532 7533 /* copy the MAC address */ 7534 if (e1000e_read_mac_addr(&adapter->hw)) 7535 dev_err(&pdev->dev, 7536 "NVM Read Error while reading MAC address\n"); 7537 7538 memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len); 7539 7540 if (!is_valid_ether_addr(netdev->dev_addr)) { 7541 dev_err(&pdev->dev, "Invalid MAC Address: %pM\n", 7542 netdev->dev_addr); 7543 err = -EIO; 7544 goto err_eeprom; 7545 } 7546 7547 timer_setup(&adapter->watchdog_timer, e1000_watchdog, 0); 7548 timer_setup(&adapter->phy_info_timer, e1000_update_phy_info, 0); 7549 7550 INIT_WORK(&adapter->reset_task, e1000_reset_task); 7551 INIT_WORK(&adapter->watchdog_task, e1000_watchdog_task); 7552 INIT_WORK(&adapter->downshift_task, e1000e_downshift_workaround); 7553 INIT_WORK(&adapter->update_phy_task, e1000e_update_phy_task); 7554 INIT_WORK(&adapter->print_hang_task, e1000_print_hw_hang); 7555 7556 /* Initialize link parameters. User can change them with ethtool */ 7557 adapter->hw.mac.autoneg = 1; 7558 adapter->fc_autoneg = true; 7559 adapter->hw.fc.requested_mode = e1000_fc_default; 7560 adapter->hw.fc.current_mode = e1000_fc_default; 7561 adapter->hw.phy.autoneg_advertised = 0x2f; 7562 7563 /* Initial Wake on LAN setting - If APM wake is enabled in 7564 * the EEPROM, enable the ACPI Magic Packet filter 7565 */ 7566 if (adapter->flags & FLAG_APME_IN_WUC) { 7567 /* APME bit in EEPROM is mapped to WUC.APME */ 7568 eeprom_data = er32(WUC); 7569 eeprom_apme_mask = E1000_WUC_APME; 7570 if ((hw->mac.type > e1000_ich10lan) && 7571 (eeprom_data & E1000_WUC_PHY_WAKE)) 7572 adapter->flags2 |= FLAG2_HAS_PHY_WAKEUP; 7573 } else if (adapter->flags & FLAG_APME_IN_CTRL3) { 7574 if (adapter->flags & FLAG_APME_CHECK_PORT_B && 7575 (adapter->hw.bus.func == 1)) 7576 ret_val = e1000_read_nvm(&adapter->hw, 7577 NVM_INIT_CONTROL3_PORT_B, 7578 1, &eeprom_data); 7579 else 7580 ret_val = e1000_read_nvm(&adapter->hw, 7581 NVM_INIT_CONTROL3_PORT_A, 7582 1, &eeprom_data); 7583 } 7584 7585 /* fetch WoL from EEPROM */ 7586 if (ret_val) 7587 e_dbg("NVM read error getting WoL initial values: %d\n", ret_val); 7588 else if (eeprom_data & eeprom_apme_mask) 7589 adapter->eeprom_wol |= E1000_WUFC_MAG; 7590 7591 /* now that we have the eeprom settings, apply the special cases 7592 * where the eeprom may be wrong or the board simply won't support 7593 * wake on lan on a particular port 7594 */ 7595 if (!(adapter->flags & FLAG_HAS_WOL)) 7596 adapter->eeprom_wol = 0; 7597 7598 /* initialize the wol settings based on the eeprom settings */ 7599 adapter->wol = adapter->eeprom_wol; 7600 7601 /* make sure adapter isn't asleep if manageability is enabled */ 7602 if (adapter->wol || (adapter->flags & FLAG_MNG_PT_ENABLED) || 7603 (hw->mac.ops.check_mng_mode(hw))) 7604 device_wakeup_enable(&pdev->dev); 7605 7606 /* save off EEPROM version number */ 7607 ret_val = e1000_read_nvm(&adapter->hw, 5, 1, &adapter->eeprom_vers); 7608 7609 if (ret_val) { 7610 e_dbg("NVM read error getting EEPROM version: %d\n", ret_val); 7611 adapter->eeprom_vers = 0; 7612 } 7613 7614 /* init PTP hardware clock */ 7615 e1000e_ptp_init(adapter); 7616 7617 /* reset the hardware with the new settings */ 7618 e1000e_reset(adapter); 7619 7620 /* If the controller has AMT, do not set DRV_LOAD until the interface 7621 * is up. For all other cases, let the f/w know that the h/w is now 7622 * under the control of the driver. 7623 */ 7624 if (!(adapter->flags & FLAG_HAS_AMT)) 7625 e1000e_get_hw_control(adapter); 7626 7627 if (hw->mac.type >= e1000_pch_cnp) 7628 adapter->flags2 |= FLAG2_ENABLE_S0IX_FLOWS; 7629 7630 strlcpy(netdev->name, "eth%d", sizeof(netdev->name)); 7631 err = register_netdev(netdev); 7632 if (err) 7633 goto err_register; 7634 7635 /* carrier off reporting is important to ethtool even BEFORE open */ 7636 netif_carrier_off(netdev); 7637 7638 e1000_print_device_info(adapter); 7639 7640 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_SMART_PREPARE); 7641 7642 if (pci_dev_run_wake(pdev) && hw->mac.type != e1000_pch_cnp) 7643 pm_runtime_put_noidle(&pdev->dev); 7644 7645 return 0; 7646 7647 err_register: 7648 if (!(adapter->flags & FLAG_HAS_AMT)) 7649 e1000e_release_hw_control(adapter); 7650 err_eeprom: 7651 if (hw->phy.ops.check_reset_block && !hw->phy.ops.check_reset_block(hw)) 7652 e1000_phy_hw_reset(&adapter->hw); 7653 err_hw_init: 7654 kfree(adapter->tx_ring); 7655 kfree(adapter->rx_ring); 7656 err_sw_init: 7657 if ((adapter->hw.flash_address) && (hw->mac.type < e1000_pch_spt)) 7658 iounmap(adapter->hw.flash_address); 7659 e1000e_reset_interrupt_capability(adapter); 7660 err_flashmap: 7661 iounmap(adapter->hw.hw_addr); 7662 err_ioremap: 7663 free_netdev(netdev); 7664 err_alloc_etherdev: 7665 pci_release_mem_regions(pdev); 7666 err_pci_reg: 7667 err_dma: 7668 pci_disable_device(pdev); 7669 return err; 7670 } 7671 7672 /** 7673 * e1000_remove - Device Removal Routine 7674 * @pdev: PCI device information struct 7675 * 7676 * e1000_remove is called by the PCI subsystem to alert the driver 7677 * that it should release a PCI device. The could be caused by a 7678 * Hot-Plug event, or because the driver is going to be removed from 7679 * memory. 7680 **/ 7681 static void e1000_remove(struct pci_dev *pdev) 7682 { 7683 struct net_device *netdev = pci_get_drvdata(pdev); 7684 struct e1000_adapter *adapter = netdev_priv(netdev); 7685 7686 e1000e_ptp_remove(adapter); 7687 7688 /* The timers may be rescheduled, so explicitly disable them 7689 * from being rescheduled. 7690 */ 7691 set_bit(__E1000_DOWN, &adapter->state); 7692 del_timer_sync(&adapter->watchdog_timer); 7693 del_timer_sync(&adapter->phy_info_timer); 7694 7695 cancel_work_sync(&adapter->reset_task); 7696 cancel_work_sync(&adapter->watchdog_task); 7697 cancel_work_sync(&adapter->downshift_task); 7698 cancel_work_sync(&adapter->update_phy_task); 7699 cancel_work_sync(&adapter->print_hang_task); 7700 7701 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) { 7702 cancel_work_sync(&adapter->tx_hwtstamp_work); 7703 if (adapter->tx_hwtstamp_skb) { 7704 dev_consume_skb_any(adapter->tx_hwtstamp_skb); 7705 adapter->tx_hwtstamp_skb = NULL; 7706 } 7707 } 7708 7709 unregister_netdev(netdev); 7710 7711 if (pci_dev_run_wake(pdev)) 7712 pm_runtime_get_noresume(&pdev->dev); 7713 7714 /* Release control of h/w to f/w. If f/w is AMT enabled, this 7715 * would have already happened in close and is redundant. 7716 */ 7717 e1000e_release_hw_control(adapter); 7718 7719 e1000e_reset_interrupt_capability(adapter); 7720 kfree(adapter->tx_ring); 7721 kfree(adapter->rx_ring); 7722 7723 iounmap(adapter->hw.hw_addr); 7724 if ((adapter->hw.flash_address) && 7725 (adapter->hw.mac.type < e1000_pch_spt)) 7726 iounmap(adapter->hw.flash_address); 7727 pci_release_mem_regions(pdev); 7728 7729 free_netdev(netdev); 7730 7731 /* AER disable */ 7732 pci_disable_pcie_error_reporting(pdev); 7733 7734 pci_disable_device(pdev); 7735 } 7736 7737 /* PCI Error Recovery (ERS) */ 7738 static const struct pci_error_handlers e1000_err_handler = { 7739 .error_detected = e1000_io_error_detected, 7740 .slot_reset = e1000_io_slot_reset, 7741 .resume = e1000_io_resume, 7742 }; 7743 7744 static const struct pci_device_id e1000_pci_tbl[] = { 7745 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER), board_82571 }, 7746 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER), board_82571 }, 7747 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER), board_82571 }, 7748 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP), 7749 board_82571 }, 7750 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER), board_82571 }, 7751 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES), board_82571 }, 7752 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL), board_82571 }, 7753 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD), board_82571 }, 7754 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER), board_82571 }, 7755 7756 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI), board_82572 }, 7757 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER), board_82572 }, 7758 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER), board_82572 }, 7759 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES), board_82572 }, 7760 7761 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E), board_82573 }, 7762 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT), board_82573 }, 7763 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L), board_82573 }, 7764 7765 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L), board_82574 }, 7766 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA), board_82574 }, 7767 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V), board_82583 }, 7768 7769 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT), 7770 board_80003es2lan }, 7771 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT), 7772 board_80003es2lan }, 7773 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT), 7774 board_80003es2lan }, 7775 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT), 7776 board_80003es2lan }, 7777 7778 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE), board_ich8lan }, 7779 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G), board_ich8lan }, 7780 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT), board_ich8lan }, 7781 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT), board_ich8lan }, 7782 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C), board_ich8lan }, 7783 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M), board_ich8lan }, 7784 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT), board_ich8lan }, 7785 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3), board_ich8lan }, 7786 7787 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE), board_ich9lan }, 7788 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G), board_ich9lan }, 7789 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT), board_ich9lan }, 7790 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT), board_ich9lan }, 7791 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C), board_ich9lan }, 7792 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM), board_ich9lan }, 7793 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M), board_ich9lan }, 7794 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT), board_ich9lan }, 7795 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V), board_ich9lan }, 7796 7797 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM), board_ich9lan }, 7798 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF), board_ich9lan }, 7799 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V), board_ich9lan }, 7800 7801 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM), board_ich10lan }, 7802 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF), board_ich10lan }, 7803 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V), board_ich10lan }, 7804 7805 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM), board_pchlan }, 7806 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC), board_pchlan }, 7807 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM), board_pchlan }, 7808 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC), board_pchlan }, 7809 7810 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM), board_pch2lan }, 7811 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V), board_pch2lan }, 7812 7813 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_LM), board_pch_lpt }, 7814 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_V), board_pch_lpt }, 7815 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_LM), board_pch_lpt }, 7816 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_V), board_pch_lpt }, 7817 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM2), board_pch_lpt }, 7818 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V2), board_pch_lpt }, 7819 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM3), board_pch_lpt }, 7820 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V3), board_pch_lpt }, 7821 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM), board_pch_spt }, 7822 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V), board_pch_spt }, 7823 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM2), board_pch_spt }, 7824 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V2), board_pch_spt }, 7825 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LBG_I219_LM3), board_pch_spt }, 7826 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM4), board_pch_spt }, 7827 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V4), board_pch_spt }, 7828 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM5), board_pch_spt }, 7829 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V5), board_pch_spt }, 7830 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM6), board_pch_cnp }, 7831 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V6), board_pch_cnp }, 7832 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM7), board_pch_cnp }, 7833 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V7), board_pch_cnp }, 7834 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM8), board_pch_cnp }, 7835 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V8), board_pch_cnp }, 7836 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM9), board_pch_cnp }, 7837 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V9), board_pch_cnp }, 7838 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM10), board_pch_cnp }, 7839 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V10), board_pch_cnp }, 7840 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM11), board_pch_cnp }, 7841 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V11), board_pch_cnp }, 7842 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM12), board_pch_spt }, 7843 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V12), board_pch_spt }, 7844 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM13), board_pch_cnp }, 7845 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V13), board_pch_cnp }, 7846 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM14), board_pch_cnp }, 7847 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V14), board_pch_cnp }, 7848 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM15), board_pch_cnp }, 7849 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V15), board_pch_cnp }, 7850 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM16), board_pch_cnp }, 7851 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V16), board_pch_cnp }, 7852 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM17), board_pch_cnp }, 7853 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V17), board_pch_cnp }, 7854 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM18), board_pch_cnp }, 7855 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V18), board_pch_cnp }, 7856 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM19), board_pch_cnp }, 7857 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V19), board_pch_cnp }, 7858 7859 { 0, 0, 0, 0, 0, 0, 0 } /* terminate list */ 7860 }; 7861 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl); 7862 7863 static const struct dev_pm_ops e1000_pm_ops = { 7864 #ifdef CONFIG_PM_SLEEP 7865 .prepare = e1000e_pm_prepare, 7866 .suspend = e1000e_pm_suspend, 7867 .resume = e1000e_pm_resume, 7868 .freeze = e1000e_pm_freeze, 7869 .thaw = e1000e_pm_thaw, 7870 .poweroff = e1000e_pm_suspend, 7871 .restore = e1000e_pm_resume, 7872 #endif 7873 SET_RUNTIME_PM_OPS(e1000e_pm_runtime_suspend, e1000e_pm_runtime_resume, 7874 e1000e_pm_runtime_idle) 7875 }; 7876 7877 /* PCI Device API Driver */ 7878 static struct pci_driver e1000_driver = { 7879 .name = e1000e_driver_name, 7880 .id_table = e1000_pci_tbl, 7881 .probe = e1000_probe, 7882 .remove = e1000_remove, 7883 .driver = { 7884 .pm = &e1000_pm_ops, 7885 }, 7886 .shutdown = e1000_shutdown, 7887 .err_handler = &e1000_err_handler 7888 }; 7889 7890 /** 7891 * e1000_init_module - Driver Registration Routine 7892 * 7893 * e1000_init_module is the first routine called when the driver is 7894 * loaded. All it does is register with the PCI subsystem. 7895 **/ 7896 static int __init e1000_init_module(void) 7897 { 7898 pr_info("Intel(R) PRO/1000 Network Driver\n"); 7899 pr_info("Copyright(c) 1999 - 2015 Intel Corporation.\n"); 7900 7901 return pci_register_driver(&e1000_driver); 7902 } 7903 module_init(e1000_init_module); 7904 7905 /** 7906 * e1000_exit_module - Driver Exit Cleanup Routine 7907 * 7908 * e1000_exit_module is called just before the driver is removed 7909 * from memory. 7910 **/ 7911 static void __exit e1000_exit_module(void) 7912 { 7913 pci_unregister_driver(&e1000_driver); 7914 } 7915 module_exit(e1000_exit_module); 7916 7917 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 7918 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver"); 7919 MODULE_LICENSE("GPL v2"); 7920 7921 /* netdev.c */ 7922