1 /******************************************************************************* 2 3 Intel 10 Gigabit PCI Express Linux driver 4 Copyright(c) 1999 - 2013 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 26 *******************************************************************************/ 27 28 #include <linux/types.h> 29 #include <linux/module.h> 30 #include <linux/pci.h> 31 #include <linux/netdevice.h> 32 #include <linux/vmalloc.h> 33 #include <linux/string.h> 34 #include <linux/in.h> 35 #include <linux/interrupt.h> 36 #include <linux/ip.h> 37 #include <linux/tcp.h> 38 #include <linux/sctp.h> 39 #include <linux/pkt_sched.h> 40 #include <linux/ipv6.h> 41 #include <linux/slab.h> 42 #include <net/checksum.h> 43 #include <net/ip6_checksum.h> 44 #include <linux/ethtool.h> 45 #include <linux/if.h> 46 #include <linux/if_vlan.h> 47 #include <linux/if_bridge.h> 48 #include <linux/prefetch.h> 49 #include <scsi/fc/fc_fcoe.h> 50 51 #include "ixgbe.h" 52 #include "ixgbe_common.h" 53 #include "ixgbe_dcb_82599.h" 54 #include "ixgbe_sriov.h" 55 56 char ixgbe_driver_name[] = "ixgbe"; 57 static const char ixgbe_driver_string[] = 58 "Intel(R) 10 Gigabit PCI Express Network Driver"; 59 #ifdef IXGBE_FCOE 60 char ixgbe_default_device_descr[] = 61 "Intel(R) 10 Gigabit Network Connection"; 62 #else 63 static char ixgbe_default_device_descr[] = 64 "Intel(R) 10 Gigabit Network Connection"; 65 #endif 66 #define DRV_VERSION "3.15.1-k" 67 const char ixgbe_driver_version[] = DRV_VERSION; 68 static const char ixgbe_copyright[] = 69 "Copyright (c) 1999-2013 Intel Corporation."; 70 71 static const struct ixgbe_info *ixgbe_info_tbl[] = { 72 [board_82598] = &ixgbe_82598_info, 73 [board_82599] = &ixgbe_82599_info, 74 [board_X540] = &ixgbe_X540_info, 75 }; 76 77 /* ixgbe_pci_tbl - PCI Device ID Table 78 * 79 * Wildcard entries (PCI_ANY_ID) should come last 80 * Last entry must be all 0s 81 * 82 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 83 * Class, Class Mask, private data (not used) } 84 */ 85 static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = { 86 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 }, 87 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 }, 88 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 }, 89 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 }, 90 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 }, 91 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 }, 92 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 }, 93 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 }, 94 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 }, 95 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 }, 96 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 }, 97 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 }, 98 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 }, 99 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 }, 100 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 }, 101 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 }, 102 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 }, 103 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 }, 104 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 }, 105 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 }, 106 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 }, 107 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 }, 108 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 }, 109 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 }, 110 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 }, 111 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 }, 112 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP), board_82599 }, 113 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 }, 114 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 }, 115 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 }, 116 /* required last entry */ 117 {0, } 118 }; 119 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl); 120 121 #ifdef CONFIG_IXGBE_DCA 122 static int ixgbe_notify_dca(struct notifier_block *, unsigned long event, 123 void *p); 124 static struct notifier_block dca_notifier = { 125 .notifier_call = ixgbe_notify_dca, 126 .next = NULL, 127 .priority = 0 128 }; 129 #endif 130 131 #ifdef CONFIG_PCI_IOV 132 static unsigned int max_vfs; 133 module_param(max_vfs, uint, 0); 134 MODULE_PARM_DESC(max_vfs, 135 "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63"); 136 #endif /* CONFIG_PCI_IOV */ 137 138 static unsigned int allow_unsupported_sfp; 139 module_param(allow_unsupported_sfp, uint, 0); 140 MODULE_PARM_DESC(allow_unsupported_sfp, 141 "Allow unsupported and untested SFP+ modules on 82599-based adapters"); 142 143 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 144 static int debug = -1; 145 module_param(debug, int, 0); 146 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 147 148 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 149 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver"); 150 MODULE_LICENSE("GPL"); 151 MODULE_VERSION(DRV_VERSION); 152 153 static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter, 154 u32 reg, u16 *value) 155 { 156 int pos = 0; 157 struct pci_dev *parent_dev; 158 struct pci_bus *parent_bus; 159 160 parent_bus = adapter->pdev->bus->parent; 161 if (!parent_bus) 162 return -1; 163 164 parent_dev = parent_bus->self; 165 if (!parent_dev) 166 return -1; 167 168 pos = pci_find_capability(parent_dev, PCI_CAP_ID_EXP); 169 if (!pos) 170 return -1; 171 172 pci_read_config_word(parent_dev, pos + reg, value); 173 return 0; 174 } 175 176 static s32 ixgbe_get_parent_bus_info(struct ixgbe_adapter *adapter) 177 { 178 struct ixgbe_hw *hw = &adapter->hw; 179 u16 link_status = 0; 180 int err; 181 182 hw->bus.type = ixgbe_bus_type_pci_express; 183 184 /* Get the negotiated link width and speed from PCI config space of the 185 * parent, as this device is behind a switch 186 */ 187 err = ixgbe_read_pci_cfg_word_parent(adapter, 18, &link_status); 188 189 /* assume caller will handle error case */ 190 if (err) 191 return err; 192 193 hw->bus.width = ixgbe_convert_bus_width(link_status); 194 hw->bus.speed = ixgbe_convert_bus_speed(link_status); 195 196 return 0; 197 } 198 199 /** 200 * ixgbe_check_from_parent - Determine whether PCIe info should come from parent 201 * @hw: hw specific details 202 * 203 * This function is used by probe to determine whether a device's PCI-Express 204 * bandwidth details should be gathered from the parent bus instead of from the 205 * device. Used to ensure that various locations all have the correct device ID 206 * checks. 207 */ 208 static inline bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw) 209 { 210 switch (hw->device_id) { 211 case IXGBE_DEV_ID_82599_SFP_SF_QP: 212 case IXGBE_DEV_ID_82599_QSFP_SF_QP: 213 return true; 214 default: 215 return false; 216 } 217 } 218 219 static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter, 220 int expected_gts) 221 { 222 int max_gts = 0; 223 enum pci_bus_speed speed = PCI_SPEED_UNKNOWN; 224 enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN; 225 struct pci_dev *pdev; 226 227 /* determine whether to use the the parent device 228 */ 229 if (ixgbe_pcie_from_parent(&adapter->hw)) 230 pdev = adapter->pdev->bus->parent->self; 231 else 232 pdev = adapter->pdev; 233 234 if (pcie_get_minimum_link(pdev, &speed, &width) || 235 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) { 236 e_dev_warn("Unable to determine PCI Express bandwidth.\n"); 237 return; 238 } 239 240 switch (speed) { 241 case PCIE_SPEED_2_5GT: 242 /* 8b/10b encoding reduces max throughput by 20% */ 243 max_gts = 2 * width; 244 break; 245 case PCIE_SPEED_5_0GT: 246 /* 8b/10b encoding reduces max throughput by 20% */ 247 max_gts = 4 * width; 248 break; 249 case PCIE_SPEED_8_0GT: 250 /* 128b/130b encoding only reduces throughput by 1% */ 251 max_gts = 8 * width; 252 break; 253 default: 254 e_dev_warn("Unable to determine PCI Express bandwidth.\n"); 255 return; 256 } 257 258 e_dev_info("PCI Express bandwidth of %dGT/s available\n", 259 max_gts); 260 e_dev_info("(Speed:%s, Width: x%d, Encoding Loss:%s)\n", 261 (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : 262 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : 263 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : 264 "Unknown"), 265 width, 266 (speed == PCIE_SPEED_2_5GT ? "20%" : 267 speed == PCIE_SPEED_5_0GT ? "20%" : 268 speed == PCIE_SPEED_8_0GT ? "N/a" : 269 "Unknown")); 270 271 if (max_gts < expected_gts) { 272 e_dev_warn("This is not sufficient for optimal performance of this card.\n"); 273 e_dev_warn("For optimal performance, at least %dGT/s of bandwidth is required.\n", 274 expected_gts); 275 e_dev_warn("A slot with more lanes and/or higher speed is suggested.\n"); 276 } 277 } 278 279 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter) 280 { 281 if (!test_bit(__IXGBE_DOWN, &adapter->state) && 282 !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) 283 schedule_work(&adapter->service_task); 284 } 285 286 static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter) 287 { 288 BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state)); 289 290 /* flush memory to make sure state is correct before next watchdog */ 291 smp_mb__before_clear_bit(); 292 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state); 293 } 294 295 struct ixgbe_reg_info { 296 u32 ofs; 297 char *name; 298 }; 299 300 static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = { 301 302 /* General Registers */ 303 {IXGBE_CTRL, "CTRL"}, 304 {IXGBE_STATUS, "STATUS"}, 305 {IXGBE_CTRL_EXT, "CTRL_EXT"}, 306 307 /* Interrupt Registers */ 308 {IXGBE_EICR, "EICR"}, 309 310 /* RX Registers */ 311 {IXGBE_SRRCTL(0), "SRRCTL"}, 312 {IXGBE_DCA_RXCTRL(0), "DRXCTL"}, 313 {IXGBE_RDLEN(0), "RDLEN"}, 314 {IXGBE_RDH(0), "RDH"}, 315 {IXGBE_RDT(0), "RDT"}, 316 {IXGBE_RXDCTL(0), "RXDCTL"}, 317 {IXGBE_RDBAL(0), "RDBAL"}, 318 {IXGBE_RDBAH(0), "RDBAH"}, 319 320 /* TX Registers */ 321 {IXGBE_TDBAL(0), "TDBAL"}, 322 {IXGBE_TDBAH(0), "TDBAH"}, 323 {IXGBE_TDLEN(0), "TDLEN"}, 324 {IXGBE_TDH(0), "TDH"}, 325 {IXGBE_TDT(0), "TDT"}, 326 {IXGBE_TXDCTL(0), "TXDCTL"}, 327 328 /* List Terminator */ 329 {} 330 }; 331 332 333 /* 334 * ixgbe_regdump - register printout routine 335 */ 336 static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo) 337 { 338 int i = 0, j = 0; 339 char rname[16]; 340 u32 regs[64]; 341 342 switch (reginfo->ofs) { 343 case IXGBE_SRRCTL(0): 344 for (i = 0; i < 64; i++) 345 regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 346 break; 347 case IXGBE_DCA_RXCTRL(0): 348 for (i = 0; i < 64; i++) 349 regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i)); 350 break; 351 case IXGBE_RDLEN(0): 352 for (i = 0; i < 64; i++) 353 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i)); 354 break; 355 case IXGBE_RDH(0): 356 for (i = 0; i < 64; i++) 357 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i)); 358 break; 359 case IXGBE_RDT(0): 360 for (i = 0; i < 64; i++) 361 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i)); 362 break; 363 case IXGBE_RXDCTL(0): 364 for (i = 0; i < 64; i++) 365 regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 366 break; 367 case IXGBE_RDBAL(0): 368 for (i = 0; i < 64; i++) 369 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i)); 370 break; 371 case IXGBE_RDBAH(0): 372 for (i = 0; i < 64; i++) 373 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i)); 374 break; 375 case IXGBE_TDBAL(0): 376 for (i = 0; i < 64; i++) 377 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i)); 378 break; 379 case IXGBE_TDBAH(0): 380 for (i = 0; i < 64; i++) 381 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i)); 382 break; 383 case IXGBE_TDLEN(0): 384 for (i = 0; i < 64; i++) 385 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i)); 386 break; 387 case IXGBE_TDH(0): 388 for (i = 0; i < 64; i++) 389 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i)); 390 break; 391 case IXGBE_TDT(0): 392 for (i = 0; i < 64; i++) 393 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i)); 394 break; 395 case IXGBE_TXDCTL(0): 396 for (i = 0; i < 64; i++) 397 regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 398 break; 399 default: 400 pr_info("%-15s %08x\n", reginfo->name, 401 IXGBE_READ_REG(hw, reginfo->ofs)); 402 return; 403 } 404 405 for (i = 0; i < 8; i++) { 406 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7); 407 pr_err("%-15s", rname); 408 for (j = 0; j < 8; j++) 409 pr_cont(" %08x", regs[i*8+j]); 410 pr_cont("\n"); 411 } 412 413 } 414 415 /* 416 * ixgbe_dump - Print registers, tx-rings and rx-rings 417 */ 418 static void ixgbe_dump(struct ixgbe_adapter *adapter) 419 { 420 struct net_device *netdev = adapter->netdev; 421 struct ixgbe_hw *hw = &adapter->hw; 422 struct ixgbe_reg_info *reginfo; 423 int n = 0; 424 struct ixgbe_ring *tx_ring; 425 struct ixgbe_tx_buffer *tx_buffer; 426 union ixgbe_adv_tx_desc *tx_desc; 427 struct my_u0 { u64 a; u64 b; } *u0; 428 struct ixgbe_ring *rx_ring; 429 union ixgbe_adv_rx_desc *rx_desc; 430 struct ixgbe_rx_buffer *rx_buffer_info; 431 u32 staterr; 432 int i = 0; 433 434 if (!netif_msg_hw(adapter)) 435 return; 436 437 /* Print netdevice Info */ 438 if (netdev) { 439 dev_info(&adapter->pdev->dev, "Net device Info\n"); 440 pr_info("Device Name state " 441 "trans_start last_rx\n"); 442 pr_info("%-15s %016lX %016lX %016lX\n", 443 netdev->name, 444 netdev->state, 445 netdev->trans_start, 446 netdev->last_rx); 447 } 448 449 /* Print Registers */ 450 dev_info(&adapter->pdev->dev, "Register Dump\n"); 451 pr_info(" Register Name Value\n"); 452 for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl; 453 reginfo->name; reginfo++) { 454 ixgbe_regdump(hw, reginfo); 455 } 456 457 /* Print TX Ring Summary */ 458 if (!netdev || !netif_running(netdev)) 459 goto exit; 460 461 dev_info(&adapter->pdev->dev, "TX Rings Summary\n"); 462 pr_info(" %s %s %s %s\n", 463 "Queue [NTU] [NTC] [bi(ntc)->dma ]", 464 "leng", "ntw", "timestamp"); 465 for (n = 0; n < adapter->num_tx_queues; n++) { 466 tx_ring = adapter->tx_ring[n]; 467 tx_buffer = &tx_ring->tx_buffer_info[tx_ring->next_to_clean]; 468 pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n", 469 n, tx_ring->next_to_use, tx_ring->next_to_clean, 470 (u64)dma_unmap_addr(tx_buffer, dma), 471 dma_unmap_len(tx_buffer, len), 472 tx_buffer->next_to_watch, 473 (u64)tx_buffer->time_stamp); 474 } 475 476 /* Print TX Rings */ 477 if (!netif_msg_tx_done(adapter)) 478 goto rx_ring_summary; 479 480 dev_info(&adapter->pdev->dev, "TX Rings Dump\n"); 481 482 /* Transmit Descriptor Formats 483 * 484 * 82598 Advanced Transmit Descriptor 485 * +--------------------------------------------------------------+ 486 * 0 | Buffer Address [63:0] | 487 * +--------------------------------------------------------------+ 488 * 8 | PAYLEN | POPTS | IDX | STA | DCMD |DTYP | RSV | DTALEN | 489 * +--------------------------------------------------------------+ 490 * 63 46 45 40 39 36 35 32 31 24 23 20 19 0 491 * 492 * 82598 Advanced Transmit Descriptor (Write-Back Format) 493 * +--------------------------------------------------------------+ 494 * 0 | RSV [63:0] | 495 * +--------------------------------------------------------------+ 496 * 8 | RSV | STA | NXTSEQ | 497 * +--------------------------------------------------------------+ 498 * 63 36 35 32 31 0 499 * 500 * 82599+ Advanced Transmit Descriptor 501 * +--------------------------------------------------------------+ 502 * 0 | Buffer Address [63:0] | 503 * +--------------------------------------------------------------+ 504 * 8 |PAYLEN |POPTS|CC|IDX |STA |DCMD |DTYP |MAC |RSV |DTALEN | 505 * +--------------------------------------------------------------+ 506 * 63 46 45 40 39 38 36 35 32 31 24 23 20 19 18 17 16 15 0 507 * 508 * 82599+ Advanced Transmit Descriptor (Write-Back Format) 509 * +--------------------------------------------------------------+ 510 * 0 | RSV [63:0] | 511 * +--------------------------------------------------------------+ 512 * 8 | RSV | STA | RSV | 513 * +--------------------------------------------------------------+ 514 * 63 36 35 32 31 0 515 */ 516 517 for (n = 0; n < adapter->num_tx_queues; n++) { 518 tx_ring = adapter->tx_ring[n]; 519 pr_info("------------------------------------\n"); 520 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index); 521 pr_info("------------------------------------\n"); 522 pr_info("%s%s %s %s %s %s\n", 523 "T [desc] [address 63:0 ] ", 524 "[PlPOIdStDDt Ln] [bi->dma ] ", 525 "leng", "ntw", "timestamp", "bi->skb"); 526 527 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { 528 tx_desc = IXGBE_TX_DESC(tx_ring, i); 529 tx_buffer = &tx_ring->tx_buffer_info[i]; 530 u0 = (struct my_u0 *)tx_desc; 531 if (dma_unmap_len(tx_buffer, len) > 0) { 532 pr_info("T [0x%03X] %016llX %016llX %016llX %08X %p %016llX %p", 533 i, 534 le64_to_cpu(u0->a), 535 le64_to_cpu(u0->b), 536 (u64)dma_unmap_addr(tx_buffer, dma), 537 dma_unmap_len(tx_buffer, len), 538 tx_buffer->next_to_watch, 539 (u64)tx_buffer->time_stamp, 540 tx_buffer->skb); 541 if (i == tx_ring->next_to_use && 542 i == tx_ring->next_to_clean) 543 pr_cont(" NTC/U\n"); 544 else if (i == tx_ring->next_to_use) 545 pr_cont(" NTU\n"); 546 else if (i == tx_ring->next_to_clean) 547 pr_cont(" NTC\n"); 548 else 549 pr_cont("\n"); 550 551 if (netif_msg_pktdata(adapter) && 552 tx_buffer->skb) 553 print_hex_dump(KERN_INFO, "", 554 DUMP_PREFIX_ADDRESS, 16, 1, 555 tx_buffer->skb->data, 556 dma_unmap_len(tx_buffer, len), 557 true); 558 } 559 } 560 } 561 562 /* Print RX Rings Summary */ 563 rx_ring_summary: 564 dev_info(&adapter->pdev->dev, "RX Rings Summary\n"); 565 pr_info("Queue [NTU] [NTC]\n"); 566 for (n = 0; n < adapter->num_rx_queues; n++) { 567 rx_ring = adapter->rx_ring[n]; 568 pr_info("%5d %5X %5X\n", 569 n, rx_ring->next_to_use, rx_ring->next_to_clean); 570 } 571 572 /* Print RX Rings */ 573 if (!netif_msg_rx_status(adapter)) 574 goto exit; 575 576 dev_info(&adapter->pdev->dev, "RX Rings Dump\n"); 577 578 /* Receive Descriptor Formats 579 * 580 * 82598 Advanced Receive Descriptor (Read) Format 581 * 63 1 0 582 * +-----------------------------------------------------+ 583 * 0 | Packet Buffer Address [63:1] |A0/NSE| 584 * +----------------------------------------------+------+ 585 * 8 | Header Buffer Address [63:1] | DD | 586 * +-----------------------------------------------------+ 587 * 588 * 589 * 82598 Advanced Receive Descriptor (Write-Back) Format 590 * 591 * 63 48 47 32 31 30 21 20 16 15 4 3 0 592 * +------------------------------------------------------+ 593 * 0 | RSS Hash / |SPH| HDR_LEN | RSV |Packet| RSS | 594 * | Packet | IP | | | | Type | Type | 595 * | Checksum | Ident | | | | | | 596 * +------------------------------------------------------+ 597 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 598 * +------------------------------------------------------+ 599 * 63 48 47 32 31 20 19 0 600 * 601 * 82599+ Advanced Receive Descriptor (Read) Format 602 * 63 1 0 603 * +-----------------------------------------------------+ 604 * 0 | Packet Buffer Address [63:1] |A0/NSE| 605 * +----------------------------------------------+------+ 606 * 8 | Header Buffer Address [63:1] | DD | 607 * +-----------------------------------------------------+ 608 * 609 * 610 * 82599+ Advanced Receive Descriptor (Write-Back) Format 611 * 612 * 63 48 47 32 31 30 21 20 17 16 4 3 0 613 * +------------------------------------------------------+ 614 * 0 |RSS / Frag Checksum|SPH| HDR_LEN |RSC- |Packet| RSS | 615 * |/ RTT / PCoE_PARAM | | | CNT | Type | Type | 616 * |/ Flow Dir Flt ID | | | | | | 617 * +------------------------------------------------------+ 618 * 8 | VLAN Tag | Length |Extended Error| Xtnd Status/NEXTP | 619 * +------------------------------------------------------+ 620 * 63 48 47 32 31 20 19 0 621 */ 622 623 for (n = 0; n < adapter->num_rx_queues; n++) { 624 rx_ring = adapter->rx_ring[n]; 625 pr_info("------------------------------------\n"); 626 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); 627 pr_info("------------------------------------\n"); 628 pr_info("%s%s%s", 629 "R [desc] [ PktBuf A0] ", 630 "[ HeadBuf DD] [bi->dma ] [bi->skb ] ", 631 "<-- Adv Rx Read format\n"); 632 pr_info("%s%s%s", 633 "RWB[desc] [PcsmIpSHl PtRs] ", 634 "[vl er S cks ln] ---------------- [bi->skb ] ", 635 "<-- Adv Rx Write-Back format\n"); 636 637 for (i = 0; i < rx_ring->count; i++) { 638 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 639 rx_desc = IXGBE_RX_DESC(rx_ring, i); 640 u0 = (struct my_u0 *)rx_desc; 641 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 642 if (staterr & IXGBE_RXD_STAT_DD) { 643 /* Descriptor Done */ 644 pr_info("RWB[0x%03X] %016llX " 645 "%016llX ---------------- %p", i, 646 le64_to_cpu(u0->a), 647 le64_to_cpu(u0->b), 648 rx_buffer_info->skb); 649 } else { 650 pr_info("R [0x%03X] %016llX " 651 "%016llX %016llX %p", i, 652 le64_to_cpu(u0->a), 653 le64_to_cpu(u0->b), 654 (u64)rx_buffer_info->dma, 655 rx_buffer_info->skb); 656 657 if (netif_msg_pktdata(adapter) && 658 rx_buffer_info->dma) { 659 print_hex_dump(KERN_INFO, "", 660 DUMP_PREFIX_ADDRESS, 16, 1, 661 page_address(rx_buffer_info->page) + 662 rx_buffer_info->page_offset, 663 ixgbe_rx_bufsz(rx_ring), true); 664 } 665 } 666 667 if (i == rx_ring->next_to_use) 668 pr_cont(" NTU\n"); 669 else if (i == rx_ring->next_to_clean) 670 pr_cont(" NTC\n"); 671 else 672 pr_cont("\n"); 673 674 } 675 } 676 677 exit: 678 return; 679 } 680 681 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter) 682 { 683 u32 ctrl_ext; 684 685 /* Let firmware take over control of h/w */ 686 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 687 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, 688 ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD); 689 } 690 691 static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter) 692 { 693 u32 ctrl_ext; 694 695 /* Let firmware know the driver has taken over */ 696 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 697 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, 698 ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD); 699 } 700 701 /** 702 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors 703 * @adapter: pointer to adapter struct 704 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 705 * @queue: queue to map the corresponding interrupt to 706 * @msix_vector: the vector to map to the corresponding queue 707 * 708 */ 709 static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction, 710 u8 queue, u8 msix_vector) 711 { 712 u32 ivar, index; 713 struct ixgbe_hw *hw = &adapter->hw; 714 switch (hw->mac.type) { 715 case ixgbe_mac_82598EB: 716 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 717 if (direction == -1) 718 direction = 0; 719 index = (((direction * 64) + queue) >> 2) & 0x1F; 720 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 721 ivar &= ~(0xFF << (8 * (queue & 0x3))); 722 ivar |= (msix_vector << (8 * (queue & 0x3))); 723 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 724 break; 725 case ixgbe_mac_82599EB: 726 case ixgbe_mac_X540: 727 if (direction == -1) { 728 /* other causes */ 729 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 730 index = ((queue & 1) * 8); 731 ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC); 732 ivar &= ~(0xFF << index); 733 ivar |= (msix_vector << index); 734 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar); 735 break; 736 } else { 737 /* tx or rx causes */ 738 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 739 index = ((16 * (queue & 1)) + (8 * direction)); 740 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1)); 741 ivar &= ~(0xFF << index); 742 ivar |= (msix_vector << index); 743 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar); 744 break; 745 } 746 default: 747 break; 748 } 749 } 750 751 static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter, 752 u64 qmask) 753 { 754 u32 mask; 755 756 switch (adapter->hw.mac.type) { 757 case ixgbe_mac_82598EB: 758 mask = (IXGBE_EIMS_RTX_QUEUE & qmask); 759 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 760 break; 761 case ixgbe_mac_82599EB: 762 case ixgbe_mac_X540: 763 mask = (qmask & 0xFFFFFFFF); 764 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 765 mask = (qmask >> 32); 766 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 767 break; 768 default: 769 break; 770 } 771 } 772 773 void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *ring, 774 struct ixgbe_tx_buffer *tx_buffer) 775 { 776 if (tx_buffer->skb) { 777 dev_kfree_skb_any(tx_buffer->skb); 778 if (dma_unmap_len(tx_buffer, len)) 779 dma_unmap_single(ring->dev, 780 dma_unmap_addr(tx_buffer, dma), 781 dma_unmap_len(tx_buffer, len), 782 DMA_TO_DEVICE); 783 } else if (dma_unmap_len(tx_buffer, len)) { 784 dma_unmap_page(ring->dev, 785 dma_unmap_addr(tx_buffer, dma), 786 dma_unmap_len(tx_buffer, len), 787 DMA_TO_DEVICE); 788 } 789 tx_buffer->next_to_watch = NULL; 790 tx_buffer->skb = NULL; 791 dma_unmap_len_set(tx_buffer, len, 0); 792 /* tx_buffer must be completely set up in the transmit path */ 793 } 794 795 static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter) 796 { 797 struct ixgbe_hw *hw = &adapter->hw; 798 struct ixgbe_hw_stats *hwstats = &adapter->stats; 799 int i; 800 u32 data; 801 802 if ((hw->fc.current_mode != ixgbe_fc_full) && 803 (hw->fc.current_mode != ixgbe_fc_rx_pause)) 804 return; 805 806 switch (hw->mac.type) { 807 case ixgbe_mac_82598EB: 808 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 809 break; 810 default: 811 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 812 } 813 hwstats->lxoffrxc += data; 814 815 /* refill credits (no tx hang) if we received xoff */ 816 if (!data) 817 return; 818 819 for (i = 0; i < adapter->num_tx_queues; i++) 820 clear_bit(__IXGBE_HANG_CHECK_ARMED, 821 &adapter->tx_ring[i]->state); 822 } 823 824 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter) 825 { 826 struct ixgbe_hw *hw = &adapter->hw; 827 struct ixgbe_hw_stats *hwstats = &adapter->stats; 828 u32 xoff[8] = {0}; 829 u8 tc; 830 int i; 831 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable; 832 833 if (adapter->ixgbe_ieee_pfc) 834 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en); 835 836 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) { 837 ixgbe_update_xoff_rx_lfc(adapter); 838 return; 839 } 840 841 /* update stats for each tc, only valid with PFC enabled */ 842 for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) { 843 u32 pxoffrxc; 844 845 switch (hw->mac.type) { 846 case ixgbe_mac_82598EB: 847 pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 848 break; 849 default: 850 pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i)); 851 } 852 hwstats->pxoffrxc[i] += pxoffrxc; 853 /* Get the TC for given UP */ 854 tc = netdev_get_prio_tc_map(adapter->netdev, i); 855 xoff[tc] += pxoffrxc; 856 } 857 858 /* disarm tx queues that have received xoff frames */ 859 for (i = 0; i < adapter->num_tx_queues; i++) { 860 struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; 861 862 tc = tx_ring->dcb_tc; 863 if (xoff[tc]) 864 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); 865 } 866 } 867 868 static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring) 869 { 870 return ring->stats.packets; 871 } 872 873 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring) 874 { 875 struct ixgbe_adapter *adapter = netdev_priv(ring->netdev); 876 struct ixgbe_hw *hw = &adapter->hw; 877 878 u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx)); 879 u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx)); 880 881 if (head != tail) 882 return (head < tail) ? 883 tail - head : (tail + ring->count - head); 884 885 return 0; 886 } 887 888 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring) 889 { 890 u32 tx_done = ixgbe_get_tx_completed(tx_ring); 891 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 892 u32 tx_pending = ixgbe_get_tx_pending(tx_ring); 893 bool ret = false; 894 895 clear_check_for_tx_hang(tx_ring); 896 897 /* 898 * Check for a hung queue, but be thorough. This verifies 899 * that a transmit has been completed since the previous 900 * check AND there is at least one packet pending. The 901 * ARMED bit is set to indicate a potential hang. The 902 * bit is cleared if a pause frame is received to remove 903 * false hang detection due to PFC or 802.3x frames. By 904 * requiring this to fail twice we avoid races with 905 * pfc clearing the ARMED bit and conditions where we 906 * run the check_tx_hang logic with a transmit completion 907 * pending but without time to complete it yet. 908 */ 909 if ((tx_done_old == tx_done) && tx_pending) { 910 /* make sure it is true for two checks in a row */ 911 ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED, 912 &tx_ring->state); 913 } else { 914 /* update completed stats and continue */ 915 tx_ring->tx_stats.tx_done_old = tx_done; 916 /* reset the countdown */ 917 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); 918 } 919 920 return ret; 921 } 922 923 /** 924 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout 925 * @adapter: driver private struct 926 **/ 927 static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter) 928 { 929 930 /* Do the reset outside of interrupt context */ 931 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 932 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED; 933 e_warn(drv, "initiating reset due to tx timeout\n"); 934 ixgbe_service_event_schedule(adapter); 935 } 936 } 937 938 /** 939 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes 940 * @q_vector: structure containing interrupt and ring information 941 * @tx_ring: tx ring to clean 942 **/ 943 static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, 944 struct ixgbe_ring *tx_ring) 945 { 946 struct ixgbe_adapter *adapter = q_vector->adapter; 947 struct ixgbe_tx_buffer *tx_buffer; 948 union ixgbe_adv_tx_desc *tx_desc; 949 unsigned int total_bytes = 0, total_packets = 0; 950 unsigned int budget = q_vector->tx.work_limit; 951 unsigned int i = tx_ring->next_to_clean; 952 953 if (test_bit(__IXGBE_DOWN, &adapter->state)) 954 return true; 955 956 tx_buffer = &tx_ring->tx_buffer_info[i]; 957 tx_desc = IXGBE_TX_DESC(tx_ring, i); 958 i -= tx_ring->count; 959 960 do { 961 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 962 963 /* if next_to_watch is not set then there is no work pending */ 964 if (!eop_desc) 965 break; 966 967 /* prevent any other reads prior to eop_desc */ 968 read_barrier_depends(); 969 970 /* if DD is not set pending work has not been completed */ 971 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 972 break; 973 974 /* clear next_to_watch to prevent false hangs */ 975 tx_buffer->next_to_watch = NULL; 976 977 /* update the statistics for this packet */ 978 total_bytes += tx_buffer->bytecount; 979 total_packets += tx_buffer->gso_segs; 980 981 /* free the skb */ 982 dev_kfree_skb_any(tx_buffer->skb); 983 984 /* unmap skb header data */ 985 dma_unmap_single(tx_ring->dev, 986 dma_unmap_addr(tx_buffer, dma), 987 dma_unmap_len(tx_buffer, len), 988 DMA_TO_DEVICE); 989 990 /* clear tx_buffer data */ 991 tx_buffer->skb = NULL; 992 dma_unmap_len_set(tx_buffer, len, 0); 993 994 /* unmap remaining buffers */ 995 while (tx_desc != eop_desc) { 996 tx_buffer++; 997 tx_desc++; 998 i++; 999 if (unlikely(!i)) { 1000 i -= tx_ring->count; 1001 tx_buffer = tx_ring->tx_buffer_info; 1002 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 1003 } 1004 1005 /* unmap any remaining paged data */ 1006 if (dma_unmap_len(tx_buffer, len)) { 1007 dma_unmap_page(tx_ring->dev, 1008 dma_unmap_addr(tx_buffer, dma), 1009 dma_unmap_len(tx_buffer, len), 1010 DMA_TO_DEVICE); 1011 dma_unmap_len_set(tx_buffer, len, 0); 1012 } 1013 } 1014 1015 /* move us one more past the eop_desc for start of next pkt */ 1016 tx_buffer++; 1017 tx_desc++; 1018 i++; 1019 if (unlikely(!i)) { 1020 i -= tx_ring->count; 1021 tx_buffer = tx_ring->tx_buffer_info; 1022 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 1023 } 1024 1025 /* issue prefetch for next Tx descriptor */ 1026 prefetch(tx_desc); 1027 1028 /* update budget accounting */ 1029 budget--; 1030 } while (likely(budget)); 1031 1032 i += tx_ring->count; 1033 tx_ring->next_to_clean = i; 1034 u64_stats_update_begin(&tx_ring->syncp); 1035 tx_ring->stats.bytes += total_bytes; 1036 tx_ring->stats.packets += total_packets; 1037 u64_stats_update_end(&tx_ring->syncp); 1038 q_vector->tx.total_bytes += total_bytes; 1039 q_vector->tx.total_packets += total_packets; 1040 1041 if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) { 1042 /* schedule immediate reset if we believe we hung */ 1043 struct ixgbe_hw *hw = &adapter->hw; 1044 e_err(drv, "Detected Tx Unit Hang\n" 1045 " Tx Queue <%d>\n" 1046 " TDH, TDT <%x>, <%x>\n" 1047 " next_to_use <%x>\n" 1048 " next_to_clean <%x>\n" 1049 "tx_buffer_info[next_to_clean]\n" 1050 " time_stamp <%lx>\n" 1051 " jiffies <%lx>\n", 1052 tx_ring->queue_index, 1053 IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), 1054 IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), 1055 tx_ring->next_to_use, i, 1056 tx_ring->tx_buffer_info[i].time_stamp, jiffies); 1057 1058 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 1059 1060 e_info(probe, 1061 "tx hang %d detected on queue %d, resetting adapter\n", 1062 adapter->tx_timeout_count + 1, tx_ring->queue_index); 1063 1064 /* schedule immediate reset if we believe we hung */ 1065 ixgbe_tx_timeout_reset(adapter); 1066 1067 /* the adapter is about to reset, no point in enabling stuff */ 1068 return true; 1069 } 1070 1071 netdev_tx_completed_queue(txring_txq(tx_ring), 1072 total_packets, total_bytes); 1073 1074 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 1075 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 1076 (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 1077 /* Make sure that anybody stopping the queue after this 1078 * sees the new next_to_clean. 1079 */ 1080 smp_mb(); 1081 if (__netif_subqueue_stopped(tx_ring->netdev, 1082 tx_ring->queue_index) 1083 && !test_bit(__IXGBE_DOWN, &adapter->state)) { 1084 netif_wake_subqueue(tx_ring->netdev, 1085 tx_ring->queue_index); 1086 ++tx_ring->tx_stats.restart_queue; 1087 } 1088 } 1089 1090 return !!budget; 1091 } 1092 1093 #ifdef CONFIG_IXGBE_DCA 1094 static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter, 1095 struct ixgbe_ring *tx_ring, 1096 int cpu) 1097 { 1098 struct ixgbe_hw *hw = &adapter->hw; 1099 u32 txctrl = dca3_get_tag(tx_ring->dev, cpu); 1100 u16 reg_offset; 1101 1102 switch (hw->mac.type) { 1103 case ixgbe_mac_82598EB: 1104 reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx); 1105 break; 1106 case ixgbe_mac_82599EB: 1107 case ixgbe_mac_X540: 1108 reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx); 1109 txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599; 1110 break; 1111 default: 1112 /* for unknown hardware do not write register */ 1113 return; 1114 } 1115 1116 /* 1117 * We can enable relaxed ordering for reads, but not writes when 1118 * DCA is enabled. This is due to a known issue in some chipsets 1119 * which will cause the DCA tag to be cleared. 1120 */ 1121 txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN | 1122 IXGBE_DCA_TXCTRL_DATA_RRO_EN | 1123 IXGBE_DCA_TXCTRL_DESC_DCA_EN; 1124 1125 IXGBE_WRITE_REG(hw, reg_offset, txctrl); 1126 } 1127 1128 static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter, 1129 struct ixgbe_ring *rx_ring, 1130 int cpu) 1131 { 1132 struct ixgbe_hw *hw = &adapter->hw; 1133 u32 rxctrl = dca3_get_tag(rx_ring->dev, cpu); 1134 u8 reg_idx = rx_ring->reg_idx; 1135 1136 1137 switch (hw->mac.type) { 1138 case ixgbe_mac_82599EB: 1139 case ixgbe_mac_X540: 1140 rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599; 1141 break; 1142 default: 1143 break; 1144 } 1145 1146 /* 1147 * We can enable relaxed ordering for reads, but not writes when 1148 * DCA is enabled. This is due to a known issue in some chipsets 1149 * which will cause the DCA tag to be cleared. 1150 */ 1151 rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN | 1152 IXGBE_DCA_RXCTRL_DESC_DCA_EN; 1153 1154 IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl); 1155 } 1156 1157 static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector) 1158 { 1159 struct ixgbe_adapter *adapter = q_vector->adapter; 1160 struct ixgbe_ring *ring; 1161 int cpu = get_cpu(); 1162 1163 if (q_vector->cpu == cpu) 1164 goto out_no_update; 1165 1166 ixgbe_for_each_ring(ring, q_vector->tx) 1167 ixgbe_update_tx_dca(adapter, ring, cpu); 1168 1169 ixgbe_for_each_ring(ring, q_vector->rx) 1170 ixgbe_update_rx_dca(adapter, ring, cpu); 1171 1172 q_vector->cpu = cpu; 1173 out_no_update: 1174 put_cpu(); 1175 } 1176 1177 static void ixgbe_setup_dca(struct ixgbe_adapter *adapter) 1178 { 1179 int i; 1180 1181 if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED)) 1182 return; 1183 1184 /* always use CB2 mode, difference is masked in the CB driver */ 1185 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2); 1186 1187 for (i = 0; i < adapter->num_q_vectors; i++) { 1188 adapter->q_vector[i]->cpu = -1; 1189 ixgbe_update_dca(adapter->q_vector[i]); 1190 } 1191 } 1192 1193 static int __ixgbe_notify_dca(struct device *dev, void *data) 1194 { 1195 struct ixgbe_adapter *adapter = dev_get_drvdata(dev); 1196 unsigned long event = *(unsigned long *)data; 1197 1198 if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE)) 1199 return 0; 1200 1201 switch (event) { 1202 case DCA_PROVIDER_ADD: 1203 /* if we're already enabled, don't do it again */ 1204 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) 1205 break; 1206 if (dca_add_requester(dev) == 0) { 1207 adapter->flags |= IXGBE_FLAG_DCA_ENABLED; 1208 ixgbe_setup_dca(adapter); 1209 break; 1210 } 1211 /* Fall Through since DCA is disabled. */ 1212 case DCA_PROVIDER_REMOVE: 1213 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) { 1214 dca_remove_requester(dev); 1215 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED; 1216 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1); 1217 } 1218 break; 1219 } 1220 1221 return 0; 1222 } 1223 1224 #endif /* CONFIG_IXGBE_DCA */ 1225 static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, 1226 union ixgbe_adv_rx_desc *rx_desc, 1227 struct sk_buff *skb) 1228 { 1229 if (ring->netdev->features & NETIF_F_RXHASH) 1230 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss); 1231 } 1232 1233 #ifdef IXGBE_FCOE 1234 /** 1235 * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type 1236 * @ring: structure containing ring specific data 1237 * @rx_desc: advanced rx descriptor 1238 * 1239 * Returns : true if it is FCoE pkt 1240 */ 1241 static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring, 1242 union ixgbe_adv_rx_desc *rx_desc) 1243 { 1244 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; 1245 1246 return test_bit(__IXGBE_RX_FCOE, &ring->state) && 1247 ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == 1248 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE << 1249 IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); 1250 } 1251 1252 #endif /* IXGBE_FCOE */ 1253 /** 1254 * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum 1255 * @ring: structure containing ring specific data 1256 * @rx_desc: current Rx descriptor being processed 1257 * @skb: skb currently being received and modified 1258 **/ 1259 static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, 1260 union ixgbe_adv_rx_desc *rx_desc, 1261 struct sk_buff *skb) 1262 { 1263 skb_checksum_none_assert(skb); 1264 1265 /* Rx csum disabled */ 1266 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1267 return; 1268 1269 /* if IP and error */ 1270 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 1271 ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 1272 ring->rx_stats.csum_err++; 1273 return; 1274 } 1275 1276 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 1277 return; 1278 1279 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 1280 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; 1281 1282 /* 1283 * 82599 errata, UDP frames with a 0 checksum can be marked as 1284 * checksum errors. 1285 */ 1286 if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) && 1287 test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) 1288 return; 1289 1290 ring->rx_stats.csum_err++; 1291 return; 1292 } 1293 1294 /* It must be a TCP or UDP packet with a valid checksum */ 1295 skb->ip_summed = CHECKSUM_UNNECESSARY; 1296 } 1297 1298 static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val) 1299 { 1300 rx_ring->next_to_use = val; 1301 1302 /* update next to alloc since we have filled the ring */ 1303 rx_ring->next_to_alloc = val; 1304 /* 1305 * Force memory writes to complete before letting h/w 1306 * know there are new descriptors to fetch. (Only 1307 * applicable for weak-ordered memory model archs, 1308 * such as IA-64). 1309 */ 1310 wmb(); 1311 writel(val, rx_ring->tail); 1312 } 1313 1314 static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, 1315 struct ixgbe_rx_buffer *bi) 1316 { 1317 struct page *page = bi->page; 1318 dma_addr_t dma = bi->dma; 1319 1320 /* since we are recycling buffers we should seldom need to alloc */ 1321 if (likely(dma)) 1322 return true; 1323 1324 /* alloc new page for storage */ 1325 if (likely(!page)) { 1326 page = __skb_alloc_pages(GFP_ATOMIC | __GFP_COLD | __GFP_COMP, 1327 bi->skb, ixgbe_rx_pg_order(rx_ring)); 1328 if (unlikely(!page)) { 1329 rx_ring->rx_stats.alloc_rx_page_failed++; 1330 return false; 1331 } 1332 bi->page = page; 1333 } 1334 1335 /* map page for use */ 1336 dma = dma_map_page(rx_ring->dev, page, 0, 1337 ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE); 1338 1339 /* 1340 * if mapping failed free memory back to system since 1341 * there isn't much point in holding memory we can't use 1342 */ 1343 if (dma_mapping_error(rx_ring->dev, dma)) { 1344 __free_pages(page, ixgbe_rx_pg_order(rx_ring)); 1345 bi->page = NULL; 1346 1347 rx_ring->rx_stats.alloc_rx_page_failed++; 1348 return false; 1349 } 1350 1351 bi->dma = dma; 1352 bi->page_offset = 0; 1353 1354 return true; 1355 } 1356 1357 /** 1358 * ixgbe_alloc_rx_buffers - Replace used receive buffers 1359 * @rx_ring: ring to place buffers on 1360 * @cleaned_count: number of buffers to replace 1361 **/ 1362 void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) 1363 { 1364 union ixgbe_adv_rx_desc *rx_desc; 1365 struct ixgbe_rx_buffer *bi; 1366 u16 i = rx_ring->next_to_use; 1367 1368 /* nothing to do */ 1369 if (!cleaned_count) 1370 return; 1371 1372 rx_desc = IXGBE_RX_DESC(rx_ring, i); 1373 bi = &rx_ring->rx_buffer_info[i]; 1374 i -= rx_ring->count; 1375 1376 do { 1377 if (!ixgbe_alloc_mapped_page(rx_ring, bi)) 1378 break; 1379 1380 /* 1381 * Refresh the desc even if buffer_addrs didn't change 1382 * because each write-back erases this info. 1383 */ 1384 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1385 1386 rx_desc++; 1387 bi++; 1388 i++; 1389 if (unlikely(!i)) { 1390 rx_desc = IXGBE_RX_DESC(rx_ring, 0); 1391 bi = rx_ring->rx_buffer_info; 1392 i -= rx_ring->count; 1393 } 1394 1395 /* clear the hdr_addr for the next_to_use descriptor */ 1396 rx_desc->read.hdr_addr = 0; 1397 1398 cleaned_count--; 1399 } while (cleaned_count); 1400 1401 i += rx_ring->count; 1402 1403 if (rx_ring->next_to_use != i) 1404 ixgbe_release_rx_desc(rx_ring, i); 1405 } 1406 1407 /** 1408 * ixgbe_get_headlen - determine size of header for RSC/LRO/GRO/FCOE 1409 * @data: pointer to the start of the headers 1410 * @max_len: total length of section to find headers in 1411 * 1412 * This function is meant to determine the length of headers that will 1413 * be recognized by hardware for LRO, GRO, and RSC offloads. The main 1414 * motivation of doing this is to only perform one pull for IPv4 TCP 1415 * packets so that we can do basic things like calculating the gso_size 1416 * based on the average data per packet. 1417 **/ 1418 static unsigned int ixgbe_get_headlen(unsigned char *data, 1419 unsigned int max_len) 1420 { 1421 union { 1422 unsigned char *network; 1423 /* l2 headers */ 1424 struct ethhdr *eth; 1425 struct vlan_hdr *vlan; 1426 /* l3 headers */ 1427 struct iphdr *ipv4; 1428 struct ipv6hdr *ipv6; 1429 } hdr; 1430 __be16 protocol; 1431 u8 nexthdr = 0; /* default to not TCP */ 1432 u8 hlen; 1433 1434 /* this should never happen, but better safe than sorry */ 1435 if (max_len < ETH_HLEN) 1436 return max_len; 1437 1438 /* initialize network frame pointer */ 1439 hdr.network = data; 1440 1441 /* set first protocol and move network header forward */ 1442 protocol = hdr.eth->h_proto; 1443 hdr.network += ETH_HLEN; 1444 1445 /* handle any vlan tag if present */ 1446 if (protocol == __constant_htons(ETH_P_8021Q)) { 1447 if ((hdr.network - data) > (max_len - VLAN_HLEN)) 1448 return max_len; 1449 1450 protocol = hdr.vlan->h_vlan_encapsulated_proto; 1451 hdr.network += VLAN_HLEN; 1452 } 1453 1454 /* handle L3 protocols */ 1455 if (protocol == __constant_htons(ETH_P_IP)) { 1456 if ((hdr.network - data) > (max_len - sizeof(struct iphdr))) 1457 return max_len; 1458 1459 /* access ihl as a u8 to avoid unaligned access on ia64 */ 1460 hlen = (hdr.network[0] & 0x0F) << 2; 1461 1462 /* verify hlen meets minimum size requirements */ 1463 if (hlen < sizeof(struct iphdr)) 1464 return hdr.network - data; 1465 1466 /* record next protocol if header is present */ 1467 if (!(hdr.ipv4->frag_off & htons(IP_OFFSET))) 1468 nexthdr = hdr.ipv4->protocol; 1469 } else if (protocol == __constant_htons(ETH_P_IPV6)) { 1470 if ((hdr.network - data) > (max_len - sizeof(struct ipv6hdr))) 1471 return max_len; 1472 1473 /* record next protocol */ 1474 nexthdr = hdr.ipv6->nexthdr; 1475 hlen = sizeof(struct ipv6hdr); 1476 #ifdef IXGBE_FCOE 1477 } else if (protocol == __constant_htons(ETH_P_FCOE)) { 1478 if ((hdr.network - data) > (max_len - FCOE_HEADER_LEN)) 1479 return max_len; 1480 hlen = FCOE_HEADER_LEN; 1481 #endif 1482 } else { 1483 return hdr.network - data; 1484 } 1485 1486 /* relocate pointer to start of L4 header */ 1487 hdr.network += hlen; 1488 1489 /* finally sort out TCP/UDP */ 1490 if (nexthdr == IPPROTO_TCP) { 1491 if ((hdr.network - data) > (max_len - sizeof(struct tcphdr))) 1492 return max_len; 1493 1494 /* access doff as a u8 to avoid unaligned access on ia64 */ 1495 hlen = (hdr.network[12] & 0xF0) >> 2; 1496 1497 /* verify hlen meets minimum size requirements */ 1498 if (hlen < sizeof(struct tcphdr)) 1499 return hdr.network - data; 1500 1501 hdr.network += hlen; 1502 } else if (nexthdr == IPPROTO_UDP) { 1503 if ((hdr.network - data) > (max_len - sizeof(struct udphdr))) 1504 return max_len; 1505 1506 hdr.network += sizeof(struct udphdr); 1507 } 1508 1509 /* 1510 * If everything has gone correctly hdr.network should be the 1511 * data section of the packet and will be the end of the header. 1512 * If not then it probably represents the end of the last recognized 1513 * header. 1514 */ 1515 if ((hdr.network - data) < max_len) 1516 return hdr.network - data; 1517 else 1518 return max_len; 1519 } 1520 1521 static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring, 1522 struct sk_buff *skb) 1523 { 1524 u16 hdr_len = skb_headlen(skb); 1525 1526 /* set gso_size to avoid messing up TCP MSS */ 1527 skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len), 1528 IXGBE_CB(skb)->append_cnt); 1529 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; 1530 } 1531 1532 static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring, 1533 struct sk_buff *skb) 1534 { 1535 /* if append_cnt is 0 then frame is not RSC */ 1536 if (!IXGBE_CB(skb)->append_cnt) 1537 return; 1538 1539 rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt; 1540 rx_ring->rx_stats.rsc_flush++; 1541 1542 ixgbe_set_rsc_gso_size(rx_ring, skb); 1543 1544 /* gso_size is computed using append_cnt so always clear it last */ 1545 IXGBE_CB(skb)->append_cnt = 0; 1546 } 1547 1548 /** 1549 * ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor 1550 * @rx_ring: rx descriptor ring packet is being transacted on 1551 * @rx_desc: pointer to the EOP Rx descriptor 1552 * @skb: pointer to current skb being populated 1553 * 1554 * This function checks the ring, descriptor, and packet information in 1555 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 1556 * other fields within the skb. 1557 **/ 1558 static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, 1559 union ixgbe_adv_rx_desc *rx_desc, 1560 struct sk_buff *skb) 1561 { 1562 struct net_device *dev = rx_ring->netdev; 1563 1564 ixgbe_update_rsc_stats(rx_ring, skb); 1565 1566 ixgbe_rx_hash(rx_ring, rx_desc, skb); 1567 1568 ixgbe_rx_checksum(rx_ring, rx_desc, skb); 1569 1570 ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb); 1571 1572 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && 1573 ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 1574 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 1575 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 1576 } 1577 1578 skb_record_rx_queue(skb, rx_ring->queue_index); 1579 1580 skb->protocol = eth_type_trans(skb, dev); 1581 } 1582 1583 static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, 1584 struct sk_buff *skb) 1585 { 1586 struct ixgbe_adapter *adapter = q_vector->adapter; 1587 1588 if (ixgbe_qv_ll_polling(q_vector)) 1589 netif_receive_skb(skb); 1590 else if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL)) 1591 napi_gro_receive(&q_vector->napi, skb); 1592 else 1593 netif_rx(skb); 1594 } 1595 1596 /** 1597 * ixgbe_is_non_eop - process handling of non-EOP buffers 1598 * @rx_ring: Rx ring being processed 1599 * @rx_desc: Rx descriptor for current buffer 1600 * @skb: Current socket buffer containing buffer in progress 1601 * 1602 * This function updates next to clean. If the buffer is an EOP buffer 1603 * this function exits returning false, otherwise it will place the 1604 * sk_buff in the next buffer to be chained and return true indicating 1605 * that this is in fact a non-EOP buffer. 1606 **/ 1607 static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, 1608 union ixgbe_adv_rx_desc *rx_desc, 1609 struct sk_buff *skb) 1610 { 1611 u32 ntc = rx_ring->next_to_clean + 1; 1612 1613 /* fetch, update, and store next to clean */ 1614 ntc = (ntc < rx_ring->count) ? ntc : 0; 1615 rx_ring->next_to_clean = ntc; 1616 1617 prefetch(IXGBE_RX_DESC(rx_ring, ntc)); 1618 1619 /* update RSC append count if present */ 1620 if (ring_is_rsc_enabled(rx_ring)) { 1621 __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data & 1622 cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK); 1623 1624 if (unlikely(rsc_enabled)) { 1625 u32 rsc_cnt = le32_to_cpu(rsc_enabled); 1626 1627 rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT; 1628 IXGBE_CB(skb)->append_cnt += rsc_cnt - 1; 1629 1630 /* update ntc based on RSC value */ 1631 ntc = le32_to_cpu(rx_desc->wb.upper.status_error); 1632 ntc &= IXGBE_RXDADV_NEXTP_MASK; 1633 ntc >>= IXGBE_RXDADV_NEXTP_SHIFT; 1634 } 1635 } 1636 1637 /* if we are the last buffer then there is nothing else to do */ 1638 if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 1639 return false; 1640 1641 /* place skb in next buffer to be received */ 1642 rx_ring->rx_buffer_info[ntc].skb = skb; 1643 rx_ring->rx_stats.non_eop_descs++; 1644 1645 return true; 1646 } 1647 1648 /** 1649 * ixgbe_pull_tail - ixgbe specific version of skb_pull_tail 1650 * @rx_ring: rx descriptor ring packet is being transacted on 1651 * @skb: pointer to current skb being adjusted 1652 * 1653 * This function is an ixgbe specific version of __pskb_pull_tail. The 1654 * main difference between this version and the original function is that 1655 * this function can make several assumptions about the state of things 1656 * that allow for significant optimizations versus the standard function. 1657 * As a result we can do things like drop a frag and maintain an accurate 1658 * truesize for the skb. 1659 */ 1660 static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, 1661 struct sk_buff *skb) 1662 { 1663 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 1664 unsigned char *va; 1665 unsigned int pull_len; 1666 1667 /* 1668 * it is valid to use page_address instead of kmap since we are 1669 * working with pages allocated out of the lomem pool per 1670 * alloc_page(GFP_ATOMIC) 1671 */ 1672 va = skb_frag_address(frag); 1673 1674 /* 1675 * we need the header to contain the greater of either ETH_HLEN or 1676 * 60 bytes if the skb->len is less than 60 for skb_pad. 1677 */ 1678 pull_len = ixgbe_get_headlen(va, IXGBE_RX_HDR_SIZE); 1679 1680 /* align pull length to size of long to optimize memcpy performance */ 1681 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); 1682 1683 /* update all of the pointers */ 1684 skb_frag_size_sub(frag, pull_len); 1685 frag->page_offset += pull_len; 1686 skb->data_len -= pull_len; 1687 skb->tail += pull_len; 1688 } 1689 1690 /** 1691 * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB 1692 * @rx_ring: rx descriptor ring packet is being transacted on 1693 * @skb: pointer to current skb being updated 1694 * 1695 * This function provides a basic DMA sync up for the first fragment of an 1696 * skb. The reason for doing this is that the first fragment cannot be 1697 * unmapped until we have reached the end of packet descriptor for a buffer 1698 * chain. 1699 */ 1700 static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring, 1701 struct sk_buff *skb) 1702 { 1703 /* if the page was released unmap it, else just sync our portion */ 1704 if (unlikely(IXGBE_CB(skb)->page_released)) { 1705 dma_unmap_page(rx_ring->dev, IXGBE_CB(skb)->dma, 1706 ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE); 1707 IXGBE_CB(skb)->page_released = false; 1708 } else { 1709 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 1710 1711 dma_sync_single_range_for_cpu(rx_ring->dev, 1712 IXGBE_CB(skb)->dma, 1713 frag->page_offset, 1714 ixgbe_rx_bufsz(rx_ring), 1715 DMA_FROM_DEVICE); 1716 } 1717 IXGBE_CB(skb)->dma = 0; 1718 } 1719 1720 /** 1721 * ixgbe_cleanup_headers - Correct corrupted or empty headers 1722 * @rx_ring: rx descriptor ring packet is being transacted on 1723 * @rx_desc: pointer to the EOP Rx descriptor 1724 * @skb: pointer to current skb being fixed 1725 * 1726 * Check for corrupted packet headers caused by senders on the local L2 1727 * embedded NIC switch not setting up their Tx Descriptors right. These 1728 * should be very rare. 1729 * 1730 * Also address the case where we are pulling data in on pages only 1731 * and as such no data is present in the skb header. 1732 * 1733 * In addition if skb is not at least 60 bytes we need to pad it so that 1734 * it is large enough to qualify as a valid Ethernet frame. 1735 * 1736 * Returns true if an error was encountered and skb was freed. 1737 **/ 1738 static bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, 1739 union ixgbe_adv_rx_desc *rx_desc, 1740 struct sk_buff *skb) 1741 { 1742 struct net_device *netdev = rx_ring->netdev; 1743 1744 /* verify that the packet does not have any known errors */ 1745 if (unlikely(ixgbe_test_staterr(rx_desc, 1746 IXGBE_RXDADV_ERR_FRAME_ERR_MASK) && 1747 !(netdev->features & NETIF_F_RXALL))) { 1748 dev_kfree_skb_any(skb); 1749 return true; 1750 } 1751 1752 /* place header in linear portion of buffer */ 1753 if (skb_is_nonlinear(skb)) 1754 ixgbe_pull_tail(rx_ring, skb); 1755 1756 #ifdef IXGBE_FCOE 1757 /* do not attempt to pad FCoE Frames as this will disrupt DDP */ 1758 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) 1759 return false; 1760 1761 #endif 1762 /* if skb_pad returns an error the skb was freed */ 1763 if (unlikely(skb->len < 60)) { 1764 int pad_len = 60 - skb->len; 1765 1766 if (skb_pad(skb, pad_len)) 1767 return true; 1768 __skb_put(skb, pad_len); 1769 } 1770 1771 return false; 1772 } 1773 1774 /** 1775 * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring 1776 * @rx_ring: rx descriptor ring to store buffers on 1777 * @old_buff: donor buffer to have page reused 1778 * 1779 * Synchronizes page for reuse by the adapter 1780 **/ 1781 static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring, 1782 struct ixgbe_rx_buffer *old_buff) 1783 { 1784 struct ixgbe_rx_buffer *new_buff; 1785 u16 nta = rx_ring->next_to_alloc; 1786 1787 new_buff = &rx_ring->rx_buffer_info[nta]; 1788 1789 /* update, and store next to alloc */ 1790 nta++; 1791 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1792 1793 /* transfer page from old buffer to new buffer */ 1794 new_buff->page = old_buff->page; 1795 new_buff->dma = old_buff->dma; 1796 new_buff->page_offset = old_buff->page_offset; 1797 1798 /* sync the buffer for use by the device */ 1799 dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma, 1800 new_buff->page_offset, 1801 ixgbe_rx_bufsz(rx_ring), 1802 DMA_FROM_DEVICE); 1803 } 1804 1805 /** 1806 * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff 1807 * @rx_ring: rx descriptor ring to transact packets on 1808 * @rx_buffer: buffer containing page to add 1809 * @rx_desc: descriptor containing length of buffer written by hardware 1810 * @skb: sk_buff to place the data into 1811 * 1812 * This function will add the data contained in rx_buffer->page to the skb. 1813 * This is done either through a direct copy if the data in the buffer is 1814 * less than the skb header size, otherwise it will just attach the page as 1815 * a frag to the skb. 1816 * 1817 * The function will then update the page offset if necessary and return 1818 * true if the buffer can be reused by the adapter. 1819 **/ 1820 static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, 1821 struct ixgbe_rx_buffer *rx_buffer, 1822 union ixgbe_adv_rx_desc *rx_desc, 1823 struct sk_buff *skb) 1824 { 1825 struct page *page = rx_buffer->page; 1826 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length); 1827 #if (PAGE_SIZE < 8192) 1828 unsigned int truesize = ixgbe_rx_bufsz(rx_ring); 1829 #else 1830 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES); 1831 unsigned int last_offset = ixgbe_rx_pg_size(rx_ring) - 1832 ixgbe_rx_bufsz(rx_ring); 1833 #endif 1834 1835 if ((size <= IXGBE_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) { 1836 unsigned char *va = page_address(page) + rx_buffer->page_offset; 1837 1838 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 1839 1840 /* we can reuse buffer as-is, just make sure it is local */ 1841 if (likely(page_to_nid(page) == numa_node_id())) 1842 return true; 1843 1844 /* this page cannot be reused so discard it */ 1845 put_page(page); 1846 return false; 1847 } 1848 1849 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 1850 rx_buffer->page_offset, size, truesize); 1851 1852 /* avoid re-using remote pages */ 1853 if (unlikely(page_to_nid(page) != numa_node_id())) 1854 return false; 1855 1856 #if (PAGE_SIZE < 8192) 1857 /* if we are only owner of page we can reuse it */ 1858 if (unlikely(page_count(page) != 1)) 1859 return false; 1860 1861 /* flip page offset to other buffer */ 1862 rx_buffer->page_offset ^= truesize; 1863 1864 /* 1865 * since we are the only owner of the page and we need to 1866 * increment it, just set the value to 2 in order to avoid 1867 * an unecessary locked operation 1868 */ 1869 atomic_set(&page->_count, 2); 1870 #else 1871 /* move offset up to the next cache line */ 1872 rx_buffer->page_offset += truesize; 1873 1874 if (rx_buffer->page_offset > last_offset) 1875 return false; 1876 1877 /* bump ref count on page before it is given to the stack */ 1878 get_page(page); 1879 #endif 1880 1881 return true; 1882 } 1883 1884 static struct sk_buff *ixgbe_fetch_rx_buffer(struct ixgbe_ring *rx_ring, 1885 union ixgbe_adv_rx_desc *rx_desc) 1886 { 1887 struct ixgbe_rx_buffer *rx_buffer; 1888 struct sk_buff *skb; 1889 struct page *page; 1890 1891 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1892 page = rx_buffer->page; 1893 prefetchw(page); 1894 1895 skb = rx_buffer->skb; 1896 1897 if (likely(!skb)) { 1898 void *page_addr = page_address(page) + 1899 rx_buffer->page_offset; 1900 1901 /* prefetch first cache line of first page */ 1902 prefetch(page_addr); 1903 #if L1_CACHE_BYTES < 128 1904 prefetch(page_addr + L1_CACHE_BYTES); 1905 #endif 1906 1907 /* allocate a skb to store the frags */ 1908 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 1909 IXGBE_RX_HDR_SIZE); 1910 if (unlikely(!skb)) { 1911 rx_ring->rx_stats.alloc_rx_buff_failed++; 1912 return NULL; 1913 } 1914 1915 /* 1916 * we will be copying header into skb->data in 1917 * pskb_may_pull so it is in our interest to prefetch 1918 * it now to avoid a possible cache miss 1919 */ 1920 prefetchw(skb->data); 1921 1922 /* 1923 * Delay unmapping of the first packet. It carries the 1924 * header information, HW may still access the header 1925 * after the writeback. Only unmap it when EOP is 1926 * reached 1927 */ 1928 if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 1929 goto dma_sync; 1930 1931 IXGBE_CB(skb)->dma = rx_buffer->dma; 1932 } else { 1933 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) 1934 ixgbe_dma_sync_frag(rx_ring, skb); 1935 1936 dma_sync: 1937 /* we are reusing so sync this buffer for CPU use */ 1938 dma_sync_single_range_for_cpu(rx_ring->dev, 1939 rx_buffer->dma, 1940 rx_buffer->page_offset, 1941 ixgbe_rx_bufsz(rx_ring), 1942 DMA_FROM_DEVICE); 1943 } 1944 1945 /* pull page into skb */ 1946 if (ixgbe_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) { 1947 /* hand second half of page back to the ring */ 1948 ixgbe_reuse_rx_page(rx_ring, rx_buffer); 1949 } else if (IXGBE_CB(skb)->dma == rx_buffer->dma) { 1950 /* the page has been released from the ring */ 1951 IXGBE_CB(skb)->page_released = true; 1952 } else { 1953 /* we are not reusing the buffer so unmap it */ 1954 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 1955 ixgbe_rx_pg_size(rx_ring), 1956 DMA_FROM_DEVICE); 1957 } 1958 1959 /* clear contents of buffer_info */ 1960 rx_buffer->skb = NULL; 1961 rx_buffer->dma = 0; 1962 rx_buffer->page = NULL; 1963 1964 return skb; 1965 } 1966 1967 /** 1968 * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf 1969 * @q_vector: structure containing interrupt and ring information 1970 * @rx_ring: rx descriptor ring to transact packets on 1971 * @budget: Total limit on number of packets to process 1972 * 1973 * This function provides a "bounce buffer" approach to Rx interrupt 1974 * processing. The advantage to this is that on systems that have 1975 * expensive overhead for IOMMU access this provides a means of avoiding 1976 * it by maintaining the mapping of the page to the syste. 1977 * 1978 * Returns amount of work completed 1979 **/ 1980 static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, 1981 struct ixgbe_ring *rx_ring, 1982 const int budget) 1983 { 1984 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1985 #ifdef IXGBE_FCOE 1986 struct ixgbe_adapter *adapter = q_vector->adapter; 1987 int ddp_bytes; 1988 unsigned int mss = 0; 1989 #endif /* IXGBE_FCOE */ 1990 u16 cleaned_count = ixgbe_desc_unused(rx_ring); 1991 1992 do { 1993 union ixgbe_adv_rx_desc *rx_desc; 1994 struct sk_buff *skb; 1995 1996 /* return some buffers to hardware, one at a time is too slow */ 1997 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) { 1998 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count); 1999 cleaned_count = 0; 2000 } 2001 2002 rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean); 2003 2004 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_DD)) 2005 break; 2006 2007 /* 2008 * This memory barrier is needed to keep us from reading 2009 * any other fields out of the rx_desc until we know the 2010 * RXD_STAT_DD bit is set 2011 */ 2012 rmb(); 2013 2014 /* retrieve a buffer from the ring */ 2015 skb = ixgbe_fetch_rx_buffer(rx_ring, rx_desc); 2016 2017 /* exit if we failed to retrieve a buffer */ 2018 if (!skb) 2019 break; 2020 2021 cleaned_count++; 2022 2023 /* place incomplete frames back on ring for completion */ 2024 if (ixgbe_is_non_eop(rx_ring, rx_desc, skb)) 2025 continue; 2026 2027 /* verify the packet layout is correct */ 2028 if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb)) 2029 continue; 2030 2031 /* probably a little skewed due to removing CRC */ 2032 total_rx_bytes += skb->len; 2033 2034 /* populate checksum, timestamp, VLAN, and protocol */ 2035 ixgbe_process_skb_fields(rx_ring, rx_desc, skb); 2036 2037 #ifdef IXGBE_FCOE 2038 /* if ddp, not passing to ULD unless for FCP_RSP or error */ 2039 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) { 2040 ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb); 2041 /* include DDPed FCoE data */ 2042 if (ddp_bytes > 0) { 2043 if (!mss) { 2044 mss = rx_ring->netdev->mtu - 2045 sizeof(struct fcoe_hdr) - 2046 sizeof(struct fc_frame_header) - 2047 sizeof(struct fcoe_crc_eof); 2048 if (mss > 512) 2049 mss &= ~511; 2050 } 2051 total_rx_bytes += ddp_bytes; 2052 total_rx_packets += DIV_ROUND_UP(ddp_bytes, 2053 mss); 2054 } 2055 if (!ddp_bytes) { 2056 dev_kfree_skb_any(skb); 2057 continue; 2058 } 2059 } 2060 2061 #endif /* IXGBE_FCOE */ 2062 skb_mark_napi_id(skb, &q_vector->napi); 2063 ixgbe_rx_skb(q_vector, skb); 2064 2065 /* update budget accounting */ 2066 total_rx_packets++; 2067 } while (likely(total_rx_packets < budget)); 2068 2069 u64_stats_update_begin(&rx_ring->syncp); 2070 rx_ring->stats.packets += total_rx_packets; 2071 rx_ring->stats.bytes += total_rx_bytes; 2072 u64_stats_update_end(&rx_ring->syncp); 2073 q_vector->rx.total_packets += total_rx_packets; 2074 q_vector->rx.total_bytes += total_rx_bytes; 2075 2076 if (cleaned_count) 2077 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count); 2078 2079 return total_rx_packets; 2080 } 2081 2082 #ifdef CONFIG_NET_RX_BUSY_POLL 2083 /* must be called with local_bh_disable()d */ 2084 static int ixgbe_low_latency_recv(struct napi_struct *napi) 2085 { 2086 struct ixgbe_q_vector *q_vector = 2087 container_of(napi, struct ixgbe_q_vector, napi); 2088 struct ixgbe_adapter *adapter = q_vector->adapter; 2089 struct ixgbe_ring *ring; 2090 int found = 0; 2091 2092 if (test_bit(__IXGBE_DOWN, &adapter->state)) 2093 return LL_FLUSH_FAILED; 2094 2095 if (!ixgbe_qv_lock_poll(q_vector)) 2096 return LL_FLUSH_BUSY; 2097 2098 ixgbe_for_each_ring(ring, q_vector->rx) { 2099 found = ixgbe_clean_rx_irq(q_vector, ring, 4); 2100 #ifdef LL_EXTENDED_STATS 2101 if (found) 2102 ring->stats.cleaned += found; 2103 else 2104 ring->stats.misses++; 2105 #endif 2106 if (found) 2107 break; 2108 } 2109 2110 ixgbe_qv_unlock_poll(q_vector); 2111 2112 return found; 2113 } 2114 #endif /* CONFIG_NET_RX_BUSY_POLL */ 2115 2116 /** 2117 * ixgbe_configure_msix - Configure MSI-X hardware 2118 * @adapter: board private structure 2119 * 2120 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X 2121 * interrupts. 2122 **/ 2123 static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) 2124 { 2125 struct ixgbe_q_vector *q_vector; 2126 int v_idx; 2127 u32 mask; 2128 2129 /* Populate MSIX to EITR Select */ 2130 if (adapter->num_vfs > 32) { 2131 u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1; 2132 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel); 2133 } 2134 2135 /* 2136 * Populate the IVAR table and set the ITR values to the 2137 * corresponding register. 2138 */ 2139 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) { 2140 struct ixgbe_ring *ring; 2141 q_vector = adapter->q_vector[v_idx]; 2142 2143 ixgbe_for_each_ring(ring, q_vector->rx) 2144 ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx); 2145 2146 ixgbe_for_each_ring(ring, q_vector->tx) 2147 ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx); 2148 2149 ixgbe_write_eitr(q_vector); 2150 } 2151 2152 switch (adapter->hw.mac.type) { 2153 case ixgbe_mac_82598EB: 2154 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX, 2155 v_idx); 2156 break; 2157 case ixgbe_mac_82599EB: 2158 case ixgbe_mac_X540: 2159 ixgbe_set_ivar(adapter, -1, 1, v_idx); 2160 break; 2161 default: 2162 break; 2163 } 2164 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950); 2165 2166 /* set up to autoclear timer, and the vectors */ 2167 mask = IXGBE_EIMS_ENABLE_MASK; 2168 mask &= ~(IXGBE_EIMS_OTHER | 2169 IXGBE_EIMS_MAILBOX | 2170 IXGBE_EIMS_LSC); 2171 2172 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask); 2173 } 2174 2175 enum latency_range { 2176 lowest_latency = 0, 2177 low_latency = 1, 2178 bulk_latency = 2, 2179 latency_invalid = 255 2180 }; 2181 2182 /** 2183 * ixgbe_update_itr - update the dynamic ITR value based on statistics 2184 * @q_vector: structure containing interrupt and ring information 2185 * @ring_container: structure containing ring performance data 2186 * 2187 * Stores a new ITR value based on packets and byte 2188 * counts during the last interrupt. The advantage of per interrupt 2189 * computation is faster updates and more accurate ITR for the current 2190 * traffic pattern. Constants in this function were computed 2191 * based on theoretical maximum wire speed and thresholds were set based 2192 * on testing data as well as attempting to minimize response time 2193 * while increasing bulk throughput. 2194 * this functionality is controlled by the InterruptThrottleRate module 2195 * parameter (see ixgbe_param.c) 2196 **/ 2197 static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, 2198 struct ixgbe_ring_container *ring_container) 2199 { 2200 int bytes = ring_container->total_bytes; 2201 int packets = ring_container->total_packets; 2202 u32 timepassed_us; 2203 u64 bytes_perint; 2204 u8 itr_setting = ring_container->itr; 2205 2206 if (packets == 0) 2207 return; 2208 2209 /* simple throttlerate management 2210 * 0-10MB/s lowest (100000 ints/s) 2211 * 10-20MB/s low (20000 ints/s) 2212 * 20-1249MB/s bulk (8000 ints/s) 2213 */ 2214 /* what was last interrupt timeslice? */ 2215 timepassed_us = q_vector->itr >> 2; 2216 if (timepassed_us == 0) 2217 return; 2218 2219 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 2220 2221 switch (itr_setting) { 2222 case lowest_latency: 2223 if (bytes_perint > 10) 2224 itr_setting = low_latency; 2225 break; 2226 case low_latency: 2227 if (bytes_perint > 20) 2228 itr_setting = bulk_latency; 2229 else if (bytes_perint <= 10) 2230 itr_setting = lowest_latency; 2231 break; 2232 case bulk_latency: 2233 if (bytes_perint <= 20) 2234 itr_setting = low_latency; 2235 break; 2236 } 2237 2238 /* clear work counters since we have the values we need */ 2239 ring_container->total_bytes = 0; 2240 ring_container->total_packets = 0; 2241 2242 /* write updated itr to ring container */ 2243 ring_container->itr = itr_setting; 2244 } 2245 2246 /** 2247 * ixgbe_write_eitr - write EITR register in hardware specific way 2248 * @q_vector: structure containing interrupt and ring information 2249 * 2250 * This function is made to be called by ethtool and by the driver 2251 * when it needs to update EITR registers at runtime. Hardware 2252 * specific quirks/differences are taken care of here. 2253 */ 2254 void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector) 2255 { 2256 struct ixgbe_adapter *adapter = q_vector->adapter; 2257 struct ixgbe_hw *hw = &adapter->hw; 2258 int v_idx = q_vector->v_idx; 2259 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 2260 2261 switch (adapter->hw.mac.type) { 2262 case ixgbe_mac_82598EB: 2263 /* must write high and low 16 bits to reset counter */ 2264 itr_reg |= (itr_reg << 16); 2265 break; 2266 case ixgbe_mac_82599EB: 2267 case ixgbe_mac_X540: 2268 /* 2269 * set the WDIS bit to not clear the timer bits and cause an 2270 * immediate assertion of the interrupt 2271 */ 2272 itr_reg |= IXGBE_EITR_CNT_WDIS; 2273 break; 2274 default: 2275 break; 2276 } 2277 IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg); 2278 } 2279 2280 static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector) 2281 { 2282 u32 new_itr = q_vector->itr; 2283 u8 current_itr; 2284 2285 ixgbe_update_itr(q_vector, &q_vector->tx); 2286 ixgbe_update_itr(q_vector, &q_vector->rx); 2287 2288 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 2289 2290 switch (current_itr) { 2291 /* counts and packets in update_itr are dependent on these numbers */ 2292 case lowest_latency: 2293 new_itr = IXGBE_100K_ITR; 2294 break; 2295 case low_latency: 2296 new_itr = IXGBE_20K_ITR; 2297 break; 2298 case bulk_latency: 2299 new_itr = IXGBE_8K_ITR; 2300 break; 2301 default: 2302 break; 2303 } 2304 2305 if (new_itr != q_vector->itr) { 2306 /* do an exponential smoothing */ 2307 new_itr = (10 * new_itr * q_vector->itr) / 2308 ((9 * new_itr) + q_vector->itr); 2309 2310 /* save the algorithm value here */ 2311 q_vector->itr = new_itr; 2312 2313 ixgbe_write_eitr(q_vector); 2314 } 2315 } 2316 2317 /** 2318 * ixgbe_check_overtemp_subtask - check for over temperature 2319 * @adapter: pointer to adapter 2320 **/ 2321 static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter) 2322 { 2323 struct ixgbe_hw *hw = &adapter->hw; 2324 u32 eicr = adapter->interrupt_event; 2325 2326 if (test_bit(__IXGBE_DOWN, &adapter->state)) 2327 return; 2328 2329 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) && 2330 !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT)) 2331 return; 2332 2333 adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT; 2334 2335 switch (hw->device_id) { 2336 case IXGBE_DEV_ID_82599_T3_LOM: 2337 /* 2338 * Since the warning interrupt is for both ports 2339 * we don't have to check if: 2340 * - This interrupt wasn't for our port. 2341 * - We may have missed the interrupt so always have to 2342 * check if we got a LSC 2343 */ 2344 if (!(eicr & IXGBE_EICR_GPI_SDP0) && 2345 !(eicr & IXGBE_EICR_LSC)) 2346 return; 2347 2348 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) { 2349 u32 speed; 2350 bool link_up = false; 2351 2352 hw->mac.ops.check_link(hw, &speed, &link_up, false); 2353 2354 if (link_up) 2355 return; 2356 } 2357 2358 /* Check if this is not due to overtemp */ 2359 if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP) 2360 return; 2361 2362 break; 2363 default: 2364 if (!(eicr & IXGBE_EICR_GPI_SDP0)) 2365 return; 2366 break; 2367 } 2368 e_crit(drv, 2369 "Network adapter has been stopped because it has over heated. " 2370 "Restart the computer. If the problem persists, " 2371 "power off the system and replace the adapter\n"); 2372 2373 adapter->interrupt_event = 0; 2374 } 2375 2376 static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr) 2377 { 2378 struct ixgbe_hw *hw = &adapter->hw; 2379 2380 if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && 2381 (eicr & IXGBE_EICR_GPI_SDP1)) { 2382 e_crit(probe, "Fan has stopped, replace the adapter\n"); 2383 /* write to clear the interrupt */ 2384 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 2385 } 2386 } 2387 2388 static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr) 2389 { 2390 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)) 2391 return; 2392 2393 switch (adapter->hw.mac.type) { 2394 case ixgbe_mac_82599EB: 2395 /* 2396 * Need to check link state so complete overtemp check 2397 * on service task 2398 */ 2399 if (((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC)) && 2400 (!test_bit(__IXGBE_DOWN, &adapter->state))) { 2401 adapter->interrupt_event = eicr; 2402 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT; 2403 ixgbe_service_event_schedule(adapter); 2404 return; 2405 } 2406 return; 2407 case ixgbe_mac_X540: 2408 if (!(eicr & IXGBE_EICR_TS)) 2409 return; 2410 break; 2411 default: 2412 return; 2413 } 2414 2415 e_crit(drv, 2416 "Network adapter has been stopped because it has over heated. " 2417 "Restart the computer. If the problem persists, " 2418 "power off the system and replace the adapter\n"); 2419 } 2420 2421 static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr) 2422 { 2423 struct ixgbe_hw *hw = &adapter->hw; 2424 2425 if (eicr & IXGBE_EICR_GPI_SDP2) { 2426 /* Clear the interrupt */ 2427 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 2428 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 2429 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; 2430 ixgbe_service_event_schedule(adapter); 2431 } 2432 } 2433 2434 if (eicr & IXGBE_EICR_GPI_SDP1) { 2435 /* Clear the interrupt */ 2436 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 2437 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 2438 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; 2439 ixgbe_service_event_schedule(adapter); 2440 } 2441 } 2442 } 2443 2444 static void ixgbe_check_lsc(struct ixgbe_adapter *adapter) 2445 { 2446 struct ixgbe_hw *hw = &adapter->hw; 2447 2448 adapter->lsc_int++; 2449 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 2450 adapter->link_check_timeout = jiffies; 2451 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 2452 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); 2453 IXGBE_WRITE_FLUSH(hw); 2454 ixgbe_service_event_schedule(adapter); 2455 } 2456 } 2457 2458 static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter, 2459 u64 qmask) 2460 { 2461 u32 mask; 2462 struct ixgbe_hw *hw = &adapter->hw; 2463 2464 switch (hw->mac.type) { 2465 case ixgbe_mac_82598EB: 2466 mask = (IXGBE_EIMS_RTX_QUEUE & qmask); 2467 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2468 break; 2469 case ixgbe_mac_82599EB: 2470 case ixgbe_mac_X540: 2471 mask = (qmask & 0xFFFFFFFF); 2472 if (mask) 2473 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 2474 mask = (qmask >> 32); 2475 if (mask) 2476 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 2477 break; 2478 default: 2479 break; 2480 } 2481 /* skip the flush */ 2482 } 2483 2484 static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter, 2485 u64 qmask) 2486 { 2487 u32 mask; 2488 struct ixgbe_hw *hw = &adapter->hw; 2489 2490 switch (hw->mac.type) { 2491 case ixgbe_mac_82598EB: 2492 mask = (IXGBE_EIMS_RTX_QUEUE & qmask); 2493 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 2494 break; 2495 case ixgbe_mac_82599EB: 2496 case ixgbe_mac_X540: 2497 mask = (qmask & 0xFFFFFFFF); 2498 if (mask) 2499 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 2500 mask = (qmask >> 32); 2501 if (mask) 2502 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 2503 break; 2504 default: 2505 break; 2506 } 2507 /* skip the flush */ 2508 } 2509 2510 /** 2511 * ixgbe_irq_enable - Enable default interrupt generation settings 2512 * @adapter: board private structure 2513 **/ 2514 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, 2515 bool flush) 2516 { 2517 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 2518 2519 /* don't reenable LSC while waiting for link */ 2520 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE) 2521 mask &= ~IXGBE_EIMS_LSC; 2522 2523 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) 2524 switch (adapter->hw.mac.type) { 2525 case ixgbe_mac_82599EB: 2526 mask |= IXGBE_EIMS_GPI_SDP0; 2527 break; 2528 case ixgbe_mac_X540: 2529 mask |= IXGBE_EIMS_TS; 2530 break; 2531 default: 2532 break; 2533 } 2534 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) 2535 mask |= IXGBE_EIMS_GPI_SDP1; 2536 switch (adapter->hw.mac.type) { 2537 case ixgbe_mac_82599EB: 2538 mask |= IXGBE_EIMS_GPI_SDP1; 2539 mask |= IXGBE_EIMS_GPI_SDP2; 2540 case ixgbe_mac_X540: 2541 mask |= IXGBE_EIMS_ECC; 2542 mask |= IXGBE_EIMS_MAILBOX; 2543 break; 2544 default: 2545 break; 2546 } 2547 2548 if (adapter->hw.mac.type == ixgbe_mac_X540) 2549 mask |= IXGBE_EIMS_TIMESYNC; 2550 2551 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) && 2552 !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) 2553 mask |= IXGBE_EIMS_FLOW_DIR; 2554 2555 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask); 2556 if (queues) 2557 ixgbe_irq_enable_queues(adapter, ~0); 2558 if (flush) 2559 IXGBE_WRITE_FLUSH(&adapter->hw); 2560 } 2561 2562 static irqreturn_t ixgbe_msix_other(int irq, void *data) 2563 { 2564 struct ixgbe_adapter *adapter = data; 2565 struct ixgbe_hw *hw = &adapter->hw; 2566 u32 eicr; 2567 2568 /* 2569 * Workaround for Silicon errata. Use clear-by-write instead 2570 * of clear-by-read. Reading with EICS will return the 2571 * interrupt causes without clearing, which later be done 2572 * with the write to EICR. 2573 */ 2574 eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 2575 2576 /* The lower 16bits of the EICR register are for the queue interrupts 2577 * which should be masked here in order to not accidently clear them if 2578 * the bits are high when ixgbe_msix_other is called. There is a race 2579 * condition otherwise which results in possible performance loss 2580 * especially if the ixgbe_msix_other interrupt is triggering 2581 * consistently (as it would when PPS is turned on for the X540 device) 2582 */ 2583 eicr &= 0xFFFF0000; 2584 2585 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 2586 2587 if (eicr & IXGBE_EICR_LSC) 2588 ixgbe_check_lsc(adapter); 2589 2590 if (eicr & IXGBE_EICR_MAILBOX) 2591 ixgbe_msg_task(adapter); 2592 2593 switch (hw->mac.type) { 2594 case ixgbe_mac_82599EB: 2595 case ixgbe_mac_X540: 2596 if (eicr & IXGBE_EICR_ECC) 2597 e_info(link, "Received unrecoverable ECC Err, please " 2598 "reboot\n"); 2599 /* Handle Flow Director Full threshold interrupt */ 2600 if (eicr & IXGBE_EICR_FLOW_DIR) { 2601 int reinit_count = 0; 2602 int i; 2603 for (i = 0; i < adapter->num_tx_queues; i++) { 2604 struct ixgbe_ring *ring = adapter->tx_ring[i]; 2605 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE, 2606 &ring->state)) 2607 reinit_count++; 2608 } 2609 if (reinit_count) { 2610 /* no more flow director interrupts until after init */ 2611 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR); 2612 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT; 2613 ixgbe_service_event_schedule(adapter); 2614 } 2615 } 2616 ixgbe_check_sfp_event(adapter, eicr); 2617 ixgbe_check_overtemp_event(adapter, eicr); 2618 break; 2619 default: 2620 break; 2621 } 2622 2623 ixgbe_check_fan_failure(adapter, eicr); 2624 2625 if (unlikely(eicr & IXGBE_EICR_TIMESYNC)) 2626 ixgbe_ptp_check_pps_event(adapter, eicr); 2627 2628 /* re-enable the original interrupt state, no lsc, no queues */ 2629 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2630 ixgbe_irq_enable(adapter, false, false); 2631 2632 return IRQ_HANDLED; 2633 } 2634 2635 static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data) 2636 { 2637 struct ixgbe_q_vector *q_vector = data; 2638 2639 /* EIAM disabled interrupts (on this vector) for us */ 2640 2641 if (q_vector->rx.ring || q_vector->tx.ring) 2642 napi_schedule(&q_vector->napi); 2643 2644 return IRQ_HANDLED; 2645 } 2646 2647 /** 2648 * ixgbe_poll - NAPI Rx polling callback 2649 * @napi: structure for representing this polling device 2650 * @budget: how many packets driver is allowed to clean 2651 * 2652 * This function is used for legacy and MSI, NAPI mode 2653 **/ 2654 int ixgbe_poll(struct napi_struct *napi, int budget) 2655 { 2656 struct ixgbe_q_vector *q_vector = 2657 container_of(napi, struct ixgbe_q_vector, napi); 2658 struct ixgbe_adapter *adapter = q_vector->adapter; 2659 struct ixgbe_ring *ring; 2660 int per_ring_budget; 2661 bool clean_complete = true; 2662 2663 #ifdef CONFIG_IXGBE_DCA 2664 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) 2665 ixgbe_update_dca(q_vector); 2666 #endif 2667 2668 ixgbe_for_each_ring(ring, q_vector->tx) 2669 clean_complete &= !!ixgbe_clean_tx_irq(q_vector, ring); 2670 2671 if (!ixgbe_qv_lock_napi(q_vector)) 2672 return budget; 2673 2674 /* attempt to distribute budget to each queue fairly, but don't allow 2675 * the budget to go below 1 because we'll exit polling */ 2676 if (q_vector->rx.count > 1) 2677 per_ring_budget = max(budget/q_vector->rx.count, 1); 2678 else 2679 per_ring_budget = budget; 2680 2681 ixgbe_for_each_ring(ring, q_vector->rx) 2682 clean_complete &= (ixgbe_clean_rx_irq(q_vector, ring, 2683 per_ring_budget) < per_ring_budget); 2684 2685 ixgbe_qv_unlock_napi(q_vector); 2686 /* If all work not completed, return budget and keep polling */ 2687 if (!clean_complete) 2688 return budget; 2689 2690 /* all work done, exit the polling mode */ 2691 napi_complete(napi); 2692 if (adapter->rx_itr_setting & 1) 2693 ixgbe_set_itr(q_vector); 2694 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2695 ixgbe_irq_enable_queues(adapter, ((u64)1 << q_vector->v_idx)); 2696 2697 return 0; 2698 } 2699 2700 /** 2701 * ixgbe_request_msix_irqs - Initialize MSI-X interrupts 2702 * @adapter: board private structure 2703 * 2704 * ixgbe_request_msix_irqs allocates MSI-X vectors and requests 2705 * interrupts from the kernel. 2706 **/ 2707 static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter) 2708 { 2709 struct net_device *netdev = adapter->netdev; 2710 int vector, err; 2711 int ri = 0, ti = 0; 2712 2713 for (vector = 0; vector < adapter->num_q_vectors; vector++) { 2714 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector]; 2715 struct msix_entry *entry = &adapter->msix_entries[vector]; 2716 2717 if (q_vector->tx.ring && q_vector->rx.ring) { 2718 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2719 "%s-%s-%d", netdev->name, "TxRx", ri++); 2720 ti++; 2721 } else if (q_vector->rx.ring) { 2722 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2723 "%s-%s-%d", netdev->name, "rx", ri++); 2724 } else if (q_vector->tx.ring) { 2725 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2726 "%s-%s-%d", netdev->name, "tx", ti++); 2727 } else { 2728 /* skip this unused q_vector */ 2729 continue; 2730 } 2731 err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0, 2732 q_vector->name, q_vector); 2733 if (err) { 2734 e_err(probe, "request_irq failed for MSIX interrupt " 2735 "Error: %d\n", err); 2736 goto free_queue_irqs; 2737 } 2738 /* If Flow Director is enabled, set interrupt affinity */ 2739 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 2740 /* assign the mask for this irq */ 2741 irq_set_affinity_hint(entry->vector, 2742 &q_vector->affinity_mask); 2743 } 2744 } 2745 2746 err = request_irq(adapter->msix_entries[vector].vector, 2747 ixgbe_msix_other, 0, netdev->name, adapter); 2748 if (err) { 2749 e_err(probe, "request_irq for msix_other failed: %d\n", err); 2750 goto free_queue_irqs; 2751 } 2752 2753 return 0; 2754 2755 free_queue_irqs: 2756 while (vector) { 2757 vector--; 2758 irq_set_affinity_hint(adapter->msix_entries[vector].vector, 2759 NULL); 2760 free_irq(adapter->msix_entries[vector].vector, 2761 adapter->q_vector[vector]); 2762 } 2763 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED; 2764 pci_disable_msix(adapter->pdev); 2765 kfree(adapter->msix_entries); 2766 adapter->msix_entries = NULL; 2767 return err; 2768 } 2769 2770 /** 2771 * ixgbe_intr - legacy mode Interrupt Handler 2772 * @irq: interrupt number 2773 * @data: pointer to a network interface device structure 2774 **/ 2775 static irqreturn_t ixgbe_intr(int irq, void *data) 2776 { 2777 struct ixgbe_adapter *adapter = data; 2778 struct ixgbe_hw *hw = &adapter->hw; 2779 struct ixgbe_q_vector *q_vector = adapter->q_vector[0]; 2780 u32 eicr; 2781 2782 /* 2783 * Workaround for silicon errata #26 on 82598. Mask the interrupt 2784 * before the read of EICR. 2785 */ 2786 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); 2787 2788 /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read 2789 * therefore no explicit interrupt disable is necessary */ 2790 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 2791 if (!eicr) { 2792 /* 2793 * shared interrupt alert! 2794 * make sure interrupts are enabled because the read will 2795 * have disabled interrupts due to EIAM 2796 * finish the workaround of silicon errata on 82598. Unmask 2797 * the interrupt that we masked before the EICR read. 2798 */ 2799 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2800 ixgbe_irq_enable(adapter, true, true); 2801 return IRQ_NONE; /* Not our interrupt */ 2802 } 2803 2804 if (eicr & IXGBE_EICR_LSC) 2805 ixgbe_check_lsc(adapter); 2806 2807 switch (hw->mac.type) { 2808 case ixgbe_mac_82599EB: 2809 ixgbe_check_sfp_event(adapter, eicr); 2810 /* Fall through */ 2811 case ixgbe_mac_X540: 2812 if (eicr & IXGBE_EICR_ECC) 2813 e_info(link, "Received unrecoverable ECC err, please " 2814 "reboot\n"); 2815 ixgbe_check_overtemp_event(adapter, eicr); 2816 break; 2817 default: 2818 break; 2819 } 2820 2821 ixgbe_check_fan_failure(adapter, eicr); 2822 if (unlikely(eicr & IXGBE_EICR_TIMESYNC)) 2823 ixgbe_ptp_check_pps_event(adapter, eicr); 2824 2825 /* would disable interrupts here but EIAM disabled it */ 2826 napi_schedule(&q_vector->napi); 2827 2828 /* 2829 * re-enable link(maybe) and non-queue interrupts, no flush. 2830 * ixgbe_poll will re-enable the queue interrupts 2831 */ 2832 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2833 ixgbe_irq_enable(adapter, false, false); 2834 2835 return IRQ_HANDLED; 2836 } 2837 2838 /** 2839 * ixgbe_request_irq - initialize interrupts 2840 * @adapter: board private structure 2841 * 2842 * Attempts to configure interrupts using the best available 2843 * capabilities of the hardware and kernel. 2844 **/ 2845 static int ixgbe_request_irq(struct ixgbe_adapter *adapter) 2846 { 2847 struct net_device *netdev = adapter->netdev; 2848 int err; 2849 2850 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) 2851 err = ixgbe_request_msix_irqs(adapter); 2852 else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) 2853 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0, 2854 netdev->name, adapter); 2855 else 2856 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED, 2857 netdev->name, adapter); 2858 2859 if (err) 2860 e_err(probe, "request_irq failed, Error %d\n", err); 2861 2862 return err; 2863 } 2864 2865 static void ixgbe_free_irq(struct ixgbe_adapter *adapter) 2866 { 2867 int vector; 2868 2869 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) { 2870 free_irq(adapter->pdev->irq, adapter); 2871 return; 2872 } 2873 2874 for (vector = 0; vector < adapter->num_q_vectors; vector++) { 2875 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector]; 2876 struct msix_entry *entry = &adapter->msix_entries[vector]; 2877 2878 /* free only the irqs that were actually requested */ 2879 if (!q_vector->rx.ring && !q_vector->tx.ring) 2880 continue; 2881 2882 /* clear the affinity_mask in the IRQ descriptor */ 2883 irq_set_affinity_hint(entry->vector, NULL); 2884 2885 free_irq(entry->vector, q_vector); 2886 } 2887 2888 free_irq(adapter->msix_entries[vector++].vector, adapter); 2889 } 2890 2891 /** 2892 * ixgbe_irq_disable - Mask off interrupt generation on the NIC 2893 * @adapter: board private structure 2894 **/ 2895 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter) 2896 { 2897 switch (adapter->hw.mac.type) { 2898 case ixgbe_mac_82598EB: 2899 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 2900 break; 2901 case ixgbe_mac_82599EB: 2902 case ixgbe_mac_X540: 2903 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); 2904 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); 2905 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); 2906 break; 2907 default: 2908 break; 2909 } 2910 IXGBE_WRITE_FLUSH(&adapter->hw); 2911 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 2912 int vector; 2913 2914 for (vector = 0; vector < adapter->num_q_vectors; vector++) 2915 synchronize_irq(adapter->msix_entries[vector].vector); 2916 2917 synchronize_irq(adapter->msix_entries[vector++].vector); 2918 } else { 2919 synchronize_irq(adapter->pdev->irq); 2920 } 2921 } 2922 2923 /** 2924 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts 2925 * 2926 **/ 2927 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter) 2928 { 2929 struct ixgbe_q_vector *q_vector = adapter->q_vector[0]; 2930 2931 ixgbe_write_eitr(q_vector); 2932 2933 ixgbe_set_ivar(adapter, 0, 0, 0); 2934 ixgbe_set_ivar(adapter, 1, 0, 0); 2935 2936 e_info(hw, "Legacy interrupt IVAR setup done\n"); 2937 } 2938 2939 /** 2940 * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset 2941 * @adapter: board private structure 2942 * @ring: structure containing ring specific data 2943 * 2944 * Configure the Tx descriptor ring after a reset. 2945 **/ 2946 void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, 2947 struct ixgbe_ring *ring) 2948 { 2949 struct ixgbe_hw *hw = &adapter->hw; 2950 u64 tdba = ring->dma; 2951 int wait_loop = 10; 2952 u32 txdctl = IXGBE_TXDCTL_ENABLE; 2953 u8 reg_idx = ring->reg_idx; 2954 2955 /* disable queue to avoid issues while updating state */ 2956 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0); 2957 IXGBE_WRITE_FLUSH(hw); 2958 2959 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx), 2960 (tdba & DMA_BIT_MASK(32))); 2961 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32)); 2962 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx), 2963 ring->count * sizeof(union ixgbe_adv_tx_desc)); 2964 IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0); 2965 IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0); 2966 ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx); 2967 2968 /* 2969 * set WTHRESH to encourage burst writeback, it should not be set 2970 * higher than 1 when: 2971 * - ITR is 0 as it could cause false TX hangs 2972 * - ITR is set to > 100k int/sec and BQL is enabled 2973 * 2974 * In order to avoid issues WTHRESH + PTHRESH should always be equal 2975 * to or less than the number of on chip descriptors, which is 2976 * currently 40. 2977 */ 2978 #if IS_ENABLED(CONFIG_BQL) 2979 if (!ring->q_vector || (ring->q_vector->itr < IXGBE_100K_ITR)) 2980 #else 2981 if (!ring->q_vector || (ring->q_vector->itr < 8)) 2982 #endif 2983 txdctl |= (1 << 16); /* WTHRESH = 1 */ 2984 else 2985 txdctl |= (8 << 16); /* WTHRESH = 8 */ 2986 2987 /* 2988 * Setting PTHRESH to 32 both improves performance 2989 * and avoids a TX hang with DFP enabled 2990 */ 2991 txdctl |= (1 << 8) | /* HTHRESH = 1 */ 2992 32; /* PTHRESH = 32 */ 2993 2994 /* reinitialize flowdirector state */ 2995 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 2996 ring->atr_sample_rate = adapter->atr_sample_rate; 2997 ring->atr_count = 0; 2998 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state); 2999 } else { 3000 ring->atr_sample_rate = 0; 3001 } 3002 3003 /* initialize XPS */ 3004 if (!test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, &ring->state)) { 3005 struct ixgbe_q_vector *q_vector = ring->q_vector; 3006 3007 if (q_vector) 3008 netif_set_xps_queue(adapter->netdev, 3009 &q_vector->affinity_mask, 3010 ring->queue_index); 3011 } 3012 3013 clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state); 3014 3015 /* enable queue */ 3016 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl); 3017 3018 /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */ 3019 if (hw->mac.type == ixgbe_mac_82598EB && 3020 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) 3021 return; 3022 3023 /* poll to verify queue is enabled */ 3024 do { 3025 usleep_range(1000, 2000); 3026 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx)); 3027 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 3028 if (!wait_loop) 3029 e_err(drv, "Could not enable Tx Queue %d\n", reg_idx); 3030 } 3031 3032 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter) 3033 { 3034 struct ixgbe_hw *hw = &adapter->hw; 3035 u32 rttdcs, mtqc; 3036 u8 tcs = netdev_get_num_tc(adapter->netdev); 3037 3038 if (hw->mac.type == ixgbe_mac_82598EB) 3039 return; 3040 3041 /* disable the arbiter while setting MTQC */ 3042 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 3043 rttdcs |= IXGBE_RTTDCS_ARBDIS; 3044 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3045 3046 /* set transmit pool layout */ 3047 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 3048 mtqc = IXGBE_MTQC_VT_ENA; 3049 if (tcs > 4) 3050 mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ; 3051 else if (tcs > 1) 3052 mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ; 3053 else if (adapter->ring_feature[RING_F_RSS].indices == 4) 3054 mtqc |= IXGBE_MTQC_32VF; 3055 else 3056 mtqc |= IXGBE_MTQC_64VF; 3057 } else { 3058 if (tcs > 4) 3059 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ; 3060 else if (tcs > 1) 3061 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ; 3062 else 3063 mtqc = IXGBE_MTQC_64Q_1PB; 3064 } 3065 3066 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc); 3067 3068 /* Enable Security TX Buffer IFG for multiple pb */ 3069 if (tcs) { 3070 u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); 3071 sectx |= IXGBE_SECTX_DCB; 3072 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx); 3073 } 3074 3075 /* re-enable the arbiter */ 3076 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 3077 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3078 } 3079 3080 /** 3081 * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset 3082 * @adapter: board private structure 3083 * 3084 * Configure the Tx unit of the MAC after a reset. 3085 **/ 3086 static void ixgbe_configure_tx(struct ixgbe_adapter *adapter) 3087 { 3088 struct ixgbe_hw *hw = &adapter->hw; 3089 u32 dmatxctl; 3090 u32 i; 3091 3092 ixgbe_setup_mtqc(adapter); 3093 3094 if (hw->mac.type != ixgbe_mac_82598EB) { 3095 /* DMATXCTL.EN must be before Tx queues are enabled */ 3096 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 3097 dmatxctl |= IXGBE_DMATXCTL_TE; 3098 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 3099 } 3100 3101 /* Setup the HW Tx Head and Tail descriptor pointers */ 3102 for (i = 0; i < adapter->num_tx_queues; i++) 3103 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]); 3104 } 3105 3106 static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter, 3107 struct ixgbe_ring *ring) 3108 { 3109 struct ixgbe_hw *hw = &adapter->hw; 3110 u8 reg_idx = ring->reg_idx; 3111 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx)); 3112 3113 srrctl |= IXGBE_SRRCTL_DROP_EN; 3114 3115 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl); 3116 } 3117 3118 static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter, 3119 struct ixgbe_ring *ring) 3120 { 3121 struct ixgbe_hw *hw = &adapter->hw; 3122 u8 reg_idx = ring->reg_idx; 3123 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx)); 3124 3125 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 3126 3127 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl); 3128 } 3129 3130 #ifdef CONFIG_IXGBE_DCB 3131 void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) 3132 #else 3133 static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) 3134 #endif 3135 { 3136 int i; 3137 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable; 3138 3139 if (adapter->ixgbe_ieee_pfc) 3140 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en); 3141 3142 /* 3143 * We should set the drop enable bit if: 3144 * SR-IOV is enabled 3145 * or 3146 * Number of Rx queues > 1 and flow control is disabled 3147 * 3148 * This allows us to avoid head of line blocking for security 3149 * and performance reasons. 3150 */ 3151 if (adapter->num_vfs || (adapter->num_rx_queues > 1 && 3152 !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) { 3153 for (i = 0; i < adapter->num_rx_queues; i++) 3154 ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]); 3155 } else { 3156 for (i = 0; i < adapter->num_rx_queues; i++) 3157 ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]); 3158 } 3159 } 3160 3161 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3162 3163 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, 3164 struct ixgbe_ring *rx_ring) 3165 { 3166 struct ixgbe_hw *hw = &adapter->hw; 3167 u32 srrctl; 3168 u8 reg_idx = rx_ring->reg_idx; 3169 3170 if (hw->mac.type == ixgbe_mac_82598EB) { 3171 u16 mask = adapter->ring_feature[RING_F_RSS].mask; 3172 3173 /* 3174 * if VMDq is not active we must program one srrctl register 3175 * per RSS queue since we have enabled RDRXCTL.MVMEN 3176 */ 3177 reg_idx &= mask; 3178 } 3179 3180 /* configure header buffer length, needed for RSC */ 3181 srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; 3182 3183 /* configure the packet buffer length */ 3184 srrctl |= ixgbe_rx_bufsz(rx_ring) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3185 3186 /* configure descriptor type */ 3187 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 3188 3189 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl); 3190 } 3191 3192 static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) 3193 { 3194 struct ixgbe_hw *hw = &adapter->hw; 3195 static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D, 3196 0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE, 3197 0x6A3E67EA, 0x14364D17, 0x3BED200D}; 3198 u32 mrqc = 0, reta = 0; 3199 u32 rxcsum; 3200 int i, j; 3201 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices; 3202 3203 /* 3204 * Program table for at least 2 queues w/ SR-IOV so that VFs can 3205 * make full use of any rings they may have. We will use the 3206 * PSRTYPE register to control how many rings we use within the PF. 3207 */ 3208 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 2)) 3209 rss_i = 2; 3210 3211 /* Fill out hash function seeds */ 3212 for (i = 0; i < 10; i++) 3213 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]); 3214 3215 /* Fill out redirection table */ 3216 for (i = 0, j = 0; i < 128; i++, j++) { 3217 if (j == rss_i) 3218 j = 0; 3219 /* reta = 4-byte sliding window of 3220 * 0x00..(indices-1)(indices-1)00..etc. */ 3221 reta = (reta << 8) | (j * 0x11); 3222 if ((i & 3) == 3) 3223 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 3224 } 3225 3226 /* Disable indicating checksum in descriptor, enables RSS hash */ 3227 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3228 rxcsum |= IXGBE_RXCSUM_PCSD; 3229 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3230 3231 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 3232 if (adapter->ring_feature[RING_F_RSS].mask) 3233 mrqc = IXGBE_MRQC_RSSEN; 3234 } else { 3235 u8 tcs = netdev_get_num_tc(adapter->netdev); 3236 3237 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 3238 if (tcs > 4) 3239 mrqc = IXGBE_MRQC_VMDQRT8TCEN; /* 8 TCs */ 3240 else if (tcs > 1) 3241 mrqc = IXGBE_MRQC_VMDQRT4TCEN; /* 4 TCs */ 3242 else if (adapter->ring_feature[RING_F_RSS].indices == 4) 3243 mrqc = IXGBE_MRQC_VMDQRSS32EN; 3244 else 3245 mrqc = IXGBE_MRQC_VMDQRSS64EN; 3246 } else { 3247 if (tcs > 4) 3248 mrqc = IXGBE_MRQC_RTRSS8TCEN; 3249 else if (tcs > 1) 3250 mrqc = IXGBE_MRQC_RTRSS4TCEN; 3251 else 3252 mrqc = IXGBE_MRQC_RSSEN; 3253 } 3254 } 3255 3256 /* Perform hash on these packet types */ 3257 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4 | 3258 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 3259 IXGBE_MRQC_RSS_FIELD_IPV6 | 3260 IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 3261 3262 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP) 3263 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 3264 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP) 3265 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 3266 3267 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 3268 } 3269 3270 /** 3271 * ixgbe_configure_rscctl - enable RSC for the indicated ring 3272 * @adapter: address of board private structure 3273 * @index: index of ring to set 3274 **/ 3275 static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, 3276 struct ixgbe_ring *ring) 3277 { 3278 struct ixgbe_hw *hw = &adapter->hw; 3279 u32 rscctrl; 3280 u8 reg_idx = ring->reg_idx; 3281 3282 if (!ring_is_rsc_enabled(ring)) 3283 return; 3284 3285 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx)); 3286 rscctrl |= IXGBE_RSCCTL_RSCEN; 3287 /* 3288 * we must limit the number of descriptors so that the 3289 * total size of max desc * buf_len is not greater 3290 * than 65536 3291 */ 3292 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 3293 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl); 3294 } 3295 3296 #define IXGBE_MAX_RX_DESC_POLL 10 3297 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, 3298 struct ixgbe_ring *ring) 3299 { 3300 struct ixgbe_hw *hw = &adapter->hw; 3301 int wait_loop = IXGBE_MAX_RX_DESC_POLL; 3302 u32 rxdctl; 3303 u8 reg_idx = ring->reg_idx; 3304 3305 /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */ 3306 if (hw->mac.type == ixgbe_mac_82598EB && 3307 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) 3308 return; 3309 3310 do { 3311 usleep_range(1000, 2000); 3312 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3313 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 3314 3315 if (!wait_loop) { 3316 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within " 3317 "the polling period\n", reg_idx); 3318 } 3319 } 3320 3321 void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter, 3322 struct ixgbe_ring *ring) 3323 { 3324 struct ixgbe_hw *hw = &adapter->hw; 3325 int wait_loop = IXGBE_MAX_RX_DESC_POLL; 3326 u32 rxdctl; 3327 u8 reg_idx = ring->reg_idx; 3328 3329 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3330 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 3331 3332 /* write value back with RXDCTL.ENABLE bit cleared */ 3333 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); 3334 3335 if (hw->mac.type == ixgbe_mac_82598EB && 3336 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) 3337 return; 3338 3339 /* the hardware may take up to 100us to really disable the rx queue */ 3340 do { 3341 udelay(10); 3342 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3343 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 3344 3345 if (!wait_loop) { 3346 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within " 3347 "the polling period\n", reg_idx); 3348 } 3349 } 3350 3351 void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, 3352 struct ixgbe_ring *ring) 3353 { 3354 struct ixgbe_hw *hw = &adapter->hw; 3355 u64 rdba = ring->dma; 3356 u32 rxdctl; 3357 u8 reg_idx = ring->reg_idx; 3358 3359 /* disable queue to avoid issues while updating state */ 3360 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3361 ixgbe_disable_rx_queue(adapter, ring); 3362 3363 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32))); 3364 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32)); 3365 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx), 3366 ring->count * sizeof(union ixgbe_adv_rx_desc)); 3367 IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0); 3368 IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0); 3369 ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx); 3370 3371 ixgbe_configure_srrctl(adapter, ring); 3372 ixgbe_configure_rscctl(adapter, ring); 3373 3374 if (hw->mac.type == ixgbe_mac_82598EB) { 3375 /* 3376 * enable cache line friendly hardware writes: 3377 * PTHRESH=32 descriptors (half the internal cache), 3378 * this also removes ugly rx_no_buffer_count increment 3379 * HTHRESH=4 descriptors (to minimize latency on fetch) 3380 * WTHRESH=8 burst writeback up to two cache lines 3381 */ 3382 rxdctl &= ~0x3FFFFF; 3383 rxdctl |= 0x080420; 3384 } 3385 3386 /* enable receive descriptor ring */ 3387 rxdctl |= IXGBE_RXDCTL_ENABLE; 3388 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); 3389 3390 ixgbe_rx_desc_queue_enable(adapter, ring); 3391 ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring)); 3392 } 3393 3394 static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter) 3395 { 3396 struct ixgbe_hw *hw = &adapter->hw; 3397 int rss_i = adapter->ring_feature[RING_F_RSS].indices; 3398 int p; 3399 3400 /* PSRTYPE must be initialized in non 82598 adapters */ 3401 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 3402 IXGBE_PSRTYPE_UDPHDR | 3403 IXGBE_PSRTYPE_IPV4HDR | 3404 IXGBE_PSRTYPE_L2HDR | 3405 IXGBE_PSRTYPE_IPV6HDR; 3406 3407 if (hw->mac.type == ixgbe_mac_82598EB) 3408 return; 3409 3410 if (rss_i > 3) 3411 psrtype |= 2 << 29; 3412 else if (rss_i > 1) 3413 psrtype |= 1 << 29; 3414 3415 for (p = 0; p < adapter->num_rx_pools; p++) 3416 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(p)), 3417 psrtype); 3418 } 3419 3420 static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter) 3421 { 3422 struct ixgbe_hw *hw = &adapter->hw; 3423 u32 reg_offset, vf_shift; 3424 u32 gcr_ext, vmdctl; 3425 int i; 3426 3427 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 3428 return; 3429 3430 vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 3431 vmdctl |= IXGBE_VMD_CTL_VMDQ_EN; 3432 vmdctl &= ~IXGBE_VT_CTL_POOL_MASK; 3433 vmdctl |= VMDQ_P(0) << IXGBE_VT_CTL_POOL_SHIFT; 3434 vmdctl |= IXGBE_VT_CTL_REPLEN; 3435 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl); 3436 3437 vf_shift = VMDQ_P(0) % 32; 3438 reg_offset = (VMDQ_P(0) >= 32) ? 1 : 0; 3439 3440 /* Enable only the PF's pool for Tx/Rx */ 3441 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (~0) << vf_shift); 3442 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), reg_offset - 1); 3443 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (~0) << vf_shift); 3444 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), reg_offset - 1); 3445 if (adapter->flags2 & IXGBE_FLAG2_BRIDGE_MODE_VEB) 3446 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN); 3447 3448 /* Map PF MAC address in RAR Entry 0 to first pool following VFs */ 3449 hw->mac.ops.set_vmdq(hw, 0, VMDQ_P(0)); 3450 3451 /* 3452 * Set up VF register offsets for selected VT Mode, 3453 * i.e. 32 or 64 VFs for SR-IOV 3454 */ 3455 switch (adapter->ring_feature[RING_F_VMDQ].mask) { 3456 case IXGBE_82599_VMDQ_8Q_MASK: 3457 gcr_ext = IXGBE_GCR_EXT_VT_MODE_16; 3458 break; 3459 case IXGBE_82599_VMDQ_4Q_MASK: 3460 gcr_ext = IXGBE_GCR_EXT_VT_MODE_32; 3461 break; 3462 default: 3463 gcr_ext = IXGBE_GCR_EXT_VT_MODE_64; 3464 break; 3465 } 3466 3467 IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext); 3468 3469 3470 /* Enable MAC Anti-Spoofing */ 3471 hw->mac.ops.set_mac_anti_spoofing(hw, (adapter->num_vfs != 0), 3472 adapter->num_vfs); 3473 /* For VFs that have spoof checking turned off */ 3474 for (i = 0; i < adapter->num_vfs; i++) { 3475 if (!adapter->vfinfo[i].spoofchk_enabled) 3476 ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i, false); 3477 } 3478 } 3479 3480 static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) 3481 { 3482 struct ixgbe_hw *hw = &adapter->hw; 3483 struct net_device *netdev = adapter->netdev; 3484 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 3485 struct ixgbe_ring *rx_ring; 3486 int i; 3487 u32 mhadd, hlreg0; 3488 3489 #ifdef IXGBE_FCOE 3490 /* adjust max frame to be able to do baby jumbo for FCoE */ 3491 if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && 3492 (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) 3493 max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; 3494 3495 #endif /* IXGBE_FCOE */ 3496 3497 /* adjust max frame to be at least the size of a standard frame */ 3498 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 3499 max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN); 3500 3501 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 3502 if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { 3503 mhadd &= ~IXGBE_MHADD_MFS_MASK; 3504 mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; 3505 3506 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 3507 } 3508 3509 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3510 /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */ 3511 hlreg0 |= IXGBE_HLREG0_JUMBOEN; 3512 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); 3513 3514 /* 3515 * Setup the HW Rx Head and Tail Descriptor Pointers and 3516 * the Base and Length of the Rx Descriptor Ring 3517 */ 3518 for (i = 0; i < adapter->num_rx_queues; i++) { 3519 rx_ring = adapter->rx_ring[i]; 3520 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) 3521 set_ring_rsc_enabled(rx_ring); 3522 else 3523 clear_ring_rsc_enabled(rx_ring); 3524 } 3525 } 3526 3527 static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) 3528 { 3529 struct ixgbe_hw *hw = &adapter->hw; 3530 u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 3531 3532 switch (hw->mac.type) { 3533 case ixgbe_mac_82598EB: 3534 /* 3535 * For VMDq support of different descriptor types or 3536 * buffer sizes through the use of multiple SRRCTL 3537 * registers, RDRXCTL.MVMEN must be set to 1 3538 * 3539 * also, the manual doesn't mention it clearly but DCA hints 3540 * will only use queue 0's tags unless this bit is set. Side 3541 * effects of setting this bit are only that SRRCTL must be 3542 * fully programmed [0..15] 3543 */ 3544 rdrxctl |= IXGBE_RDRXCTL_MVMEN; 3545 break; 3546 case ixgbe_mac_82599EB: 3547 case ixgbe_mac_X540: 3548 /* Disable RSC for ACK packets */ 3549 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 3550 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 3551 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 3552 /* hardware requires some bits to be set by default */ 3553 rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX); 3554 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 3555 break; 3556 default: 3557 /* We should do nothing since we don't know this hardware */ 3558 return; 3559 } 3560 3561 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 3562 } 3563 3564 /** 3565 * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset 3566 * @adapter: board private structure 3567 * 3568 * Configure the Rx unit of the MAC after a reset. 3569 **/ 3570 static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) 3571 { 3572 struct ixgbe_hw *hw = &adapter->hw; 3573 int i; 3574 u32 rxctrl; 3575 3576 /* disable receives while setting up the descriptors */ 3577 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 3578 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); 3579 3580 ixgbe_setup_psrtype(adapter); 3581 ixgbe_setup_rdrxctl(adapter); 3582 3583 /* Program registers for the distribution of queues */ 3584 ixgbe_setup_mrqc(adapter); 3585 3586 /* set_rx_buffer_len must be called before ring initialization */ 3587 ixgbe_set_rx_buffer_len(adapter); 3588 3589 /* 3590 * Setup the HW Rx Head and Tail Descriptor Pointers and 3591 * the Base and Length of the Rx Descriptor Ring 3592 */ 3593 for (i = 0; i < adapter->num_rx_queues; i++) 3594 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]); 3595 3596 /* disable drop enable for 82598 parts */ 3597 if (hw->mac.type == ixgbe_mac_82598EB) 3598 rxctrl |= IXGBE_RXCTRL_DMBYPS; 3599 3600 /* enable all receives */ 3601 rxctrl |= IXGBE_RXCTRL_RXEN; 3602 hw->mac.ops.enable_rx_dma(hw, rxctrl); 3603 } 3604 3605 static int ixgbe_vlan_rx_add_vid(struct net_device *netdev, 3606 __be16 proto, u16 vid) 3607 { 3608 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3609 struct ixgbe_hw *hw = &adapter->hw; 3610 3611 /* add VID to filter table */ 3612 hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), true); 3613 set_bit(vid, adapter->active_vlans); 3614 3615 return 0; 3616 } 3617 3618 static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev, 3619 __be16 proto, u16 vid) 3620 { 3621 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3622 struct ixgbe_hw *hw = &adapter->hw; 3623 3624 /* remove VID from filter table */ 3625 hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), false); 3626 clear_bit(vid, adapter->active_vlans); 3627 3628 return 0; 3629 } 3630 3631 /** 3632 * ixgbe_vlan_filter_disable - helper to disable hw vlan filtering 3633 * @adapter: driver data 3634 */ 3635 static void ixgbe_vlan_filter_disable(struct ixgbe_adapter *adapter) 3636 { 3637 struct ixgbe_hw *hw = &adapter->hw; 3638 u32 vlnctrl; 3639 3640 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3641 vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN); 3642 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 3643 } 3644 3645 /** 3646 * ixgbe_vlan_filter_enable - helper to enable hw vlan filtering 3647 * @adapter: driver data 3648 */ 3649 static void ixgbe_vlan_filter_enable(struct ixgbe_adapter *adapter) 3650 { 3651 struct ixgbe_hw *hw = &adapter->hw; 3652 u32 vlnctrl; 3653 3654 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3655 vlnctrl |= IXGBE_VLNCTRL_VFE; 3656 vlnctrl &= ~IXGBE_VLNCTRL_CFIEN; 3657 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 3658 } 3659 3660 /** 3661 * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping 3662 * @adapter: driver data 3663 */ 3664 static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter) 3665 { 3666 struct ixgbe_hw *hw = &adapter->hw; 3667 u32 vlnctrl; 3668 int i, j; 3669 3670 switch (hw->mac.type) { 3671 case ixgbe_mac_82598EB: 3672 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3673 vlnctrl &= ~IXGBE_VLNCTRL_VME; 3674 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 3675 break; 3676 case ixgbe_mac_82599EB: 3677 case ixgbe_mac_X540: 3678 for (i = 0; i < adapter->num_rx_queues; i++) { 3679 j = adapter->rx_ring[i]->reg_idx; 3680 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j)); 3681 vlnctrl &= ~IXGBE_RXDCTL_VME; 3682 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl); 3683 } 3684 break; 3685 default: 3686 break; 3687 } 3688 } 3689 3690 /** 3691 * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping 3692 * @adapter: driver data 3693 */ 3694 static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter) 3695 { 3696 struct ixgbe_hw *hw = &adapter->hw; 3697 u32 vlnctrl; 3698 int i, j; 3699 3700 switch (hw->mac.type) { 3701 case ixgbe_mac_82598EB: 3702 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3703 vlnctrl |= IXGBE_VLNCTRL_VME; 3704 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 3705 break; 3706 case ixgbe_mac_82599EB: 3707 case ixgbe_mac_X540: 3708 for (i = 0; i < adapter->num_rx_queues; i++) { 3709 j = adapter->rx_ring[i]->reg_idx; 3710 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j)); 3711 vlnctrl |= IXGBE_RXDCTL_VME; 3712 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl); 3713 } 3714 break; 3715 default: 3716 break; 3717 } 3718 } 3719 3720 static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter) 3721 { 3722 u16 vid; 3723 3724 ixgbe_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0); 3725 3726 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 3727 ixgbe_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid); 3728 } 3729 3730 /** 3731 * ixgbe_write_uc_addr_list - write unicast addresses to RAR table 3732 * @netdev: network interface device structure 3733 * 3734 * Writes unicast address list to the RAR table. 3735 * Returns: -ENOMEM on failure/insufficient address space 3736 * 0 on no addresses written 3737 * X on writing X addresses to the RAR table 3738 **/ 3739 static int ixgbe_write_uc_addr_list(struct net_device *netdev) 3740 { 3741 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3742 struct ixgbe_hw *hw = &adapter->hw; 3743 unsigned int rar_entries = hw->mac.num_rar_entries - 1; 3744 int count = 0; 3745 3746 /* In SR-IOV mode significantly less RAR entries are available */ 3747 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 3748 rar_entries = IXGBE_MAX_PF_MACVLANS - 1; 3749 3750 /* return ENOMEM indicating insufficient memory for addresses */ 3751 if (netdev_uc_count(netdev) > rar_entries) 3752 return -ENOMEM; 3753 3754 if (!netdev_uc_empty(netdev)) { 3755 struct netdev_hw_addr *ha; 3756 /* return error if we do not support writing to RAR table */ 3757 if (!hw->mac.ops.set_rar) 3758 return -ENOMEM; 3759 3760 netdev_for_each_uc_addr(ha, netdev) { 3761 if (!rar_entries) 3762 break; 3763 hw->mac.ops.set_rar(hw, rar_entries--, ha->addr, 3764 VMDQ_P(0), IXGBE_RAH_AV); 3765 count++; 3766 } 3767 } 3768 /* write the addresses in reverse order to avoid write combining */ 3769 for (; rar_entries > 0 ; rar_entries--) 3770 hw->mac.ops.clear_rar(hw, rar_entries); 3771 3772 return count; 3773 } 3774 3775 /** 3776 * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set 3777 * @netdev: network interface device structure 3778 * 3779 * The set_rx_method entry point is called whenever the unicast/multicast 3780 * address list or the network interface flags are updated. This routine is 3781 * responsible for configuring the hardware for proper unicast, multicast and 3782 * promiscuous mode. 3783 **/ 3784 void ixgbe_set_rx_mode(struct net_device *netdev) 3785 { 3786 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3787 struct ixgbe_hw *hw = &adapter->hw; 3788 u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE; 3789 int count; 3790 3791 /* Check for Promiscuous and All Multicast modes */ 3792 3793 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 3794 3795 /* set all bits that we expect to always be set */ 3796 fctrl &= ~IXGBE_FCTRL_SBP; /* disable store-bad-packets */ 3797 fctrl |= IXGBE_FCTRL_BAM; 3798 fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */ 3799 fctrl |= IXGBE_FCTRL_PMCF; 3800 3801 /* clear the bits we are changing the status of */ 3802 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 3803 3804 if (netdev->flags & IFF_PROMISC) { 3805 hw->addr_ctrl.user_set_promisc = true; 3806 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 3807 vmolr |= (IXGBE_VMOLR_ROPE | IXGBE_VMOLR_MPE); 3808 /* Only disable hardware filter vlans in promiscuous mode 3809 * if SR-IOV and VMDQ are disabled - otherwise ensure 3810 * that hardware VLAN filters remain enabled. 3811 */ 3812 if (!(adapter->flags & (IXGBE_FLAG_VMDQ_ENABLED | 3813 IXGBE_FLAG_SRIOV_ENABLED))) 3814 ixgbe_vlan_filter_disable(adapter); 3815 else 3816 ixgbe_vlan_filter_enable(adapter); 3817 } else { 3818 if (netdev->flags & IFF_ALLMULTI) { 3819 fctrl |= IXGBE_FCTRL_MPE; 3820 vmolr |= IXGBE_VMOLR_MPE; 3821 } else { 3822 /* 3823 * Write addresses to the MTA, if the attempt fails 3824 * then we should just turn on promiscuous mode so 3825 * that we can at least receive multicast traffic 3826 */ 3827 hw->mac.ops.update_mc_addr_list(hw, netdev); 3828 vmolr |= IXGBE_VMOLR_ROMPE; 3829 } 3830 ixgbe_vlan_filter_enable(adapter); 3831 hw->addr_ctrl.user_set_promisc = false; 3832 } 3833 3834 /* 3835 * Write addresses to available RAR registers, if there is not 3836 * sufficient space to store all the addresses then enable 3837 * unicast promiscuous mode 3838 */ 3839 count = ixgbe_write_uc_addr_list(netdev); 3840 if (count < 0) { 3841 fctrl |= IXGBE_FCTRL_UPE; 3842 vmolr |= IXGBE_VMOLR_ROPE; 3843 } 3844 3845 if (adapter->num_vfs) 3846 ixgbe_restore_vf_multicasts(adapter); 3847 3848 if (hw->mac.type != ixgbe_mac_82598EB) { 3849 vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(VMDQ_P(0))) & 3850 ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE | 3851 IXGBE_VMOLR_ROPE); 3852 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(0)), vmolr); 3853 } 3854 3855 /* This is useful for sniffing bad packets. */ 3856 if (adapter->netdev->features & NETIF_F_RXALL) { 3857 /* UPE and MPE will be handled by normal PROMISC logic 3858 * in e1000e_set_rx_mode */ 3859 fctrl |= (IXGBE_FCTRL_SBP | /* Receive bad packets */ 3860 IXGBE_FCTRL_BAM | /* RX All Bcast Pkts */ 3861 IXGBE_FCTRL_PMCF); /* RX All MAC Ctrl Pkts */ 3862 3863 fctrl &= ~(IXGBE_FCTRL_DPF); 3864 /* NOTE: VLAN filtering is disabled by setting PROMISC */ 3865 } 3866 3867 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 3868 3869 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 3870 ixgbe_vlan_strip_enable(adapter); 3871 else 3872 ixgbe_vlan_strip_disable(adapter); 3873 } 3874 3875 static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter) 3876 { 3877 int q_idx; 3878 3879 for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++) { 3880 ixgbe_qv_init_lock(adapter->q_vector[q_idx]); 3881 napi_enable(&adapter->q_vector[q_idx]->napi); 3882 } 3883 } 3884 3885 static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter) 3886 { 3887 int q_idx; 3888 3889 local_bh_disable(); /* for ixgbe_qv_lock_napi() */ 3890 for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++) { 3891 napi_disable(&adapter->q_vector[q_idx]->napi); 3892 while (!ixgbe_qv_lock_napi(adapter->q_vector[q_idx])) { 3893 pr_info("QV %d locked\n", q_idx); 3894 mdelay(1); 3895 } 3896 } 3897 local_bh_enable(); 3898 } 3899 3900 #ifdef CONFIG_IXGBE_DCB 3901 /** 3902 * ixgbe_configure_dcb - Configure DCB hardware 3903 * @adapter: ixgbe adapter struct 3904 * 3905 * This is called by the driver on open to configure the DCB hardware. 3906 * This is also called by the gennetlink interface when reconfiguring 3907 * the DCB state. 3908 */ 3909 static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter) 3910 { 3911 struct ixgbe_hw *hw = &adapter->hw; 3912 int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 3913 3914 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) { 3915 if (hw->mac.type == ixgbe_mac_82598EB) 3916 netif_set_gso_max_size(adapter->netdev, 65536); 3917 return; 3918 } 3919 3920 if (hw->mac.type == ixgbe_mac_82598EB) 3921 netif_set_gso_max_size(adapter->netdev, 32768); 3922 3923 #ifdef IXGBE_FCOE 3924 if (adapter->netdev->features & NETIF_F_FCOE_MTU) 3925 max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); 3926 #endif 3927 3928 /* reconfigure the hardware */ 3929 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) { 3930 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame, 3931 DCB_TX_CONFIG); 3932 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame, 3933 DCB_RX_CONFIG); 3934 ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg); 3935 } else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) { 3936 ixgbe_dcb_hw_ets(&adapter->hw, 3937 adapter->ixgbe_ieee_ets, 3938 max_frame); 3939 ixgbe_dcb_hw_pfc_config(&adapter->hw, 3940 adapter->ixgbe_ieee_pfc->pfc_en, 3941 adapter->ixgbe_ieee_ets->prio_tc); 3942 } 3943 3944 /* Enable RSS Hash per TC */ 3945 if (hw->mac.type != ixgbe_mac_82598EB) { 3946 u32 msb = 0; 3947 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices - 1; 3948 3949 while (rss_i) { 3950 msb++; 3951 rss_i >>= 1; 3952 } 3953 3954 /* write msb to all 8 TCs in one write */ 3955 IXGBE_WRITE_REG(hw, IXGBE_RQTC, msb * 0x11111111); 3956 } 3957 } 3958 #endif 3959 3960 /* Additional bittime to account for IXGBE framing */ 3961 #define IXGBE_ETH_FRAMING 20 3962 3963 /** 3964 * ixgbe_hpbthresh - calculate high water mark for flow control 3965 * 3966 * @adapter: board private structure to calculate for 3967 * @pb: packet buffer to calculate 3968 */ 3969 static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) 3970 { 3971 struct ixgbe_hw *hw = &adapter->hw; 3972 struct net_device *dev = adapter->netdev; 3973 int link, tc, kb, marker; 3974 u32 dv_id, rx_pba; 3975 3976 /* Calculate max LAN frame size */ 3977 tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING; 3978 3979 #ifdef IXGBE_FCOE 3980 /* FCoE traffic class uses FCOE jumbo frames */ 3981 if ((dev->features & NETIF_F_FCOE_MTU) && 3982 (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) && 3983 (pb == ixgbe_fcoe_get_tc(adapter))) 3984 tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; 3985 3986 #endif 3987 /* Calculate delay value for device */ 3988 switch (hw->mac.type) { 3989 case ixgbe_mac_X540: 3990 dv_id = IXGBE_DV_X540(link, tc); 3991 break; 3992 default: 3993 dv_id = IXGBE_DV(link, tc); 3994 break; 3995 } 3996 3997 /* Loopback switch introduces additional latency */ 3998 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 3999 dv_id += IXGBE_B2BT(tc); 4000 4001 /* Delay value is calculated in bit times convert to KB */ 4002 kb = IXGBE_BT2KB(dv_id); 4003 rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10; 4004 4005 marker = rx_pba - kb; 4006 4007 /* It is possible that the packet buffer is not large enough 4008 * to provide required headroom. In this case throw an error 4009 * to user and a do the best we can. 4010 */ 4011 if (marker < 0) { 4012 e_warn(drv, "Packet Buffer(%i) can not provide enough" 4013 "headroom to support flow control." 4014 "Decrease MTU or number of traffic classes\n", pb); 4015 marker = tc + 1; 4016 } 4017 4018 return marker; 4019 } 4020 4021 /** 4022 * ixgbe_lpbthresh - calculate low water mark for for flow control 4023 * 4024 * @adapter: board private structure to calculate for 4025 * @pb: packet buffer to calculate 4026 */ 4027 static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter) 4028 { 4029 struct ixgbe_hw *hw = &adapter->hw; 4030 struct net_device *dev = adapter->netdev; 4031 int tc; 4032 u32 dv_id; 4033 4034 /* Calculate max LAN frame size */ 4035 tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN; 4036 4037 /* Calculate delay value for device */ 4038 switch (hw->mac.type) { 4039 case ixgbe_mac_X540: 4040 dv_id = IXGBE_LOW_DV_X540(tc); 4041 break; 4042 default: 4043 dv_id = IXGBE_LOW_DV(tc); 4044 break; 4045 } 4046 4047 /* Delay value is calculated in bit times convert to KB */ 4048 return IXGBE_BT2KB(dv_id); 4049 } 4050 4051 /* 4052 * ixgbe_pbthresh_setup - calculate and setup high low water marks 4053 */ 4054 static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter) 4055 { 4056 struct ixgbe_hw *hw = &adapter->hw; 4057 int num_tc = netdev_get_num_tc(adapter->netdev); 4058 int i; 4059 4060 if (!num_tc) 4061 num_tc = 1; 4062 4063 hw->fc.low_water = ixgbe_lpbthresh(adapter); 4064 4065 for (i = 0; i < num_tc; i++) { 4066 hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i); 4067 4068 /* Low water marks must not be larger than high water marks */ 4069 if (hw->fc.low_water > hw->fc.high_water[i]) 4070 hw->fc.low_water = 0; 4071 } 4072 } 4073 4074 static void ixgbe_configure_pb(struct ixgbe_adapter *adapter) 4075 { 4076 struct ixgbe_hw *hw = &adapter->hw; 4077 int hdrm; 4078 u8 tc = netdev_get_num_tc(adapter->netdev); 4079 4080 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE || 4081 adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) 4082 hdrm = 32 << adapter->fdir_pballoc; 4083 else 4084 hdrm = 0; 4085 4086 hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL); 4087 ixgbe_pbthresh_setup(adapter); 4088 } 4089 4090 static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) 4091 { 4092 struct ixgbe_hw *hw = &adapter->hw; 4093 struct hlist_node *node2; 4094 struct ixgbe_fdir_filter *filter; 4095 4096 spin_lock(&adapter->fdir_perfect_lock); 4097 4098 if (!hlist_empty(&adapter->fdir_filter_list)) 4099 ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask); 4100 4101 hlist_for_each_entry_safe(filter, node2, 4102 &adapter->fdir_filter_list, fdir_node) { 4103 ixgbe_fdir_write_perfect_filter_82599(hw, 4104 &filter->filter, 4105 filter->sw_idx, 4106 (filter->action == IXGBE_FDIR_DROP_QUEUE) ? 4107 IXGBE_FDIR_DROP_QUEUE : 4108 adapter->rx_ring[filter->action]->reg_idx); 4109 } 4110 4111 spin_unlock(&adapter->fdir_perfect_lock); 4112 } 4113 4114 static void ixgbe_configure(struct ixgbe_adapter *adapter) 4115 { 4116 struct ixgbe_hw *hw = &adapter->hw; 4117 4118 ixgbe_configure_pb(adapter); 4119 #ifdef CONFIG_IXGBE_DCB 4120 ixgbe_configure_dcb(adapter); 4121 #endif 4122 /* 4123 * We must restore virtualization before VLANs or else 4124 * the VLVF registers will not be populated 4125 */ 4126 ixgbe_configure_virtualization(adapter); 4127 4128 ixgbe_set_rx_mode(adapter->netdev); 4129 ixgbe_restore_vlan(adapter); 4130 4131 switch (hw->mac.type) { 4132 case ixgbe_mac_82599EB: 4133 case ixgbe_mac_X540: 4134 hw->mac.ops.disable_rx_buff(hw); 4135 break; 4136 default: 4137 break; 4138 } 4139 4140 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 4141 ixgbe_init_fdir_signature_82599(&adapter->hw, 4142 adapter->fdir_pballoc); 4143 } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) { 4144 ixgbe_init_fdir_perfect_82599(&adapter->hw, 4145 adapter->fdir_pballoc); 4146 ixgbe_fdir_filter_restore(adapter); 4147 } 4148 4149 switch (hw->mac.type) { 4150 case ixgbe_mac_82599EB: 4151 case ixgbe_mac_X540: 4152 hw->mac.ops.enable_rx_buff(hw); 4153 break; 4154 default: 4155 break; 4156 } 4157 4158 #ifdef IXGBE_FCOE 4159 /* configure FCoE L2 filters, redirection table, and Rx control */ 4160 ixgbe_configure_fcoe(adapter); 4161 4162 #endif /* IXGBE_FCOE */ 4163 ixgbe_configure_tx(adapter); 4164 ixgbe_configure_rx(adapter); 4165 } 4166 4167 static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw) 4168 { 4169 switch (hw->phy.type) { 4170 case ixgbe_phy_sfp_avago: 4171 case ixgbe_phy_sfp_ftl: 4172 case ixgbe_phy_sfp_intel: 4173 case ixgbe_phy_sfp_unknown: 4174 case ixgbe_phy_sfp_passive_tyco: 4175 case ixgbe_phy_sfp_passive_unknown: 4176 case ixgbe_phy_sfp_active_unknown: 4177 case ixgbe_phy_sfp_ftl_active: 4178 case ixgbe_phy_qsfp_passive_unknown: 4179 case ixgbe_phy_qsfp_active_unknown: 4180 case ixgbe_phy_qsfp_intel: 4181 case ixgbe_phy_qsfp_unknown: 4182 return true; 4183 case ixgbe_phy_nl: 4184 if (hw->mac.type == ixgbe_mac_82598EB) 4185 return true; 4186 default: 4187 return false; 4188 } 4189 } 4190 4191 /** 4192 * ixgbe_sfp_link_config - set up SFP+ link 4193 * @adapter: pointer to private adapter struct 4194 **/ 4195 static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter) 4196 { 4197 /* 4198 * We are assuming the worst case scenario here, and that 4199 * is that an SFP was inserted/removed after the reset 4200 * but before SFP detection was enabled. As such the best 4201 * solution is to just start searching as soon as we start 4202 */ 4203 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 4204 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP; 4205 4206 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; 4207 } 4208 4209 /** 4210 * ixgbe_non_sfp_link_config - set up non-SFP+ link 4211 * @hw: pointer to private hardware struct 4212 * 4213 * Returns 0 on success, negative on failure 4214 **/ 4215 static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw) 4216 { 4217 u32 speed; 4218 bool autoneg, link_up = false; 4219 u32 ret = IXGBE_ERR_LINK_SETUP; 4220 4221 if (hw->mac.ops.check_link) 4222 ret = hw->mac.ops.check_link(hw, &speed, &link_up, false); 4223 4224 if (ret) 4225 goto link_cfg_out; 4226 4227 speed = hw->phy.autoneg_advertised; 4228 if ((!speed) && (hw->mac.ops.get_link_capabilities)) 4229 ret = hw->mac.ops.get_link_capabilities(hw, &speed, 4230 &autoneg); 4231 if (ret) 4232 goto link_cfg_out; 4233 4234 if (hw->mac.ops.setup_link) 4235 ret = hw->mac.ops.setup_link(hw, speed, link_up); 4236 link_cfg_out: 4237 return ret; 4238 } 4239 4240 static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter) 4241 { 4242 struct ixgbe_hw *hw = &adapter->hw; 4243 u32 gpie = 0; 4244 4245 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 4246 gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | 4247 IXGBE_GPIE_OCD; 4248 gpie |= IXGBE_GPIE_EIAME; 4249 /* 4250 * use EIAM to auto-mask when MSI-X interrupt is asserted 4251 * this saves a register write for every interrupt 4252 */ 4253 switch (hw->mac.type) { 4254 case ixgbe_mac_82598EB: 4255 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4256 break; 4257 case ixgbe_mac_82599EB: 4258 case ixgbe_mac_X540: 4259 default: 4260 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 4261 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 4262 break; 4263 } 4264 } else { 4265 /* legacy interrupts, use EIAM to auto-mask when reading EICR, 4266 * specifically only auto mask tx and rx interrupts */ 4267 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4268 } 4269 4270 /* XXX: to interrupt immediately for EICS writes, enable this */ 4271 /* gpie |= IXGBE_GPIE_EIMEN; */ 4272 4273 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 4274 gpie &= ~IXGBE_GPIE_VTMODE_MASK; 4275 4276 switch (adapter->ring_feature[RING_F_VMDQ].mask) { 4277 case IXGBE_82599_VMDQ_8Q_MASK: 4278 gpie |= IXGBE_GPIE_VTMODE_16; 4279 break; 4280 case IXGBE_82599_VMDQ_4Q_MASK: 4281 gpie |= IXGBE_GPIE_VTMODE_32; 4282 break; 4283 default: 4284 gpie |= IXGBE_GPIE_VTMODE_64; 4285 break; 4286 } 4287 } 4288 4289 /* Enable Thermal over heat sensor interrupt */ 4290 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) { 4291 switch (adapter->hw.mac.type) { 4292 case ixgbe_mac_82599EB: 4293 gpie |= IXGBE_SDP0_GPIEN; 4294 break; 4295 case ixgbe_mac_X540: 4296 gpie |= IXGBE_EIMS_TS; 4297 break; 4298 default: 4299 break; 4300 } 4301 } 4302 4303 /* Enable fan failure interrupt */ 4304 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) 4305 gpie |= IXGBE_SDP1_GPIEN; 4306 4307 if (hw->mac.type == ixgbe_mac_82599EB) { 4308 gpie |= IXGBE_SDP1_GPIEN; 4309 gpie |= IXGBE_SDP2_GPIEN; 4310 } 4311 4312 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 4313 } 4314 4315 static void ixgbe_up_complete(struct ixgbe_adapter *adapter) 4316 { 4317 struct ixgbe_hw *hw = &adapter->hw; 4318 int err; 4319 u32 ctrl_ext; 4320 4321 ixgbe_get_hw_control(adapter); 4322 ixgbe_setup_gpie(adapter); 4323 4324 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) 4325 ixgbe_configure_msix(adapter); 4326 else 4327 ixgbe_configure_msi_and_legacy(adapter); 4328 4329 /* enable the optics for 82599 SFP+ fiber */ 4330 if (hw->mac.ops.enable_tx_laser) 4331 hw->mac.ops.enable_tx_laser(hw); 4332 4333 clear_bit(__IXGBE_DOWN, &adapter->state); 4334 ixgbe_napi_enable_all(adapter); 4335 4336 if (ixgbe_is_sfp(hw)) { 4337 ixgbe_sfp_link_config(adapter); 4338 } else { 4339 err = ixgbe_non_sfp_link_config(hw); 4340 if (err) 4341 e_err(probe, "link_config FAILED %d\n", err); 4342 } 4343 4344 /* clear any pending interrupts, may auto mask */ 4345 IXGBE_READ_REG(hw, IXGBE_EICR); 4346 ixgbe_irq_enable(adapter, true, true); 4347 4348 /* 4349 * If this adapter has a fan, check to see if we had a failure 4350 * before we enabled the interrupt. 4351 */ 4352 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) { 4353 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 4354 if (esdp & IXGBE_ESDP_SDP1) 4355 e_crit(drv, "Fan has stopped, replace the adapter\n"); 4356 } 4357 4358 /* enable transmits */ 4359 netif_tx_start_all_queues(adapter->netdev); 4360 4361 /* bring the link up in the watchdog, this could race with our first 4362 * link up interrupt but shouldn't be a problem */ 4363 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 4364 adapter->link_check_timeout = jiffies; 4365 mod_timer(&adapter->service_timer, jiffies); 4366 4367 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 4368 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4369 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 4370 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4371 } 4372 4373 void ixgbe_reinit_locked(struct ixgbe_adapter *adapter) 4374 { 4375 WARN_ON(in_interrupt()); 4376 /* put off any impending NetWatchDogTimeout */ 4377 adapter->netdev->trans_start = jiffies; 4378 4379 while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state)) 4380 usleep_range(1000, 2000); 4381 ixgbe_down(adapter); 4382 /* 4383 * If SR-IOV enabled then wait a bit before bringing the adapter 4384 * back up to give the VFs time to respond to the reset. The 4385 * two second wait is based upon the watchdog timer cycle in 4386 * the VF driver. 4387 */ 4388 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 4389 msleep(2000); 4390 ixgbe_up(adapter); 4391 clear_bit(__IXGBE_RESETTING, &adapter->state); 4392 } 4393 4394 void ixgbe_up(struct ixgbe_adapter *adapter) 4395 { 4396 /* hardware has been reset, we need to reload some things */ 4397 ixgbe_configure(adapter); 4398 4399 ixgbe_up_complete(adapter); 4400 } 4401 4402 void ixgbe_reset(struct ixgbe_adapter *adapter) 4403 { 4404 struct ixgbe_hw *hw = &adapter->hw; 4405 int err; 4406 4407 /* lock SFP init bit to prevent race conditions with the watchdog */ 4408 while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) 4409 usleep_range(1000, 2000); 4410 4411 /* clear all SFP and link config related flags while holding SFP_INIT */ 4412 adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP | 4413 IXGBE_FLAG2_SFP_NEEDS_RESET); 4414 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; 4415 4416 err = hw->mac.ops.init_hw(hw); 4417 switch (err) { 4418 case 0: 4419 case IXGBE_ERR_SFP_NOT_PRESENT: 4420 case IXGBE_ERR_SFP_NOT_SUPPORTED: 4421 break; 4422 case IXGBE_ERR_MASTER_REQUESTS_PENDING: 4423 e_dev_err("master disable timed out\n"); 4424 break; 4425 case IXGBE_ERR_EEPROM_VERSION: 4426 /* We are running on a pre-production device, log a warning */ 4427 e_dev_warn("This device is a pre-production adapter/LOM. " 4428 "Please be aware there may be issues associated with " 4429 "your hardware. If you are experiencing problems " 4430 "please contact your Intel or hardware " 4431 "representative who provided you with this " 4432 "hardware.\n"); 4433 break; 4434 default: 4435 e_dev_err("Hardware Error: %d\n", err); 4436 } 4437 4438 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); 4439 4440 /* reprogram the RAR[0] in case user changed it. */ 4441 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, VMDQ_P(0), IXGBE_RAH_AV); 4442 4443 /* update SAN MAC vmdq pool selection */ 4444 if (hw->mac.san_mac_rar_index) 4445 hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0)); 4446 4447 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) 4448 ixgbe_ptp_reset(adapter); 4449 } 4450 4451 /** 4452 * ixgbe_clean_rx_ring - Free Rx Buffers per Queue 4453 * @rx_ring: ring to free buffers from 4454 **/ 4455 static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) 4456 { 4457 struct device *dev = rx_ring->dev; 4458 unsigned long size; 4459 u16 i; 4460 4461 /* ring already cleared, nothing to do */ 4462 if (!rx_ring->rx_buffer_info) 4463 return; 4464 4465 /* Free all the Rx ring sk_buffs */ 4466 for (i = 0; i < rx_ring->count; i++) { 4467 struct ixgbe_rx_buffer *rx_buffer; 4468 4469 rx_buffer = &rx_ring->rx_buffer_info[i]; 4470 if (rx_buffer->skb) { 4471 struct sk_buff *skb = rx_buffer->skb; 4472 if (IXGBE_CB(skb)->page_released) { 4473 dma_unmap_page(dev, 4474 IXGBE_CB(skb)->dma, 4475 ixgbe_rx_bufsz(rx_ring), 4476 DMA_FROM_DEVICE); 4477 IXGBE_CB(skb)->page_released = false; 4478 } 4479 dev_kfree_skb(skb); 4480 } 4481 rx_buffer->skb = NULL; 4482 if (rx_buffer->dma) 4483 dma_unmap_page(dev, rx_buffer->dma, 4484 ixgbe_rx_pg_size(rx_ring), 4485 DMA_FROM_DEVICE); 4486 rx_buffer->dma = 0; 4487 if (rx_buffer->page) 4488 __free_pages(rx_buffer->page, 4489 ixgbe_rx_pg_order(rx_ring)); 4490 rx_buffer->page = NULL; 4491 } 4492 4493 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; 4494 memset(rx_ring->rx_buffer_info, 0, size); 4495 4496 /* Zero out the descriptor ring */ 4497 memset(rx_ring->desc, 0, rx_ring->size); 4498 4499 rx_ring->next_to_alloc = 0; 4500 rx_ring->next_to_clean = 0; 4501 rx_ring->next_to_use = 0; 4502 } 4503 4504 /** 4505 * ixgbe_clean_tx_ring - Free Tx Buffers 4506 * @tx_ring: ring to be cleaned 4507 **/ 4508 static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) 4509 { 4510 struct ixgbe_tx_buffer *tx_buffer_info; 4511 unsigned long size; 4512 u16 i; 4513 4514 /* ring already cleared, nothing to do */ 4515 if (!tx_ring->tx_buffer_info) 4516 return; 4517 4518 /* Free all the Tx ring sk_buffs */ 4519 for (i = 0; i < tx_ring->count; i++) { 4520 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 4521 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 4522 } 4523 4524 netdev_tx_reset_queue(txring_txq(tx_ring)); 4525 4526 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; 4527 memset(tx_ring->tx_buffer_info, 0, size); 4528 4529 /* Zero out the descriptor ring */ 4530 memset(tx_ring->desc, 0, tx_ring->size); 4531 4532 tx_ring->next_to_use = 0; 4533 tx_ring->next_to_clean = 0; 4534 } 4535 4536 /** 4537 * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues 4538 * @adapter: board private structure 4539 **/ 4540 static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter) 4541 { 4542 int i; 4543 4544 for (i = 0; i < adapter->num_rx_queues; i++) 4545 ixgbe_clean_rx_ring(adapter->rx_ring[i]); 4546 } 4547 4548 /** 4549 * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues 4550 * @adapter: board private structure 4551 **/ 4552 static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter) 4553 { 4554 int i; 4555 4556 for (i = 0; i < adapter->num_tx_queues; i++) 4557 ixgbe_clean_tx_ring(adapter->tx_ring[i]); 4558 } 4559 4560 static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter) 4561 { 4562 struct hlist_node *node2; 4563 struct ixgbe_fdir_filter *filter; 4564 4565 spin_lock(&adapter->fdir_perfect_lock); 4566 4567 hlist_for_each_entry_safe(filter, node2, 4568 &adapter->fdir_filter_list, fdir_node) { 4569 hlist_del(&filter->fdir_node); 4570 kfree(filter); 4571 } 4572 adapter->fdir_filter_count = 0; 4573 4574 spin_unlock(&adapter->fdir_perfect_lock); 4575 } 4576 4577 void ixgbe_down(struct ixgbe_adapter *adapter) 4578 { 4579 struct net_device *netdev = adapter->netdev; 4580 struct ixgbe_hw *hw = &adapter->hw; 4581 u32 rxctrl; 4582 int i; 4583 4584 /* signal that we are down to the interrupt handler */ 4585 set_bit(__IXGBE_DOWN, &adapter->state); 4586 4587 /* disable receives */ 4588 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4589 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); 4590 4591 /* disable all enabled rx queues */ 4592 for (i = 0; i < adapter->num_rx_queues; i++) 4593 /* this call also flushes the previous write */ 4594 ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]); 4595 4596 usleep_range(10000, 20000); 4597 4598 netif_tx_stop_all_queues(netdev); 4599 4600 /* call carrier off first to avoid false dev_watchdog timeouts */ 4601 netif_carrier_off(netdev); 4602 netif_tx_disable(netdev); 4603 4604 ixgbe_irq_disable(adapter); 4605 4606 ixgbe_napi_disable_all(adapter); 4607 4608 adapter->flags2 &= ~(IXGBE_FLAG2_FDIR_REQUIRES_REINIT | 4609 IXGBE_FLAG2_RESET_REQUESTED); 4610 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; 4611 4612 del_timer_sync(&adapter->service_timer); 4613 4614 if (adapter->num_vfs) { 4615 /* Clear EITR Select mapping */ 4616 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0); 4617 4618 /* Mark all the VFs as inactive */ 4619 for (i = 0 ; i < adapter->num_vfs; i++) 4620 adapter->vfinfo[i].clear_to_send = false; 4621 4622 /* ping all the active vfs to let them know we are going down */ 4623 ixgbe_ping_all_vfs(adapter); 4624 4625 /* Disable all VFTE/VFRE TX/RX */ 4626 ixgbe_disable_tx_rx(adapter); 4627 } 4628 4629 /* disable transmits in the hardware now that interrupts are off */ 4630 for (i = 0; i < adapter->num_tx_queues; i++) { 4631 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 4632 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 4633 } 4634 4635 /* Disable the Tx DMA engine on 82599 and X540 */ 4636 switch (hw->mac.type) { 4637 case ixgbe_mac_82599EB: 4638 case ixgbe_mac_X540: 4639 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, 4640 (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) & 4641 ~IXGBE_DMATXCTL_TE)); 4642 break; 4643 default: 4644 break; 4645 } 4646 4647 if (!pci_channel_offline(adapter->pdev)) 4648 ixgbe_reset(adapter); 4649 4650 /* power down the optics for 82599 SFP+ fiber */ 4651 if (hw->mac.ops.disable_tx_laser) 4652 hw->mac.ops.disable_tx_laser(hw); 4653 4654 ixgbe_clean_all_tx_rings(adapter); 4655 ixgbe_clean_all_rx_rings(adapter); 4656 4657 #ifdef CONFIG_IXGBE_DCA 4658 /* since we reset the hardware DCA settings were cleared */ 4659 ixgbe_setup_dca(adapter); 4660 #endif 4661 } 4662 4663 /** 4664 * ixgbe_tx_timeout - Respond to a Tx Hang 4665 * @netdev: network interface device structure 4666 **/ 4667 static void ixgbe_tx_timeout(struct net_device *netdev) 4668 { 4669 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4670 4671 /* Do the reset outside of interrupt context */ 4672 ixgbe_tx_timeout_reset(adapter); 4673 } 4674 4675 /** 4676 * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter) 4677 * @adapter: board private structure to initialize 4678 * 4679 * ixgbe_sw_init initializes the Adapter private data structure. 4680 * Fields are initialized based on PCI device information and 4681 * OS network device settings (MTU size). 4682 **/ 4683 static int ixgbe_sw_init(struct ixgbe_adapter *adapter) 4684 { 4685 struct ixgbe_hw *hw = &adapter->hw; 4686 struct pci_dev *pdev = adapter->pdev; 4687 unsigned int rss, fdir; 4688 u32 fwsm; 4689 #ifdef CONFIG_IXGBE_DCB 4690 int j; 4691 struct tc_configuration *tc; 4692 #endif 4693 4694 /* PCI config space info */ 4695 4696 hw->vendor_id = pdev->vendor; 4697 hw->device_id = pdev->device; 4698 hw->revision_id = pdev->revision; 4699 hw->subsystem_vendor_id = pdev->subsystem_vendor; 4700 hw->subsystem_device_id = pdev->subsystem_device; 4701 4702 /* Set common capability flags and settings */ 4703 rss = min_t(int, IXGBE_MAX_RSS_INDICES, num_online_cpus()); 4704 adapter->ring_feature[RING_F_RSS].limit = rss; 4705 adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE; 4706 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; 4707 adapter->max_q_vectors = MAX_Q_VECTORS_82599; 4708 adapter->atr_sample_rate = 20; 4709 fdir = min_t(int, IXGBE_MAX_FDIR_INDICES, num_online_cpus()); 4710 adapter->ring_feature[RING_F_FDIR].limit = fdir; 4711 adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K; 4712 #ifdef CONFIG_IXGBE_DCA 4713 adapter->flags |= IXGBE_FLAG_DCA_CAPABLE; 4714 #endif 4715 #ifdef IXGBE_FCOE 4716 adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE; 4717 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED; 4718 #ifdef CONFIG_IXGBE_DCB 4719 /* Default traffic class to use for FCoE */ 4720 adapter->fcoe.up = IXGBE_FCOE_DEFTC; 4721 #endif /* CONFIG_IXGBE_DCB */ 4722 #endif /* IXGBE_FCOE */ 4723 4724 /* Set MAC specific capability flags and exceptions */ 4725 switch (hw->mac.type) { 4726 case ixgbe_mac_82598EB: 4727 adapter->flags2 &= ~IXGBE_FLAG2_RSC_CAPABLE; 4728 adapter->flags2 &= ~IXGBE_FLAG2_RSC_ENABLED; 4729 4730 if (hw->device_id == IXGBE_DEV_ID_82598AT) 4731 adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; 4732 4733 adapter->max_q_vectors = MAX_Q_VECTORS_82598; 4734 adapter->ring_feature[RING_F_FDIR].limit = 0; 4735 adapter->atr_sample_rate = 0; 4736 adapter->fdir_pballoc = 0; 4737 #ifdef IXGBE_FCOE 4738 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE; 4739 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED; 4740 #ifdef CONFIG_IXGBE_DCB 4741 adapter->fcoe.up = 0; 4742 #endif /* IXGBE_DCB */ 4743 #endif /* IXGBE_FCOE */ 4744 break; 4745 case ixgbe_mac_82599EB: 4746 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) 4747 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; 4748 break; 4749 case ixgbe_mac_X540: 4750 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 4751 if (fwsm & IXGBE_FWSM_TS_ENABLED) 4752 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; 4753 break; 4754 default: 4755 break; 4756 } 4757 4758 #ifdef IXGBE_FCOE 4759 /* FCoE support exists, always init the FCoE lock */ 4760 spin_lock_init(&adapter->fcoe.lock); 4761 4762 #endif 4763 /* n-tuple support exists, always init our spinlock */ 4764 spin_lock_init(&adapter->fdir_perfect_lock); 4765 4766 #ifdef CONFIG_IXGBE_DCB 4767 switch (hw->mac.type) { 4768 case ixgbe_mac_X540: 4769 adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS; 4770 adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS; 4771 break; 4772 default: 4773 adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS; 4774 adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS; 4775 break; 4776 } 4777 4778 /* Configure DCB traffic classes */ 4779 for (j = 0; j < MAX_TRAFFIC_CLASS; j++) { 4780 tc = &adapter->dcb_cfg.tc_config[j]; 4781 tc->path[DCB_TX_CONFIG].bwg_id = 0; 4782 tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1); 4783 tc->path[DCB_RX_CONFIG].bwg_id = 0; 4784 tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1); 4785 tc->dcb_pfc = pfc_disabled; 4786 } 4787 4788 /* Initialize default user to priority mapping, UPx->TC0 */ 4789 tc = &adapter->dcb_cfg.tc_config[0]; 4790 tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF; 4791 tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF; 4792 4793 adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100; 4794 adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100; 4795 adapter->dcb_cfg.pfc_mode_enable = false; 4796 adapter->dcb_set_bitmap = 0x00; 4797 adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE; 4798 memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg, 4799 sizeof(adapter->temp_dcb_cfg)); 4800 4801 #endif 4802 4803 /* default flow control settings */ 4804 hw->fc.requested_mode = ixgbe_fc_full; 4805 hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */ 4806 ixgbe_pbthresh_setup(adapter); 4807 hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE; 4808 hw->fc.send_xon = true; 4809 hw->fc.disable_fc_autoneg = ixgbe_device_supports_autoneg_fc(hw); 4810 4811 #ifdef CONFIG_PCI_IOV 4812 /* assign number of SR-IOV VFs */ 4813 if (hw->mac.type != ixgbe_mac_82598EB) 4814 adapter->num_vfs = (max_vfs > 63) ? 0 : max_vfs; 4815 4816 #endif 4817 /* enable itr by default in dynamic mode */ 4818 adapter->rx_itr_setting = 1; 4819 adapter->tx_itr_setting = 1; 4820 4821 /* set default ring sizes */ 4822 adapter->tx_ring_count = IXGBE_DEFAULT_TXD; 4823 adapter->rx_ring_count = IXGBE_DEFAULT_RXD; 4824 4825 /* set default work limits */ 4826 adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK; 4827 4828 /* initialize eeprom parameters */ 4829 if (ixgbe_init_eeprom_params_generic(hw)) { 4830 e_dev_err("EEPROM initialization failed\n"); 4831 return -EIO; 4832 } 4833 4834 set_bit(__IXGBE_DOWN, &adapter->state); 4835 4836 return 0; 4837 } 4838 4839 /** 4840 * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors) 4841 * @tx_ring: tx descriptor ring (for a specific queue) to setup 4842 * 4843 * Return 0 on success, negative on failure 4844 **/ 4845 int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring) 4846 { 4847 struct device *dev = tx_ring->dev; 4848 int orig_node = dev_to_node(dev); 4849 int numa_node = -1; 4850 int size; 4851 4852 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; 4853 4854 if (tx_ring->q_vector) 4855 numa_node = tx_ring->q_vector->numa_node; 4856 4857 tx_ring->tx_buffer_info = vzalloc_node(size, numa_node); 4858 if (!tx_ring->tx_buffer_info) 4859 tx_ring->tx_buffer_info = vzalloc(size); 4860 if (!tx_ring->tx_buffer_info) 4861 goto err; 4862 4863 /* round up to nearest 4K */ 4864 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 4865 tx_ring->size = ALIGN(tx_ring->size, 4096); 4866 4867 set_dev_node(dev, numa_node); 4868 tx_ring->desc = dma_alloc_coherent(dev, 4869 tx_ring->size, 4870 &tx_ring->dma, 4871 GFP_KERNEL); 4872 set_dev_node(dev, orig_node); 4873 if (!tx_ring->desc) 4874 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 4875 &tx_ring->dma, GFP_KERNEL); 4876 if (!tx_ring->desc) 4877 goto err; 4878 4879 tx_ring->next_to_use = 0; 4880 tx_ring->next_to_clean = 0; 4881 return 0; 4882 4883 err: 4884 vfree(tx_ring->tx_buffer_info); 4885 tx_ring->tx_buffer_info = NULL; 4886 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n"); 4887 return -ENOMEM; 4888 } 4889 4890 /** 4891 * ixgbe_setup_all_tx_resources - allocate all queues Tx resources 4892 * @adapter: board private structure 4893 * 4894 * If this function returns with an error, then it's possible one or 4895 * more of the rings is populated (while the rest are not). It is the 4896 * callers duty to clean those orphaned rings. 4897 * 4898 * Return 0 on success, negative on failure 4899 **/ 4900 static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) 4901 { 4902 int i, err = 0; 4903 4904 for (i = 0; i < adapter->num_tx_queues; i++) { 4905 err = ixgbe_setup_tx_resources(adapter->tx_ring[i]); 4906 if (!err) 4907 continue; 4908 4909 e_err(probe, "Allocation for Tx Queue %u failed\n", i); 4910 goto err_setup_tx; 4911 } 4912 4913 return 0; 4914 err_setup_tx: 4915 /* rewind the index freeing the rings as we go */ 4916 while (i--) 4917 ixgbe_free_tx_resources(adapter->tx_ring[i]); 4918 return err; 4919 } 4920 4921 /** 4922 * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) 4923 * @rx_ring: rx descriptor ring (for a specific queue) to setup 4924 * 4925 * Returns 0 on success, negative on failure 4926 **/ 4927 int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring) 4928 { 4929 struct device *dev = rx_ring->dev; 4930 int orig_node = dev_to_node(dev); 4931 int numa_node = -1; 4932 int size; 4933 4934 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; 4935 4936 if (rx_ring->q_vector) 4937 numa_node = rx_ring->q_vector->numa_node; 4938 4939 rx_ring->rx_buffer_info = vzalloc_node(size, numa_node); 4940 if (!rx_ring->rx_buffer_info) 4941 rx_ring->rx_buffer_info = vzalloc(size); 4942 if (!rx_ring->rx_buffer_info) 4943 goto err; 4944 4945 /* Round up to nearest 4K */ 4946 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 4947 rx_ring->size = ALIGN(rx_ring->size, 4096); 4948 4949 set_dev_node(dev, numa_node); 4950 rx_ring->desc = dma_alloc_coherent(dev, 4951 rx_ring->size, 4952 &rx_ring->dma, 4953 GFP_KERNEL); 4954 set_dev_node(dev, orig_node); 4955 if (!rx_ring->desc) 4956 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 4957 &rx_ring->dma, GFP_KERNEL); 4958 if (!rx_ring->desc) 4959 goto err; 4960 4961 rx_ring->next_to_clean = 0; 4962 rx_ring->next_to_use = 0; 4963 4964 return 0; 4965 err: 4966 vfree(rx_ring->rx_buffer_info); 4967 rx_ring->rx_buffer_info = NULL; 4968 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); 4969 return -ENOMEM; 4970 } 4971 4972 /** 4973 * ixgbe_setup_all_rx_resources - allocate all queues Rx resources 4974 * @adapter: board private structure 4975 * 4976 * If this function returns with an error, then it's possible one or 4977 * more of the rings is populated (while the rest are not). It is the 4978 * callers duty to clean those orphaned rings. 4979 * 4980 * Return 0 on success, negative on failure 4981 **/ 4982 static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter) 4983 { 4984 int i, err = 0; 4985 4986 for (i = 0; i < adapter->num_rx_queues; i++) { 4987 err = ixgbe_setup_rx_resources(adapter->rx_ring[i]); 4988 if (!err) 4989 continue; 4990 4991 e_err(probe, "Allocation for Rx Queue %u failed\n", i); 4992 goto err_setup_rx; 4993 } 4994 4995 #ifdef IXGBE_FCOE 4996 err = ixgbe_setup_fcoe_ddp_resources(adapter); 4997 if (!err) 4998 #endif 4999 return 0; 5000 err_setup_rx: 5001 /* rewind the index freeing the rings as we go */ 5002 while (i--) 5003 ixgbe_free_rx_resources(adapter->rx_ring[i]); 5004 return err; 5005 } 5006 5007 /** 5008 * ixgbe_free_tx_resources - Free Tx Resources per Queue 5009 * @tx_ring: Tx descriptor ring for a specific queue 5010 * 5011 * Free all transmit software resources 5012 **/ 5013 void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring) 5014 { 5015 ixgbe_clean_tx_ring(tx_ring); 5016 5017 vfree(tx_ring->tx_buffer_info); 5018 tx_ring->tx_buffer_info = NULL; 5019 5020 /* if not set, then don't free */ 5021 if (!tx_ring->desc) 5022 return; 5023 5024 dma_free_coherent(tx_ring->dev, tx_ring->size, 5025 tx_ring->desc, tx_ring->dma); 5026 5027 tx_ring->desc = NULL; 5028 } 5029 5030 /** 5031 * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues 5032 * @adapter: board private structure 5033 * 5034 * Free all transmit software resources 5035 **/ 5036 static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) 5037 { 5038 int i; 5039 5040 for (i = 0; i < adapter->num_tx_queues; i++) 5041 if (adapter->tx_ring[i]->desc) 5042 ixgbe_free_tx_resources(adapter->tx_ring[i]); 5043 } 5044 5045 /** 5046 * ixgbe_free_rx_resources - Free Rx Resources 5047 * @rx_ring: ring to clean the resources from 5048 * 5049 * Free all receive software resources 5050 **/ 5051 void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) 5052 { 5053 ixgbe_clean_rx_ring(rx_ring); 5054 5055 vfree(rx_ring->rx_buffer_info); 5056 rx_ring->rx_buffer_info = NULL; 5057 5058 /* if not set, then don't free */ 5059 if (!rx_ring->desc) 5060 return; 5061 5062 dma_free_coherent(rx_ring->dev, rx_ring->size, 5063 rx_ring->desc, rx_ring->dma); 5064 5065 rx_ring->desc = NULL; 5066 } 5067 5068 /** 5069 * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues 5070 * @adapter: board private structure 5071 * 5072 * Free all receive software resources 5073 **/ 5074 static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) 5075 { 5076 int i; 5077 5078 #ifdef IXGBE_FCOE 5079 ixgbe_free_fcoe_ddp_resources(adapter); 5080 5081 #endif 5082 for (i = 0; i < adapter->num_rx_queues; i++) 5083 if (adapter->rx_ring[i]->desc) 5084 ixgbe_free_rx_resources(adapter->rx_ring[i]); 5085 } 5086 5087 /** 5088 * ixgbe_change_mtu - Change the Maximum Transfer Unit 5089 * @netdev: network interface device structure 5090 * @new_mtu: new value for maximum frame size 5091 * 5092 * Returns 0 on success, negative on failure 5093 **/ 5094 static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) 5095 { 5096 struct ixgbe_adapter *adapter = netdev_priv(netdev); 5097 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 5098 5099 /* MTU < 68 is an error and causes problems on some kernels */ 5100 if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE)) 5101 return -EINVAL; 5102 5103 /* 5104 * For 82599EB we cannot allow legacy VFs to enable their receive 5105 * paths when MTU greater than 1500 is configured. So display a 5106 * warning that legacy VFs will be disabled. 5107 */ 5108 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && 5109 (adapter->hw.mac.type == ixgbe_mac_82599EB) && 5110 (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN))) 5111 e_warn(probe, "Setting MTU > 1500 will disable legacy VFs\n"); 5112 5113 e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 5114 5115 /* must set new MTU before calling down or up */ 5116 netdev->mtu = new_mtu; 5117 5118 if (netif_running(netdev)) 5119 ixgbe_reinit_locked(adapter); 5120 5121 return 0; 5122 } 5123 5124 /** 5125 * ixgbe_open - Called when a network interface is made active 5126 * @netdev: network interface device structure 5127 * 5128 * Returns 0 on success, negative value on failure 5129 * 5130 * The open entry point is called when a network interface is made 5131 * active by the system (IFF_UP). At this point all resources needed 5132 * for transmit and receive operations are allocated, the interrupt 5133 * handler is registered with the OS, the watchdog timer is started, 5134 * and the stack is notified that the interface is ready. 5135 **/ 5136 static int ixgbe_open(struct net_device *netdev) 5137 { 5138 struct ixgbe_adapter *adapter = netdev_priv(netdev); 5139 int err; 5140 5141 /* disallow open during test */ 5142 if (test_bit(__IXGBE_TESTING, &adapter->state)) 5143 return -EBUSY; 5144 5145 netif_carrier_off(netdev); 5146 5147 /* allocate transmit descriptors */ 5148 err = ixgbe_setup_all_tx_resources(adapter); 5149 if (err) 5150 goto err_setup_tx; 5151 5152 /* allocate receive descriptors */ 5153 err = ixgbe_setup_all_rx_resources(adapter); 5154 if (err) 5155 goto err_setup_rx; 5156 5157 ixgbe_configure(adapter); 5158 5159 err = ixgbe_request_irq(adapter); 5160 if (err) 5161 goto err_req_irq; 5162 5163 /* Notify the stack of the actual queue counts. */ 5164 err = netif_set_real_num_tx_queues(netdev, 5165 adapter->num_rx_pools > 1 ? 1 : 5166 adapter->num_tx_queues); 5167 if (err) 5168 goto err_set_queues; 5169 5170 5171 err = netif_set_real_num_rx_queues(netdev, 5172 adapter->num_rx_pools > 1 ? 1 : 5173 adapter->num_rx_queues); 5174 if (err) 5175 goto err_set_queues; 5176 5177 ixgbe_ptp_init(adapter); 5178 5179 ixgbe_up_complete(adapter); 5180 5181 return 0; 5182 5183 err_set_queues: 5184 ixgbe_free_irq(adapter); 5185 err_req_irq: 5186 ixgbe_free_all_rx_resources(adapter); 5187 err_setup_rx: 5188 ixgbe_free_all_tx_resources(adapter); 5189 err_setup_tx: 5190 ixgbe_reset(adapter); 5191 5192 return err; 5193 } 5194 5195 /** 5196 * ixgbe_close - Disables a network interface 5197 * @netdev: network interface device structure 5198 * 5199 * Returns 0, this is not allowed to fail 5200 * 5201 * The close entry point is called when an interface is de-activated 5202 * by the OS. The hardware is still under the drivers control, but 5203 * needs to be disabled. A global MAC reset is issued to stop the 5204 * hardware, and all transmit and receive resources are freed. 5205 **/ 5206 static int ixgbe_close(struct net_device *netdev) 5207 { 5208 struct ixgbe_adapter *adapter = netdev_priv(netdev); 5209 5210 ixgbe_ptp_stop(adapter); 5211 5212 ixgbe_down(adapter); 5213 ixgbe_free_irq(adapter); 5214 5215 ixgbe_fdir_filter_exit(adapter); 5216 5217 ixgbe_free_all_tx_resources(adapter); 5218 ixgbe_free_all_rx_resources(adapter); 5219 5220 ixgbe_release_hw_control(adapter); 5221 5222 return 0; 5223 } 5224 5225 #ifdef CONFIG_PM 5226 static int ixgbe_resume(struct pci_dev *pdev) 5227 { 5228 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 5229 struct net_device *netdev = adapter->netdev; 5230 u32 err; 5231 5232 pci_set_power_state(pdev, PCI_D0); 5233 pci_restore_state(pdev); 5234 /* 5235 * pci_restore_state clears dev->state_saved so call 5236 * pci_save_state to restore it. 5237 */ 5238 pci_save_state(pdev); 5239 5240 err = pci_enable_device_mem(pdev); 5241 if (err) { 5242 e_dev_err("Cannot enable PCI device from suspend\n"); 5243 return err; 5244 } 5245 pci_set_master(pdev); 5246 5247 pci_wake_from_d3(pdev, false); 5248 5249 ixgbe_reset(adapter); 5250 5251 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); 5252 5253 rtnl_lock(); 5254 err = ixgbe_init_interrupt_scheme(adapter); 5255 if (!err && netif_running(netdev)) 5256 err = ixgbe_open(netdev); 5257 5258 rtnl_unlock(); 5259 5260 if (err) 5261 return err; 5262 5263 netif_device_attach(netdev); 5264 5265 return 0; 5266 } 5267 #endif /* CONFIG_PM */ 5268 5269 static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake) 5270 { 5271 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 5272 struct net_device *netdev = adapter->netdev; 5273 struct ixgbe_hw *hw = &adapter->hw; 5274 u32 ctrl, fctrl; 5275 u32 wufc = adapter->wol; 5276 #ifdef CONFIG_PM 5277 int retval = 0; 5278 #endif 5279 5280 netif_device_detach(netdev); 5281 5282 rtnl_lock(); 5283 if (netif_running(netdev)) { 5284 ixgbe_down(adapter); 5285 ixgbe_free_irq(adapter); 5286 ixgbe_free_all_tx_resources(adapter); 5287 ixgbe_free_all_rx_resources(adapter); 5288 } 5289 rtnl_unlock(); 5290 5291 ixgbe_clear_interrupt_scheme(adapter); 5292 5293 #ifdef CONFIG_PM 5294 retval = pci_save_state(pdev); 5295 if (retval) 5296 return retval; 5297 5298 #endif 5299 if (hw->mac.ops.stop_link_on_d3) 5300 hw->mac.ops.stop_link_on_d3(hw); 5301 5302 if (wufc) { 5303 ixgbe_set_rx_mode(netdev); 5304 5305 /* enable the optics for 82599 SFP+ fiber as we can WoL */ 5306 if (hw->mac.ops.enable_tx_laser) 5307 hw->mac.ops.enable_tx_laser(hw); 5308 5309 /* turn on all-multi mode if wake on multicast is enabled */ 5310 if (wufc & IXGBE_WUFC_MC) { 5311 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 5312 fctrl |= IXGBE_FCTRL_MPE; 5313 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 5314 } 5315 5316 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL); 5317 ctrl |= IXGBE_CTRL_GIO_DIS; 5318 IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl); 5319 5320 IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc); 5321 } else { 5322 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 5323 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 5324 } 5325 5326 switch (hw->mac.type) { 5327 case ixgbe_mac_82598EB: 5328 pci_wake_from_d3(pdev, false); 5329 break; 5330 case ixgbe_mac_82599EB: 5331 case ixgbe_mac_X540: 5332 pci_wake_from_d3(pdev, !!wufc); 5333 break; 5334 default: 5335 break; 5336 } 5337 5338 *enable_wake = !!wufc; 5339 5340 ixgbe_release_hw_control(adapter); 5341 5342 pci_disable_device(pdev); 5343 5344 return 0; 5345 } 5346 5347 #ifdef CONFIG_PM 5348 static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state) 5349 { 5350 int retval; 5351 bool wake; 5352 5353 retval = __ixgbe_shutdown(pdev, &wake); 5354 if (retval) 5355 return retval; 5356 5357 if (wake) { 5358 pci_prepare_to_sleep(pdev); 5359 } else { 5360 pci_wake_from_d3(pdev, false); 5361 pci_set_power_state(pdev, PCI_D3hot); 5362 } 5363 5364 return 0; 5365 } 5366 #endif /* CONFIG_PM */ 5367 5368 static void ixgbe_shutdown(struct pci_dev *pdev) 5369 { 5370 bool wake; 5371 5372 __ixgbe_shutdown(pdev, &wake); 5373 5374 if (system_state == SYSTEM_POWER_OFF) { 5375 pci_wake_from_d3(pdev, wake); 5376 pci_set_power_state(pdev, PCI_D3hot); 5377 } 5378 } 5379 5380 /** 5381 * ixgbe_update_stats - Update the board statistics counters. 5382 * @adapter: board private structure 5383 **/ 5384 void ixgbe_update_stats(struct ixgbe_adapter *adapter) 5385 { 5386 struct net_device *netdev = adapter->netdev; 5387 struct ixgbe_hw *hw = &adapter->hw; 5388 struct ixgbe_hw_stats *hwstats = &adapter->stats; 5389 u64 total_mpc = 0; 5390 u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot; 5391 u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0; 5392 u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0; 5393 u64 bytes = 0, packets = 0, hw_csum_rx_error = 0; 5394 5395 if (test_bit(__IXGBE_DOWN, &adapter->state) || 5396 test_bit(__IXGBE_RESETTING, &adapter->state)) 5397 return; 5398 5399 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) { 5400 u64 rsc_count = 0; 5401 u64 rsc_flush = 0; 5402 for (i = 0; i < adapter->num_rx_queues; i++) { 5403 rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count; 5404 rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush; 5405 } 5406 adapter->rsc_total_count = rsc_count; 5407 adapter->rsc_total_flush = rsc_flush; 5408 } 5409 5410 for (i = 0; i < adapter->num_rx_queues; i++) { 5411 struct ixgbe_ring *rx_ring = adapter->rx_ring[i]; 5412 non_eop_descs += rx_ring->rx_stats.non_eop_descs; 5413 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed; 5414 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed; 5415 hw_csum_rx_error += rx_ring->rx_stats.csum_err; 5416 bytes += rx_ring->stats.bytes; 5417 packets += rx_ring->stats.packets; 5418 } 5419 adapter->non_eop_descs = non_eop_descs; 5420 adapter->alloc_rx_page_failed = alloc_rx_page_failed; 5421 adapter->alloc_rx_buff_failed = alloc_rx_buff_failed; 5422 adapter->hw_csum_rx_error = hw_csum_rx_error; 5423 netdev->stats.rx_bytes = bytes; 5424 netdev->stats.rx_packets = packets; 5425 5426 bytes = 0; 5427 packets = 0; 5428 /* gather some stats to the adapter struct that are per queue */ 5429 for (i = 0; i < adapter->num_tx_queues; i++) { 5430 struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; 5431 restart_queue += tx_ring->tx_stats.restart_queue; 5432 tx_busy += tx_ring->tx_stats.tx_busy; 5433 bytes += tx_ring->stats.bytes; 5434 packets += tx_ring->stats.packets; 5435 } 5436 adapter->restart_queue = restart_queue; 5437 adapter->tx_busy = tx_busy; 5438 netdev->stats.tx_bytes = bytes; 5439 netdev->stats.tx_packets = packets; 5440 5441 hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 5442 5443 /* 8 register reads */ 5444 for (i = 0; i < 8; i++) { 5445 /* for packet buffers not used, the register should read 0 */ 5446 mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 5447 missed_rx += mpc; 5448 hwstats->mpc[i] += mpc; 5449 total_mpc += hwstats->mpc[i]; 5450 hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 5451 hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 5452 switch (hw->mac.type) { 5453 case ixgbe_mac_82598EB: 5454 hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 5455 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 5456 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 5457 hwstats->pxonrxc[i] += 5458 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 5459 break; 5460 case ixgbe_mac_82599EB: 5461 case ixgbe_mac_X540: 5462 hwstats->pxonrxc[i] += 5463 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i)); 5464 break; 5465 default: 5466 break; 5467 } 5468 } 5469 5470 /*16 register reads */ 5471 for (i = 0; i < 16; i++) { 5472 hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 5473 hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 5474 if ((hw->mac.type == ixgbe_mac_82599EB) || 5475 (hw->mac.type == ixgbe_mac_X540)) { 5476 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)); 5477 IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */ 5478 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)); 5479 IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */ 5480 } 5481 } 5482 5483 hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 5484 /* work around hardware counting issue */ 5485 hwstats->gprc -= missed_rx; 5486 5487 ixgbe_update_xoff_received(adapter); 5488 5489 /* 82598 hardware only has a 32 bit counter in the high register */ 5490 switch (hw->mac.type) { 5491 case ixgbe_mac_82598EB: 5492 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 5493 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 5494 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 5495 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH); 5496 break; 5497 case ixgbe_mac_X540: 5498 /* OS2BMC stats are X540 only*/ 5499 hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC); 5500 hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC); 5501 hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC); 5502 hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC); 5503 case ixgbe_mac_82599EB: 5504 for (i = 0; i < 16; i++) 5505 adapter->hw_rx_no_dma_resources += 5506 IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 5507 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); 5508 IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */ 5509 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); 5510 IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */ 5511 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL); 5512 IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */ 5513 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 5514 hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH); 5515 hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS); 5516 #ifdef IXGBE_FCOE 5517 hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); 5518 hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 5519 hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 5520 hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 5521 hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 5522 hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 5523 /* Add up per cpu counters for total ddp aloc fail */ 5524 if (adapter->fcoe.ddp_pool) { 5525 struct ixgbe_fcoe *fcoe = &adapter->fcoe; 5526 struct ixgbe_fcoe_ddp_pool *ddp_pool; 5527 unsigned int cpu; 5528 u64 noddp = 0, noddp_ext_buff = 0; 5529 for_each_possible_cpu(cpu) { 5530 ddp_pool = per_cpu_ptr(fcoe->ddp_pool, cpu); 5531 noddp += ddp_pool->noddp; 5532 noddp_ext_buff += ddp_pool->noddp_ext_buff; 5533 } 5534 hwstats->fcoe_noddp = noddp; 5535 hwstats->fcoe_noddp_ext_buff = noddp_ext_buff; 5536 } 5537 #endif /* IXGBE_FCOE */ 5538 break; 5539 default: 5540 break; 5541 } 5542 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 5543 hwstats->bprc += bprc; 5544 hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 5545 if (hw->mac.type == ixgbe_mac_82598EB) 5546 hwstats->mprc -= bprc; 5547 hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC); 5548 hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 5549 hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 5550 hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 5551 hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 5552 hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 5553 hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 5554 hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 5555 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 5556 hwstats->lxontxc += lxon; 5557 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 5558 hwstats->lxofftxc += lxoff; 5559 hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 5560 hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 5561 /* 5562 * 82598 errata - tx of flow control packets is included in tx counters 5563 */ 5564 xon_off_tot = lxon + lxoff; 5565 hwstats->gptc -= xon_off_tot; 5566 hwstats->mptc -= xon_off_tot; 5567 hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN)); 5568 hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 5569 hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 5570 hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 5571 hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 5572 hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 5573 hwstats->ptc64 -= xon_off_tot; 5574 hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 5575 hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 5576 hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 5577 hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 5578 hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 5579 hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 5580 5581 /* Fill out the OS statistics structure */ 5582 netdev->stats.multicast = hwstats->mprc; 5583 5584 /* Rx Errors */ 5585 netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec; 5586 netdev->stats.rx_dropped = 0; 5587 netdev->stats.rx_length_errors = hwstats->rlec; 5588 netdev->stats.rx_crc_errors = hwstats->crcerrs; 5589 netdev->stats.rx_missed_errors = total_mpc; 5590 } 5591 5592 /** 5593 * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table 5594 * @adapter: pointer to the device adapter structure 5595 **/ 5596 static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter) 5597 { 5598 struct ixgbe_hw *hw = &adapter->hw; 5599 int i; 5600 5601 if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) 5602 return; 5603 5604 adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT; 5605 5606 /* if interface is down do nothing */ 5607 if (test_bit(__IXGBE_DOWN, &adapter->state)) 5608 return; 5609 5610 /* do nothing if we are not using signature filters */ 5611 if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) 5612 return; 5613 5614 adapter->fdir_overflow++; 5615 5616 if (ixgbe_reinit_fdir_tables_82599(hw) == 0) { 5617 for (i = 0; i < adapter->num_tx_queues; i++) 5618 set_bit(__IXGBE_TX_FDIR_INIT_DONE, 5619 &(adapter->tx_ring[i]->state)); 5620 /* re-enable flow director interrupts */ 5621 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR); 5622 } else { 5623 e_err(probe, "failed to finish FDIR re-initialization, " 5624 "ignored adding FDIR ATR filters\n"); 5625 } 5626 } 5627 5628 /** 5629 * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts 5630 * @adapter: pointer to the device adapter structure 5631 * 5632 * This function serves two purposes. First it strobes the interrupt lines 5633 * in order to make certain interrupts are occurring. Secondly it sets the 5634 * bits needed to check for TX hangs. As a result we should immediately 5635 * determine if a hang has occurred. 5636 */ 5637 static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter) 5638 { 5639 struct ixgbe_hw *hw = &adapter->hw; 5640 u64 eics = 0; 5641 int i; 5642 5643 /* If we're down or resetting, just bail */ 5644 if (test_bit(__IXGBE_DOWN, &adapter->state) || 5645 test_bit(__IXGBE_RESETTING, &adapter->state)) 5646 return; 5647 5648 /* Force detection of hung controller */ 5649 if (netif_carrier_ok(adapter->netdev)) { 5650 for (i = 0; i < adapter->num_tx_queues; i++) 5651 set_check_for_tx_hang(adapter->tx_ring[i]); 5652 } 5653 5654 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) { 5655 /* 5656 * for legacy and MSI interrupts don't set any bits 5657 * that are enabled for EIAM, because this operation 5658 * would set *both* EIMS and EICS for any bit in EIAM 5659 */ 5660 IXGBE_WRITE_REG(hw, IXGBE_EICS, 5661 (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER)); 5662 } else { 5663 /* get one bit for every active tx/rx interrupt vector */ 5664 for (i = 0; i < adapter->num_q_vectors; i++) { 5665 struct ixgbe_q_vector *qv = adapter->q_vector[i]; 5666 if (qv->rx.ring || qv->tx.ring) 5667 eics |= ((u64)1 << i); 5668 } 5669 } 5670 5671 /* Cause software interrupt to ensure rings are cleaned */ 5672 ixgbe_irq_rearm_queues(adapter, eics); 5673 5674 } 5675 5676 /** 5677 * ixgbe_watchdog_update_link - update the link status 5678 * @adapter: pointer to the device adapter structure 5679 * @link_speed: pointer to a u32 to store the link_speed 5680 **/ 5681 static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter) 5682 { 5683 struct ixgbe_hw *hw = &adapter->hw; 5684 u32 link_speed = adapter->link_speed; 5685 bool link_up = adapter->link_up; 5686 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable; 5687 5688 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) 5689 return; 5690 5691 if (hw->mac.ops.check_link) { 5692 hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 5693 } else { 5694 /* always assume link is up, if no check link function */ 5695 link_speed = IXGBE_LINK_SPEED_10GB_FULL; 5696 link_up = true; 5697 } 5698 5699 if (adapter->ixgbe_ieee_pfc) 5700 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en); 5701 5702 if (link_up && !((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && pfc_en)) { 5703 hw->mac.ops.fc_enable(hw); 5704 ixgbe_set_rx_drop_en(adapter); 5705 } 5706 5707 if (link_up || 5708 time_after(jiffies, (adapter->link_check_timeout + 5709 IXGBE_TRY_LINK_TIMEOUT))) { 5710 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; 5711 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC); 5712 IXGBE_WRITE_FLUSH(hw); 5713 } 5714 5715 adapter->link_up = link_up; 5716 adapter->link_speed = link_speed; 5717 } 5718 5719 static void ixgbe_update_default_up(struct ixgbe_adapter *adapter) 5720 { 5721 #ifdef CONFIG_IXGBE_DCB 5722 struct net_device *netdev = adapter->netdev; 5723 struct dcb_app app = { 5724 .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE, 5725 .protocol = 0, 5726 }; 5727 u8 up = 0; 5728 5729 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE) 5730 up = dcb_ieee_getapp_mask(netdev, &app); 5731 5732 adapter->default_up = (up > 1) ? (ffs(up) - 1) : 0; 5733 #endif 5734 } 5735 5736 /** 5737 * ixgbe_watchdog_link_is_up - update netif_carrier status and 5738 * print link up message 5739 * @adapter: pointer to the device adapter structure 5740 **/ 5741 static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter) 5742 { 5743 struct net_device *netdev = adapter->netdev; 5744 struct ixgbe_hw *hw = &adapter->hw; 5745 u32 link_speed = adapter->link_speed; 5746 bool flow_rx, flow_tx; 5747 5748 /* only continue if link was previously down */ 5749 if (netif_carrier_ok(netdev)) 5750 return; 5751 5752 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP; 5753 5754 switch (hw->mac.type) { 5755 case ixgbe_mac_82598EB: { 5756 u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 5757 u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS); 5758 flow_rx = !!(frctl & IXGBE_FCTRL_RFCE); 5759 flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X); 5760 } 5761 break; 5762 case ixgbe_mac_X540: 5763 case ixgbe_mac_82599EB: { 5764 u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN); 5765 u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG); 5766 flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE); 5767 flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X); 5768 } 5769 break; 5770 default: 5771 flow_tx = false; 5772 flow_rx = false; 5773 break; 5774 } 5775 5776 adapter->last_rx_ptp_check = jiffies; 5777 5778 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) 5779 ixgbe_ptp_start_cyclecounter(adapter); 5780 5781 e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", 5782 (link_speed == IXGBE_LINK_SPEED_10GB_FULL ? 5783 "10 Gbps" : 5784 (link_speed == IXGBE_LINK_SPEED_1GB_FULL ? 5785 "1 Gbps" : 5786 (link_speed == IXGBE_LINK_SPEED_100_FULL ? 5787 "100 Mbps" : 5788 "unknown speed"))), 5789 ((flow_rx && flow_tx) ? "RX/TX" : 5790 (flow_rx ? "RX" : 5791 (flow_tx ? "TX" : "None")))); 5792 5793 netif_carrier_on(netdev); 5794 ixgbe_check_vf_rate_limit(adapter); 5795 5796 /* update the default user priority for VFs */ 5797 ixgbe_update_default_up(adapter); 5798 5799 /* ping all the active vfs to let them know link has changed */ 5800 ixgbe_ping_all_vfs(adapter); 5801 } 5802 5803 /** 5804 * ixgbe_watchdog_link_is_down - update netif_carrier status and 5805 * print link down message 5806 * @adapter: pointer to the adapter structure 5807 **/ 5808 static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *adapter) 5809 { 5810 struct net_device *netdev = adapter->netdev; 5811 struct ixgbe_hw *hw = &adapter->hw; 5812 5813 adapter->link_up = false; 5814 adapter->link_speed = 0; 5815 5816 /* only continue if link was up previously */ 5817 if (!netif_carrier_ok(netdev)) 5818 return; 5819 5820 /* poll for SFP+ cable when link is down */ 5821 if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB) 5822 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP; 5823 5824 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) 5825 ixgbe_ptp_start_cyclecounter(adapter); 5826 5827 e_info(drv, "NIC Link is Down\n"); 5828 netif_carrier_off(netdev); 5829 5830 /* ping all the active vfs to let them know link has changed */ 5831 ixgbe_ping_all_vfs(adapter); 5832 } 5833 5834 /** 5835 * ixgbe_watchdog_flush_tx - flush queues on link down 5836 * @adapter: pointer to the device adapter structure 5837 **/ 5838 static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter) 5839 { 5840 int i; 5841 int some_tx_pending = 0; 5842 5843 if (!netif_carrier_ok(adapter->netdev)) { 5844 for (i = 0; i < adapter->num_tx_queues; i++) { 5845 struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; 5846 if (tx_ring->next_to_use != tx_ring->next_to_clean) { 5847 some_tx_pending = 1; 5848 break; 5849 } 5850 } 5851 5852 if (some_tx_pending) { 5853 /* We've lost link, so the controller stops DMA, 5854 * but we've got queued Tx work that's never going 5855 * to get done, so reset controller to flush Tx. 5856 * (Do the reset outside of interrupt context). 5857 */ 5858 e_warn(drv, "initiating reset to clear Tx work after link loss\n"); 5859 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED; 5860 } 5861 } 5862 } 5863 5864 static void ixgbe_spoof_check(struct ixgbe_adapter *adapter) 5865 { 5866 u32 ssvpc; 5867 5868 /* Do not perform spoof check for 82598 or if not in IOV mode */ 5869 if (adapter->hw.mac.type == ixgbe_mac_82598EB || 5870 adapter->num_vfs == 0) 5871 return; 5872 5873 ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC); 5874 5875 /* 5876 * ssvpc register is cleared on read, if zero then no 5877 * spoofed packets in the last interval. 5878 */ 5879 if (!ssvpc) 5880 return; 5881 5882 e_warn(drv, "%u Spoofed packets detected\n", ssvpc); 5883 } 5884 5885 /** 5886 * ixgbe_watchdog_subtask - check and bring link up 5887 * @adapter: pointer to the device adapter structure 5888 **/ 5889 static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter) 5890 { 5891 /* if interface is down do nothing */ 5892 if (test_bit(__IXGBE_DOWN, &adapter->state) || 5893 test_bit(__IXGBE_RESETTING, &adapter->state)) 5894 return; 5895 5896 ixgbe_watchdog_update_link(adapter); 5897 5898 if (adapter->link_up) 5899 ixgbe_watchdog_link_is_up(adapter); 5900 else 5901 ixgbe_watchdog_link_is_down(adapter); 5902 5903 ixgbe_spoof_check(adapter); 5904 ixgbe_update_stats(adapter); 5905 5906 ixgbe_watchdog_flush_tx(adapter); 5907 } 5908 5909 /** 5910 * ixgbe_sfp_detection_subtask - poll for SFP+ cable 5911 * @adapter: the ixgbe adapter structure 5912 **/ 5913 static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) 5914 { 5915 struct ixgbe_hw *hw = &adapter->hw; 5916 s32 err; 5917 5918 /* not searching for SFP so there is nothing to do here */ 5919 if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) && 5920 !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) 5921 return; 5922 5923 /* someone else is in init, wait until next service event */ 5924 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) 5925 return; 5926 5927 err = hw->phy.ops.identify_sfp(hw); 5928 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) 5929 goto sfp_out; 5930 5931 if (err == IXGBE_ERR_SFP_NOT_PRESENT) { 5932 /* If no cable is present, then we need to reset 5933 * the next time we find a good cable. */ 5934 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; 5935 } 5936 5937 /* exit on error */ 5938 if (err) 5939 goto sfp_out; 5940 5941 /* exit if reset not needed */ 5942 if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) 5943 goto sfp_out; 5944 5945 adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET; 5946 5947 /* 5948 * A module may be identified correctly, but the EEPROM may not have 5949 * support for that module. setup_sfp() will fail in that case, so 5950 * we should not allow that module to load. 5951 */ 5952 if (hw->mac.type == ixgbe_mac_82598EB) 5953 err = hw->phy.ops.reset(hw); 5954 else 5955 err = hw->mac.ops.setup_sfp(hw); 5956 5957 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) 5958 goto sfp_out; 5959 5960 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; 5961 e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type); 5962 5963 sfp_out: 5964 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); 5965 5966 if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) && 5967 (adapter->netdev->reg_state == NETREG_REGISTERED)) { 5968 e_dev_err("failed to initialize because an unsupported " 5969 "SFP+ module type was detected.\n"); 5970 e_dev_err("Reload the driver after installing a " 5971 "supported module.\n"); 5972 unregister_netdev(adapter->netdev); 5973 } 5974 } 5975 5976 /** 5977 * ixgbe_sfp_link_config_subtask - set up link SFP after module install 5978 * @adapter: the ixgbe adapter structure 5979 **/ 5980 static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter) 5981 { 5982 struct ixgbe_hw *hw = &adapter->hw; 5983 u32 speed; 5984 bool autoneg = false; 5985 5986 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG)) 5987 return; 5988 5989 /* someone else is in init, wait until next service event */ 5990 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) 5991 return; 5992 5993 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; 5994 5995 speed = hw->phy.autoneg_advertised; 5996 if ((!speed) && (hw->mac.ops.get_link_capabilities)) 5997 hw->mac.ops.get_link_capabilities(hw, &speed, &autoneg); 5998 if (hw->mac.ops.setup_link) 5999 hw->mac.ops.setup_link(hw, speed, true); 6000 6001 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 6002 adapter->link_check_timeout = jiffies; 6003 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); 6004 } 6005 6006 #ifdef CONFIG_PCI_IOV 6007 static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter) 6008 { 6009 int vf; 6010 struct ixgbe_hw *hw = &adapter->hw; 6011 struct net_device *netdev = adapter->netdev; 6012 u32 gpc; 6013 u32 ciaa, ciad; 6014 6015 gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC); 6016 if (gpc) /* If incrementing then no need for the check below */ 6017 return; 6018 /* 6019 * Check to see if a bad DMA write target from an errant or 6020 * malicious VF has caused a PCIe error. If so then we can 6021 * issue a VFLR to the offending VF(s) and then resume without 6022 * requesting a full slot reset. 6023 */ 6024 6025 for (vf = 0; vf < adapter->num_vfs; vf++) { 6026 ciaa = (vf << 16) | 0x80000000; 6027 /* 32 bit read so align, we really want status at offset 6 */ 6028 ciaa |= PCI_COMMAND; 6029 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); 6030 ciad = IXGBE_READ_REG(hw, IXGBE_CIAD_82599); 6031 ciaa &= 0x7FFFFFFF; 6032 /* disable debug mode asap after reading data */ 6033 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); 6034 /* Get the upper 16 bits which will be the PCI status reg */ 6035 ciad >>= 16; 6036 if (ciad & PCI_STATUS_REC_MASTER_ABORT) { 6037 netdev_err(netdev, "VF %d Hung DMA\n", vf); 6038 /* Issue VFLR */ 6039 ciaa = (vf << 16) | 0x80000000; 6040 ciaa |= 0xA8; 6041 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); 6042 ciad = 0x00008000; /* VFLR */ 6043 IXGBE_WRITE_REG(hw, IXGBE_CIAD_82599, ciad); 6044 ciaa &= 0x7FFFFFFF; 6045 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); 6046 } 6047 } 6048 } 6049 6050 #endif 6051 /** 6052 * ixgbe_service_timer - Timer Call-back 6053 * @data: pointer to adapter cast into an unsigned long 6054 **/ 6055 static void ixgbe_service_timer(unsigned long data) 6056 { 6057 struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data; 6058 unsigned long next_event_offset; 6059 bool ready = true; 6060 6061 /* poll faster when waiting for link */ 6062 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE) 6063 next_event_offset = HZ / 10; 6064 else 6065 next_event_offset = HZ * 2; 6066 6067 #ifdef CONFIG_PCI_IOV 6068 /* 6069 * don't bother with SR-IOV VF DMA hang check if there are 6070 * no VFs or the link is down 6071 */ 6072 if (!adapter->num_vfs || 6073 (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) 6074 goto normal_timer_service; 6075 6076 /* If we have VFs allocated then we must check for DMA hangs */ 6077 ixgbe_check_for_bad_vf(adapter); 6078 next_event_offset = HZ / 50; 6079 adapter->timer_event_accumulator++; 6080 6081 if (adapter->timer_event_accumulator >= 100) 6082 adapter->timer_event_accumulator = 0; 6083 else 6084 ready = false; 6085 6086 normal_timer_service: 6087 #endif 6088 /* Reset the timer */ 6089 mod_timer(&adapter->service_timer, next_event_offset + jiffies); 6090 6091 if (ready) 6092 ixgbe_service_event_schedule(adapter); 6093 } 6094 6095 static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter) 6096 { 6097 if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED)) 6098 return; 6099 6100 adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED; 6101 6102 /* If we're already down or resetting, just bail */ 6103 if (test_bit(__IXGBE_DOWN, &adapter->state) || 6104 test_bit(__IXGBE_RESETTING, &adapter->state)) 6105 return; 6106 6107 ixgbe_dump(adapter); 6108 netdev_err(adapter->netdev, "Reset adapter\n"); 6109 adapter->tx_timeout_count++; 6110 6111 ixgbe_reinit_locked(adapter); 6112 } 6113 6114 /** 6115 * ixgbe_service_task - manages and runs subtasks 6116 * @work: pointer to work_struct containing our data 6117 **/ 6118 static void ixgbe_service_task(struct work_struct *work) 6119 { 6120 struct ixgbe_adapter *adapter = container_of(work, 6121 struct ixgbe_adapter, 6122 service_task); 6123 ixgbe_reset_subtask(adapter); 6124 ixgbe_sfp_detection_subtask(adapter); 6125 ixgbe_sfp_link_config_subtask(adapter); 6126 ixgbe_check_overtemp_subtask(adapter); 6127 ixgbe_watchdog_subtask(adapter); 6128 ixgbe_fdir_reinit_subtask(adapter); 6129 ixgbe_check_hang_subtask(adapter); 6130 6131 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) { 6132 ixgbe_ptp_overflow_check(adapter); 6133 ixgbe_ptp_rx_hang(adapter); 6134 } 6135 6136 ixgbe_service_event_complete(adapter); 6137 } 6138 6139 static int ixgbe_tso(struct ixgbe_ring *tx_ring, 6140 struct ixgbe_tx_buffer *first, 6141 u8 *hdr_len) 6142 { 6143 struct sk_buff *skb = first->skb; 6144 u32 vlan_macip_lens, type_tucmd; 6145 u32 mss_l4len_idx, l4len; 6146 6147 if (skb->ip_summed != CHECKSUM_PARTIAL) 6148 return 0; 6149 6150 if (!skb_is_gso(skb)) 6151 return 0; 6152 6153 if (skb_header_cloned(skb)) { 6154 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 6155 if (err) 6156 return err; 6157 } 6158 6159 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 6160 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 6161 6162 if (first->protocol == __constant_htons(ETH_P_IP)) { 6163 struct iphdr *iph = ip_hdr(skb); 6164 iph->tot_len = 0; 6165 iph->check = 0; 6166 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 6167 iph->daddr, 0, 6168 IPPROTO_TCP, 6169 0); 6170 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 6171 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 6172 IXGBE_TX_FLAGS_CSUM | 6173 IXGBE_TX_FLAGS_IPV4; 6174 } else if (skb_is_gso_v6(skb)) { 6175 ipv6_hdr(skb)->payload_len = 0; 6176 tcp_hdr(skb)->check = 6177 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 6178 &ipv6_hdr(skb)->daddr, 6179 0, IPPROTO_TCP, 0); 6180 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 6181 IXGBE_TX_FLAGS_CSUM; 6182 } 6183 6184 /* compute header lengths */ 6185 l4len = tcp_hdrlen(skb); 6186 *hdr_len = skb_transport_offset(skb) + l4len; 6187 6188 /* update gso size and bytecount with header size */ 6189 first->gso_segs = skb_shinfo(skb)->gso_segs; 6190 first->bytecount += (first->gso_segs - 1) * *hdr_len; 6191 6192 /* mss_l4len_id: use 0 as index for TSO */ 6193 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; 6194 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 6195 6196 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 6197 vlan_macip_lens = skb_network_header_len(skb); 6198 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 6199 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 6200 6201 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd, 6202 mss_l4len_idx); 6203 6204 return 1; 6205 } 6206 6207 static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring, 6208 struct ixgbe_tx_buffer *first) 6209 { 6210 struct sk_buff *skb = first->skb; 6211 u32 vlan_macip_lens = 0; 6212 u32 mss_l4len_idx = 0; 6213 u32 type_tucmd = 0; 6214 6215 if (skb->ip_summed != CHECKSUM_PARTIAL) { 6216 if (!(first->tx_flags & IXGBE_TX_FLAGS_HW_VLAN) && 6217 !(first->tx_flags & IXGBE_TX_FLAGS_CC)) 6218 return; 6219 } else { 6220 u8 l4_hdr = 0; 6221 switch (first->protocol) { 6222 case __constant_htons(ETH_P_IP): 6223 vlan_macip_lens |= skb_network_header_len(skb); 6224 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 6225 l4_hdr = ip_hdr(skb)->protocol; 6226 break; 6227 case __constant_htons(ETH_P_IPV6): 6228 vlan_macip_lens |= skb_network_header_len(skb); 6229 l4_hdr = ipv6_hdr(skb)->nexthdr; 6230 break; 6231 default: 6232 if (unlikely(net_ratelimit())) { 6233 dev_warn(tx_ring->dev, 6234 "partial checksum but proto=%x!\n", 6235 first->protocol); 6236 } 6237 break; 6238 } 6239 6240 switch (l4_hdr) { 6241 case IPPROTO_TCP: 6242 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 6243 mss_l4len_idx = tcp_hdrlen(skb) << 6244 IXGBE_ADVTXD_L4LEN_SHIFT; 6245 break; 6246 case IPPROTO_SCTP: 6247 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 6248 mss_l4len_idx = sizeof(struct sctphdr) << 6249 IXGBE_ADVTXD_L4LEN_SHIFT; 6250 break; 6251 case IPPROTO_UDP: 6252 mss_l4len_idx = sizeof(struct udphdr) << 6253 IXGBE_ADVTXD_L4LEN_SHIFT; 6254 break; 6255 default: 6256 if (unlikely(net_ratelimit())) { 6257 dev_warn(tx_ring->dev, 6258 "partial checksum but l4 proto=%x!\n", 6259 l4_hdr); 6260 } 6261 break; 6262 } 6263 6264 /* update TX checksum flag */ 6265 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 6266 } 6267 6268 /* vlan_macip_lens: MACLEN, VLAN tag */ 6269 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 6270 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 6271 6272 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, 6273 type_tucmd, mss_l4len_idx); 6274 } 6275 6276 #define IXGBE_SET_FLAG(_input, _flag, _result) \ 6277 ((_flag <= _result) ? \ 6278 ((u32)(_input & _flag) * (_result / _flag)) : \ 6279 ((u32)(_input & _flag) / (_flag / _result))) 6280 6281 static u32 ixgbe_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 6282 { 6283 /* set type for advanced descriptor with frame checksum insertion */ 6284 u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | 6285 IXGBE_ADVTXD_DCMD_DEXT | 6286 IXGBE_ADVTXD_DCMD_IFCS; 6287 6288 /* set HW vlan bit if vlan is present */ 6289 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_HW_VLAN, 6290 IXGBE_ADVTXD_DCMD_VLE); 6291 6292 /* set segmentation enable bits for TSO/FSO */ 6293 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSO, 6294 IXGBE_ADVTXD_DCMD_TSE); 6295 6296 /* set timestamp bit if present */ 6297 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSTAMP, 6298 IXGBE_ADVTXD_MAC_TSTAMP); 6299 6300 /* insert frame checksum */ 6301 cmd_type ^= IXGBE_SET_FLAG(skb->no_fcs, 1, IXGBE_ADVTXD_DCMD_IFCS); 6302 6303 return cmd_type; 6304 } 6305 6306 static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 6307 u32 tx_flags, unsigned int paylen) 6308 { 6309 u32 olinfo_status = paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 6310 6311 /* enable L4 checksum for TSO and TX checksum offload */ 6312 olinfo_status |= IXGBE_SET_FLAG(tx_flags, 6313 IXGBE_TX_FLAGS_CSUM, 6314 IXGBE_ADVTXD_POPTS_TXSM); 6315 6316 /* enble IPv4 checksum for TSO */ 6317 olinfo_status |= IXGBE_SET_FLAG(tx_flags, 6318 IXGBE_TX_FLAGS_IPV4, 6319 IXGBE_ADVTXD_POPTS_IXSM); 6320 6321 /* 6322 * Check Context must be set if Tx switch is enabled, which it 6323 * always is for case where virtual functions are running 6324 */ 6325 olinfo_status |= IXGBE_SET_FLAG(tx_flags, 6326 IXGBE_TX_FLAGS_CC, 6327 IXGBE_ADVTXD_CC); 6328 6329 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 6330 } 6331 6332 #define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \ 6333 IXGBE_TXD_CMD_RS) 6334 6335 static void ixgbe_tx_map(struct ixgbe_ring *tx_ring, 6336 struct ixgbe_tx_buffer *first, 6337 const u8 hdr_len) 6338 { 6339 struct sk_buff *skb = first->skb; 6340 struct ixgbe_tx_buffer *tx_buffer; 6341 union ixgbe_adv_tx_desc *tx_desc; 6342 struct skb_frag_struct *frag; 6343 dma_addr_t dma; 6344 unsigned int data_len, size; 6345 u32 tx_flags = first->tx_flags; 6346 u32 cmd_type = ixgbe_tx_cmd_type(skb, tx_flags); 6347 u16 i = tx_ring->next_to_use; 6348 6349 tx_desc = IXGBE_TX_DESC(tx_ring, i); 6350 6351 ixgbe_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len); 6352 6353 size = skb_headlen(skb); 6354 data_len = skb->data_len; 6355 6356 #ifdef IXGBE_FCOE 6357 if (tx_flags & IXGBE_TX_FLAGS_FCOE) { 6358 if (data_len < sizeof(struct fcoe_crc_eof)) { 6359 size -= sizeof(struct fcoe_crc_eof) - data_len; 6360 data_len = 0; 6361 } else { 6362 data_len -= sizeof(struct fcoe_crc_eof); 6363 } 6364 } 6365 6366 #endif 6367 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 6368 6369 tx_buffer = first; 6370 6371 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 6372 if (dma_mapping_error(tx_ring->dev, dma)) 6373 goto dma_error; 6374 6375 /* record length, and DMA address */ 6376 dma_unmap_len_set(tx_buffer, len, size); 6377 dma_unmap_addr_set(tx_buffer, dma, dma); 6378 6379 tx_desc->read.buffer_addr = cpu_to_le64(dma); 6380 6381 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 6382 tx_desc->read.cmd_type_len = 6383 cpu_to_le32(cmd_type ^ IXGBE_MAX_DATA_PER_TXD); 6384 6385 i++; 6386 tx_desc++; 6387 if (i == tx_ring->count) { 6388 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 6389 i = 0; 6390 } 6391 tx_desc->read.olinfo_status = 0; 6392 6393 dma += IXGBE_MAX_DATA_PER_TXD; 6394 size -= IXGBE_MAX_DATA_PER_TXD; 6395 6396 tx_desc->read.buffer_addr = cpu_to_le64(dma); 6397 } 6398 6399 if (likely(!data_len)) 6400 break; 6401 6402 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 6403 6404 i++; 6405 tx_desc++; 6406 if (i == tx_ring->count) { 6407 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 6408 i = 0; 6409 } 6410 tx_desc->read.olinfo_status = 0; 6411 6412 #ifdef IXGBE_FCOE 6413 size = min_t(unsigned int, data_len, skb_frag_size(frag)); 6414 #else 6415 size = skb_frag_size(frag); 6416 #endif 6417 data_len -= size; 6418 6419 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 6420 DMA_TO_DEVICE); 6421 6422 tx_buffer = &tx_ring->tx_buffer_info[i]; 6423 } 6424 6425 /* write last descriptor with RS and EOP bits */ 6426 cmd_type |= size | IXGBE_TXD_CMD; 6427 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 6428 6429 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 6430 6431 /* set the timestamp */ 6432 first->time_stamp = jiffies; 6433 6434 /* 6435 * Force memory writes to complete before letting h/w know there 6436 * are new descriptors to fetch. (Only applicable for weak-ordered 6437 * memory model archs, such as IA-64). 6438 * 6439 * We also need this memory barrier to make certain all of the 6440 * status bits have been updated before next_to_watch is written. 6441 */ 6442 wmb(); 6443 6444 /* set next_to_watch value indicating a packet is present */ 6445 first->next_to_watch = tx_desc; 6446 6447 i++; 6448 if (i == tx_ring->count) 6449 i = 0; 6450 6451 tx_ring->next_to_use = i; 6452 6453 /* notify HW of packet */ 6454 writel(i, tx_ring->tail); 6455 6456 return; 6457 dma_error: 6458 dev_err(tx_ring->dev, "TX DMA map failed\n"); 6459 6460 /* clear dma mappings for failed tx_buffer_info map */ 6461 for (;;) { 6462 tx_buffer = &tx_ring->tx_buffer_info[i]; 6463 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer); 6464 if (tx_buffer == first) 6465 break; 6466 if (i == 0) 6467 i = tx_ring->count; 6468 i--; 6469 } 6470 6471 tx_ring->next_to_use = i; 6472 } 6473 6474 static void ixgbe_atr(struct ixgbe_ring *ring, 6475 struct ixgbe_tx_buffer *first) 6476 { 6477 struct ixgbe_q_vector *q_vector = ring->q_vector; 6478 union ixgbe_atr_hash_dword input = { .dword = 0 }; 6479 union ixgbe_atr_hash_dword common = { .dword = 0 }; 6480 union { 6481 unsigned char *network; 6482 struct iphdr *ipv4; 6483 struct ipv6hdr *ipv6; 6484 } hdr; 6485 struct tcphdr *th; 6486 __be16 vlan_id; 6487 6488 /* if ring doesn't have a interrupt vector, cannot perform ATR */ 6489 if (!q_vector) 6490 return; 6491 6492 /* do nothing if sampling is disabled */ 6493 if (!ring->atr_sample_rate) 6494 return; 6495 6496 ring->atr_count++; 6497 6498 /* snag network header to get L4 type and address */ 6499 hdr.network = skb_network_header(first->skb); 6500 6501 /* Currently only IPv4/IPv6 with TCP is supported */ 6502 if ((first->protocol != __constant_htons(ETH_P_IPV6) || 6503 hdr.ipv6->nexthdr != IPPROTO_TCP) && 6504 (first->protocol != __constant_htons(ETH_P_IP) || 6505 hdr.ipv4->protocol != IPPROTO_TCP)) 6506 return; 6507 6508 th = tcp_hdr(first->skb); 6509 6510 /* skip this packet since it is invalid or the socket is closing */ 6511 if (!th || th->fin) 6512 return; 6513 6514 /* sample on all syn packets or once every atr sample count */ 6515 if (!th->syn && (ring->atr_count < ring->atr_sample_rate)) 6516 return; 6517 6518 /* reset sample count */ 6519 ring->atr_count = 0; 6520 6521 vlan_id = htons(first->tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT); 6522 6523 /* 6524 * src and dst are inverted, think how the receiver sees them 6525 * 6526 * The input is broken into two sections, a non-compressed section 6527 * containing vm_pool, vlan_id, and flow_type. The rest of the data 6528 * is XORed together and stored in the compressed dword. 6529 */ 6530 input.formatted.vlan_id = vlan_id; 6531 6532 /* 6533 * since src port and flex bytes occupy the same word XOR them together 6534 * and write the value to source port portion of compressed dword 6535 */ 6536 if (first->tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN)) 6537 common.port.src ^= th->dest ^ __constant_htons(ETH_P_8021Q); 6538 else 6539 common.port.src ^= th->dest ^ first->protocol; 6540 common.port.dst ^= th->source; 6541 6542 if (first->protocol == __constant_htons(ETH_P_IP)) { 6543 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4; 6544 common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr; 6545 } else { 6546 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6; 6547 common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^ 6548 hdr.ipv6->saddr.s6_addr32[1] ^ 6549 hdr.ipv6->saddr.s6_addr32[2] ^ 6550 hdr.ipv6->saddr.s6_addr32[3] ^ 6551 hdr.ipv6->daddr.s6_addr32[0] ^ 6552 hdr.ipv6->daddr.s6_addr32[1] ^ 6553 hdr.ipv6->daddr.s6_addr32[2] ^ 6554 hdr.ipv6->daddr.s6_addr32[3]; 6555 } 6556 6557 /* This assumes the Rx queue and Tx queue are bound to the same CPU */ 6558 ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw, 6559 input, common, ring->queue_index); 6560 } 6561 6562 static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) 6563 { 6564 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 6565 /* Herbert's original patch had: 6566 * smp_mb__after_netif_stop_queue(); 6567 * but since that doesn't exist yet, just open code it. */ 6568 smp_mb(); 6569 6570 /* We need to check again in a case another CPU has just 6571 * made room available. */ 6572 if (likely(ixgbe_desc_unused(tx_ring) < size)) 6573 return -EBUSY; 6574 6575 /* A reprieve! - use start_queue because it doesn't call schedule */ 6576 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 6577 ++tx_ring->tx_stats.restart_queue; 6578 return 0; 6579 } 6580 6581 static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) 6582 { 6583 if (likely(ixgbe_desc_unused(tx_ring) >= size)) 6584 return 0; 6585 return __ixgbe_maybe_stop_tx(tx_ring, size); 6586 } 6587 6588 #ifdef IXGBE_FCOE 6589 static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb) 6590 { 6591 struct ixgbe_adapter *adapter; 6592 struct ixgbe_ring_feature *f; 6593 int txq; 6594 6595 /* 6596 * only execute the code below if protocol is FCoE 6597 * or FIP and we have FCoE enabled on the adapter 6598 */ 6599 switch (vlan_get_protocol(skb)) { 6600 case __constant_htons(ETH_P_FCOE): 6601 case __constant_htons(ETH_P_FIP): 6602 adapter = netdev_priv(dev); 6603 6604 if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) 6605 break; 6606 default: 6607 return __netdev_pick_tx(dev, skb); 6608 } 6609 6610 f = &adapter->ring_feature[RING_F_FCOE]; 6611 6612 txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : 6613 smp_processor_id(); 6614 6615 while (txq >= f->indices) 6616 txq -= f->indices; 6617 6618 return txq + f->offset; 6619 } 6620 6621 #endif 6622 netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, 6623 struct ixgbe_adapter *adapter, 6624 struct ixgbe_ring *tx_ring) 6625 { 6626 struct ixgbe_tx_buffer *first; 6627 int tso; 6628 u32 tx_flags = 0; 6629 unsigned short f; 6630 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 6631 __be16 protocol = skb->protocol; 6632 u8 hdr_len = 0; 6633 6634 /* 6635 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 6636 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 6637 * + 2 desc gap to keep tail from touching head, 6638 * + 1 desc for context descriptor, 6639 * otherwise try next time 6640 */ 6641 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 6642 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 6643 6644 if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) { 6645 tx_ring->tx_stats.tx_busy++; 6646 return NETDEV_TX_BUSY; 6647 } 6648 6649 /* record the location of the first descriptor for this packet */ 6650 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 6651 first->skb = skb; 6652 first->bytecount = skb->len; 6653 first->gso_segs = 1; 6654 6655 /* if we have a HW VLAN tag being added default to the HW one */ 6656 if (vlan_tx_tag_present(skb)) { 6657 tx_flags |= vlan_tx_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT; 6658 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN; 6659 /* else if it is a SW VLAN check the next protocol and store the tag */ 6660 } else if (protocol == __constant_htons(ETH_P_8021Q)) { 6661 struct vlan_hdr *vhdr, _vhdr; 6662 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr); 6663 if (!vhdr) 6664 goto out_drop; 6665 6666 protocol = vhdr->h_vlan_encapsulated_proto; 6667 tx_flags |= ntohs(vhdr->h_vlan_TCI) << 6668 IXGBE_TX_FLAGS_VLAN_SHIFT; 6669 tx_flags |= IXGBE_TX_FLAGS_SW_VLAN; 6670 } 6671 6672 skb_tx_timestamp(skb); 6673 6674 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 6675 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 6676 tx_flags |= IXGBE_TX_FLAGS_TSTAMP; 6677 6678 /* schedule check for Tx timestamp */ 6679 adapter->ptp_tx_skb = skb_get(skb); 6680 adapter->ptp_tx_start = jiffies; 6681 schedule_work(&adapter->ptp_tx_work); 6682 } 6683 6684 #ifdef CONFIG_PCI_IOV 6685 /* 6686 * Use the l2switch_enable flag - would be false if the DMA 6687 * Tx switch had been disabled. 6688 */ 6689 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 6690 tx_flags |= IXGBE_TX_FLAGS_CC; 6691 6692 #endif 6693 /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */ 6694 if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && 6695 ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || 6696 (skb->priority != TC_PRIO_CONTROL))) { 6697 tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK; 6698 tx_flags |= (skb->priority & 0x7) << 6699 IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT; 6700 if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) { 6701 struct vlan_ethhdr *vhdr; 6702 if (skb_header_cloned(skb) && 6703 pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) 6704 goto out_drop; 6705 vhdr = (struct vlan_ethhdr *)skb->data; 6706 vhdr->h_vlan_TCI = htons(tx_flags >> 6707 IXGBE_TX_FLAGS_VLAN_SHIFT); 6708 } else { 6709 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN; 6710 } 6711 } 6712 6713 /* record initial flags and protocol */ 6714 first->tx_flags = tx_flags; 6715 first->protocol = protocol; 6716 6717 #ifdef IXGBE_FCOE 6718 /* setup tx offload for FCoE */ 6719 if ((protocol == __constant_htons(ETH_P_FCOE)) && 6720 (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) { 6721 tso = ixgbe_fso(tx_ring, first, &hdr_len); 6722 if (tso < 0) 6723 goto out_drop; 6724 6725 goto xmit_fcoe; 6726 } 6727 6728 #endif /* IXGBE_FCOE */ 6729 tso = ixgbe_tso(tx_ring, first, &hdr_len); 6730 if (tso < 0) 6731 goto out_drop; 6732 else if (!tso) 6733 ixgbe_tx_csum(tx_ring, first); 6734 6735 /* add the ATR filter if ATR is on */ 6736 if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state)) 6737 ixgbe_atr(tx_ring, first); 6738 6739 #ifdef IXGBE_FCOE 6740 xmit_fcoe: 6741 #endif /* IXGBE_FCOE */ 6742 ixgbe_tx_map(tx_ring, first, hdr_len); 6743 6744 ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED); 6745 6746 return NETDEV_TX_OK; 6747 6748 out_drop: 6749 dev_kfree_skb_any(first->skb); 6750 first->skb = NULL; 6751 6752 return NETDEV_TX_OK; 6753 } 6754 6755 static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, 6756 struct net_device *netdev) 6757 { 6758 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6759 struct ixgbe_ring *tx_ring; 6760 6761 /* 6762 * The minimum packet size for olinfo paylen is 17 so pad the skb 6763 * in order to meet this minimum size requirement. 6764 */ 6765 if (unlikely(skb->len < 17)) { 6766 if (skb_pad(skb, 17 - skb->len)) 6767 return NETDEV_TX_OK; 6768 skb->len = 17; 6769 skb_set_tail_pointer(skb, 17); 6770 } 6771 6772 tx_ring = adapter->tx_ring[skb->queue_mapping]; 6773 return ixgbe_xmit_frame_ring(skb, adapter, tx_ring); 6774 } 6775 6776 /** 6777 * ixgbe_set_mac - Change the Ethernet Address of the NIC 6778 * @netdev: network interface device structure 6779 * @p: pointer to an address structure 6780 * 6781 * Returns 0 on success, negative on failure 6782 **/ 6783 static int ixgbe_set_mac(struct net_device *netdev, void *p) 6784 { 6785 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6786 struct ixgbe_hw *hw = &adapter->hw; 6787 struct sockaddr *addr = p; 6788 6789 if (!is_valid_ether_addr(addr->sa_data)) 6790 return -EADDRNOTAVAIL; 6791 6792 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 6793 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 6794 6795 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, VMDQ_P(0), IXGBE_RAH_AV); 6796 6797 return 0; 6798 } 6799 6800 static int 6801 ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr) 6802 { 6803 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6804 struct ixgbe_hw *hw = &adapter->hw; 6805 u16 value; 6806 int rc; 6807 6808 if (prtad != hw->phy.mdio.prtad) 6809 return -EINVAL; 6810 rc = hw->phy.ops.read_reg(hw, addr, devad, &value); 6811 if (!rc) 6812 rc = value; 6813 return rc; 6814 } 6815 6816 static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad, 6817 u16 addr, u16 value) 6818 { 6819 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6820 struct ixgbe_hw *hw = &adapter->hw; 6821 6822 if (prtad != hw->phy.mdio.prtad) 6823 return -EINVAL; 6824 return hw->phy.ops.write_reg(hw, addr, devad, value); 6825 } 6826 6827 static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd) 6828 { 6829 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6830 6831 switch (cmd) { 6832 case SIOCSHWTSTAMP: 6833 return ixgbe_ptp_hwtstamp_ioctl(adapter, req, cmd); 6834 default: 6835 return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd); 6836 } 6837 } 6838 6839 /** 6840 * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding 6841 * netdev->dev_addrs 6842 * @netdev: network interface device structure 6843 * 6844 * Returns non-zero on failure 6845 **/ 6846 static int ixgbe_add_sanmac_netdev(struct net_device *dev) 6847 { 6848 int err = 0; 6849 struct ixgbe_adapter *adapter = netdev_priv(dev); 6850 struct ixgbe_hw *hw = &adapter->hw; 6851 6852 if (is_valid_ether_addr(hw->mac.san_addr)) { 6853 rtnl_lock(); 6854 err = dev_addr_add(dev, hw->mac.san_addr, NETDEV_HW_ADDR_T_SAN); 6855 rtnl_unlock(); 6856 6857 /* update SAN MAC vmdq pool selection */ 6858 hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0)); 6859 } 6860 return err; 6861 } 6862 6863 /** 6864 * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding 6865 * netdev->dev_addrs 6866 * @netdev: network interface device structure 6867 * 6868 * Returns non-zero on failure 6869 **/ 6870 static int ixgbe_del_sanmac_netdev(struct net_device *dev) 6871 { 6872 int err = 0; 6873 struct ixgbe_adapter *adapter = netdev_priv(dev); 6874 struct ixgbe_mac_info *mac = &adapter->hw.mac; 6875 6876 if (is_valid_ether_addr(mac->san_addr)) { 6877 rtnl_lock(); 6878 err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN); 6879 rtnl_unlock(); 6880 } 6881 return err; 6882 } 6883 6884 #ifdef CONFIG_NET_POLL_CONTROLLER 6885 /* 6886 * Polling 'interrupt' - used by things like netconsole to send skbs 6887 * without having to re-enable interrupts. It's not called while 6888 * the interrupt routine is executing. 6889 */ 6890 static void ixgbe_netpoll(struct net_device *netdev) 6891 { 6892 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6893 int i; 6894 6895 /* if interface is down do nothing */ 6896 if (test_bit(__IXGBE_DOWN, &adapter->state)) 6897 return; 6898 6899 adapter->flags |= IXGBE_FLAG_IN_NETPOLL; 6900 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 6901 for (i = 0; i < adapter->num_q_vectors; i++) 6902 ixgbe_msix_clean_rings(0, adapter->q_vector[i]); 6903 } else { 6904 ixgbe_intr(adapter->pdev->irq, netdev); 6905 } 6906 adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL; 6907 } 6908 6909 #endif 6910 static struct rtnl_link_stats64 *ixgbe_get_stats64(struct net_device *netdev, 6911 struct rtnl_link_stats64 *stats) 6912 { 6913 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6914 int i; 6915 6916 rcu_read_lock(); 6917 for (i = 0; i < adapter->num_rx_queues; i++) { 6918 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]); 6919 u64 bytes, packets; 6920 unsigned int start; 6921 6922 if (ring) { 6923 do { 6924 start = u64_stats_fetch_begin_bh(&ring->syncp); 6925 packets = ring->stats.packets; 6926 bytes = ring->stats.bytes; 6927 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 6928 stats->rx_packets += packets; 6929 stats->rx_bytes += bytes; 6930 } 6931 } 6932 6933 for (i = 0; i < adapter->num_tx_queues; i++) { 6934 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->tx_ring[i]); 6935 u64 bytes, packets; 6936 unsigned int start; 6937 6938 if (ring) { 6939 do { 6940 start = u64_stats_fetch_begin_bh(&ring->syncp); 6941 packets = ring->stats.packets; 6942 bytes = ring->stats.bytes; 6943 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 6944 stats->tx_packets += packets; 6945 stats->tx_bytes += bytes; 6946 } 6947 } 6948 rcu_read_unlock(); 6949 /* following stats updated by ixgbe_watchdog_task() */ 6950 stats->multicast = netdev->stats.multicast; 6951 stats->rx_errors = netdev->stats.rx_errors; 6952 stats->rx_length_errors = netdev->stats.rx_length_errors; 6953 stats->rx_crc_errors = netdev->stats.rx_crc_errors; 6954 stats->rx_missed_errors = netdev->stats.rx_missed_errors; 6955 return stats; 6956 } 6957 6958 #ifdef CONFIG_IXGBE_DCB 6959 /** 6960 * ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid. 6961 * @adapter: pointer to ixgbe_adapter 6962 * @tc: number of traffic classes currently enabled 6963 * 6964 * Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm 6965 * 802.1Q priority maps to a packet buffer that exists. 6966 */ 6967 static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc) 6968 { 6969 struct ixgbe_hw *hw = &adapter->hw; 6970 u32 reg, rsave; 6971 int i; 6972 6973 /* 82598 have a static priority to TC mapping that can not 6974 * be changed so no validation is needed. 6975 */ 6976 if (hw->mac.type == ixgbe_mac_82598EB) 6977 return; 6978 6979 reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC); 6980 rsave = reg; 6981 6982 for (i = 0; i < MAX_TRAFFIC_CLASS; i++) { 6983 u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT); 6984 6985 /* If up2tc is out of bounds default to zero */ 6986 if (up2tc > tc) 6987 reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT); 6988 } 6989 6990 if (reg != rsave) 6991 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg); 6992 6993 return; 6994 } 6995 6996 /** 6997 * ixgbe_set_prio_tc_map - Configure netdev prio tc map 6998 * @adapter: Pointer to adapter struct 6999 * 7000 * Populate the netdev user priority to tc map 7001 */ 7002 static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter) 7003 { 7004 struct net_device *dev = adapter->netdev; 7005 struct ixgbe_dcb_config *dcb_cfg = &adapter->dcb_cfg; 7006 struct ieee_ets *ets = adapter->ixgbe_ieee_ets; 7007 u8 prio; 7008 7009 for (prio = 0; prio < MAX_USER_PRIORITY; prio++) { 7010 u8 tc = 0; 7011 7012 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) 7013 tc = ixgbe_dcb_get_tc_from_up(dcb_cfg, 0, prio); 7014 else if (ets) 7015 tc = ets->prio_tc[prio]; 7016 7017 netdev_set_prio_tc_map(dev, prio, tc); 7018 } 7019 } 7020 7021 #endif /* CONFIG_IXGBE_DCB */ 7022 /** 7023 * ixgbe_setup_tc - configure net_device for multiple traffic classes 7024 * 7025 * @netdev: net device to configure 7026 * @tc: number of traffic classes to enable 7027 */ 7028 int ixgbe_setup_tc(struct net_device *dev, u8 tc) 7029 { 7030 struct ixgbe_adapter *adapter = netdev_priv(dev); 7031 struct ixgbe_hw *hw = &adapter->hw; 7032 7033 /* Hardware supports up to 8 traffic classes */ 7034 if (tc > adapter->dcb_cfg.num_tcs.pg_tcs || 7035 (hw->mac.type == ixgbe_mac_82598EB && 7036 tc < MAX_TRAFFIC_CLASS)) 7037 return -EINVAL; 7038 7039 /* Hardware has to reinitialize queues and interrupts to 7040 * match packet buffer alignment. Unfortunately, the 7041 * hardware is not flexible enough to do this dynamically. 7042 */ 7043 if (netif_running(dev)) 7044 ixgbe_close(dev); 7045 ixgbe_clear_interrupt_scheme(adapter); 7046 7047 #ifdef CONFIG_IXGBE_DCB 7048 if (tc) { 7049 netdev_set_num_tc(dev, tc); 7050 ixgbe_set_prio_tc_map(adapter); 7051 7052 adapter->flags |= IXGBE_FLAG_DCB_ENABLED; 7053 7054 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 7055 adapter->last_lfc_mode = adapter->hw.fc.requested_mode; 7056 adapter->hw.fc.requested_mode = ixgbe_fc_none; 7057 } 7058 } else { 7059 netdev_reset_tc(dev); 7060 7061 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 7062 adapter->hw.fc.requested_mode = adapter->last_lfc_mode; 7063 7064 adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED; 7065 7066 adapter->temp_dcb_cfg.pfc_mode_enable = false; 7067 adapter->dcb_cfg.pfc_mode_enable = false; 7068 } 7069 7070 ixgbe_validate_rtr(adapter, tc); 7071 7072 #endif /* CONFIG_IXGBE_DCB */ 7073 ixgbe_init_interrupt_scheme(adapter); 7074 7075 if (netif_running(dev)) 7076 return ixgbe_open(dev); 7077 7078 return 0; 7079 } 7080 7081 #ifdef CONFIG_PCI_IOV 7082 void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter) 7083 { 7084 struct net_device *netdev = adapter->netdev; 7085 7086 rtnl_lock(); 7087 ixgbe_setup_tc(netdev, netdev_get_num_tc(netdev)); 7088 rtnl_unlock(); 7089 } 7090 7091 #endif 7092 void ixgbe_do_reset(struct net_device *netdev) 7093 { 7094 struct ixgbe_adapter *adapter = netdev_priv(netdev); 7095 7096 if (netif_running(netdev)) 7097 ixgbe_reinit_locked(adapter); 7098 else 7099 ixgbe_reset(adapter); 7100 } 7101 7102 static netdev_features_t ixgbe_fix_features(struct net_device *netdev, 7103 netdev_features_t features) 7104 { 7105 struct ixgbe_adapter *adapter = netdev_priv(netdev); 7106 7107 /* If Rx checksum is disabled, then RSC/LRO should also be disabled */ 7108 if (!(features & NETIF_F_RXCSUM)) 7109 features &= ~NETIF_F_LRO; 7110 7111 /* Turn off LRO if not RSC capable */ 7112 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)) 7113 features &= ~NETIF_F_LRO; 7114 7115 return features; 7116 } 7117 7118 static int ixgbe_set_features(struct net_device *netdev, 7119 netdev_features_t features) 7120 { 7121 struct ixgbe_adapter *adapter = netdev_priv(netdev); 7122 netdev_features_t changed = netdev->features ^ features; 7123 bool need_reset = false; 7124 7125 /* Make sure RSC matches LRO, reset if change */ 7126 if (!(features & NETIF_F_LRO)) { 7127 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) 7128 need_reset = true; 7129 adapter->flags2 &= ~IXGBE_FLAG2_RSC_ENABLED; 7130 } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) && 7131 !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) { 7132 if (adapter->rx_itr_setting == 1 || 7133 adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { 7134 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; 7135 need_reset = true; 7136 } else if ((changed ^ features) & NETIF_F_LRO) { 7137 e_info(probe, "rx-usecs set too low, " 7138 "disabling RSC\n"); 7139 } 7140 } 7141 7142 /* 7143 * Check if Flow Director n-tuple support was enabled or disabled. If 7144 * the state changed, we need to reset. 7145 */ 7146 switch (features & NETIF_F_NTUPLE) { 7147 case NETIF_F_NTUPLE: 7148 /* turn off ATR, enable perfect filters and reset */ 7149 if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)) 7150 need_reset = true; 7151 7152 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; 7153 adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE; 7154 break; 7155 default: 7156 /* turn off perfect filters, enable ATR and reset */ 7157 if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) 7158 need_reset = true; 7159 7160 adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE; 7161 7162 /* We cannot enable ATR if SR-IOV is enabled */ 7163 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 7164 break; 7165 7166 /* We cannot enable ATR if we have 2 or more traffic classes */ 7167 if (netdev_get_num_tc(netdev) > 1) 7168 break; 7169 7170 /* We cannot enable ATR if RSS is disabled */ 7171 if (adapter->ring_feature[RING_F_RSS].limit <= 1) 7172 break; 7173 7174 /* A sample rate of 0 indicates ATR disabled */ 7175 if (!adapter->atr_sample_rate) 7176 break; 7177 7178 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE; 7179 break; 7180 } 7181 7182 if (features & NETIF_F_HW_VLAN_CTAG_RX) 7183 ixgbe_vlan_strip_enable(adapter); 7184 else 7185 ixgbe_vlan_strip_disable(adapter); 7186 7187 if (changed & NETIF_F_RXALL) 7188 need_reset = true; 7189 7190 netdev->features = features; 7191 if (need_reset) 7192 ixgbe_do_reset(netdev); 7193 7194 return 0; 7195 } 7196 7197 static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 7198 struct net_device *dev, 7199 const unsigned char *addr, 7200 u16 flags) 7201 { 7202 struct ixgbe_adapter *adapter = netdev_priv(dev); 7203 int err; 7204 7205 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 7206 return ndo_dflt_fdb_add(ndm, tb, dev, addr, flags); 7207 7208 /* Hardware does not support aging addresses so if a 7209 * ndm_state is given only allow permanent addresses 7210 */ 7211 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 7212 pr_info("%s: FDB only supports static addresses\n", 7213 ixgbe_driver_name); 7214 return -EINVAL; 7215 } 7216 7217 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) { 7218 u32 rar_uc_entries = IXGBE_MAX_PF_MACVLANS; 7219 7220 if (netdev_uc_count(dev) < rar_uc_entries) 7221 err = dev_uc_add_excl(dev, addr); 7222 else 7223 err = -ENOMEM; 7224 } else if (is_multicast_ether_addr(addr)) { 7225 err = dev_mc_add_excl(dev, addr); 7226 } else { 7227 err = -EINVAL; 7228 } 7229 7230 /* Only return duplicate errors if NLM_F_EXCL is set */ 7231 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 7232 err = 0; 7233 7234 return err; 7235 } 7236 7237 static int ixgbe_ndo_bridge_setlink(struct net_device *dev, 7238 struct nlmsghdr *nlh) 7239 { 7240 struct ixgbe_adapter *adapter = netdev_priv(dev); 7241 struct nlattr *attr, *br_spec; 7242 int rem; 7243 7244 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 7245 return -EOPNOTSUPP; 7246 7247 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 7248 7249 nla_for_each_nested(attr, br_spec, rem) { 7250 __u16 mode; 7251 u32 reg = 0; 7252 7253 if (nla_type(attr) != IFLA_BRIDGE_MODE) 7254 continue; 7255 7256 mode = nla_get_u16(attr); 7257 if (mode == BRIDGE_MODE_VEPA) { 7258 reg = 0; 7259 adapter->flags2 &= ~IXGBE_FLAG2_BRIDGE_MODE_VEB; 7260 } else if (mode == BRIDGE_MODE_VEB) { 7261 reg = IXGBE_PFDTXGSWC_VT_LBEN; 7262 adapter->flags2 |= IXGBE_FLAG2_BRIDGE_MODE_VEB; 7263 } else 7264 return -EINVAL; 7265 7266 IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC, reg); 7267 7268 e_info(drv, "enabling bridge mode: %s\n", 7269 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 7270 } 7271 7272 return 0; 7273 } 7274 7275 static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 7276 struct net_device *dev, 7277 u32 filter_mask) 7278 { 7279 struct ixgbe_adapter *adapter = netdev_priv(dev); 7280 u16 mode; 7281 7282 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 7283 return 0; 7284 7285 if (adapter->flags2 & IXGBE_FLAG2_BRIDGE_MODE_VEB) 7286 mode = BRIDGE_MODE_VEB; 7287 else 7288 mode = BRIDGE_MODE_VEPA; 7289 7290 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, mode); 7291 } 7292 7293 static const struct net_device_ops ixgbe_netdev_ops = { 7294 .ndo_open = ixgbe_open, 7295 .ndo_stop = ixgbe_close, 7296 .ndo_start_xmit = ixgbe_xmit_frame, 7297 #ifdef IXGBE_FCOE 7298 .ndo_select_queue = ixgbe_select_queue, 7299 #endif 7300 .ndo_set_rx_mode = ixgbe_set_rx_mode, 7301 .ndo_validate_addr = eth_validate_addr, 7302 .ndo_set_mac_address = ixgbe_set_mac, 7303 .ndo_change_mtu = ixgbe_change_mtu, 7304 .ndo_tx_timeout = ixgbe_tx_timeout, 7305 .ndo_vlan_rx_add_vid = ixgbe_vlan_rx_add_vid, 7306 .ndo_vlan_rx_kill_vid = ixgbe_vlan_rx_kill_vid, 7307 .ndo_do_ioctl = ixgbe_ioctl, 7308 .ndo_set_vf_mac = ixgbe_ndo_set_vf_mac, 7309 .ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan, 7310 .ndo_set_vf_tx_rate = ixgbe_ndo_set_vf_bw, 7311 .ndo_set_vf_spoofchk = ixgbe_ndo_set_vf_spoofchk, 7312 .ndo_get_vf_config = ixgbe_ndo_get_vf_config, 7313 .ndo_get_stats64 = ixgbe_get_stats64, 7314 #ifdef CONFIG_IXGBE_DCB 7315 .ndo_setup_tc = ixgbe_setup_tc, 7316 #endif 7317 #ifdef CONFIG_NET_POLL_CONTROLLER 7318 .ndo_poll_controller = ixgbe_netpoll, 7319 #endif 7320 #ifdef CONFIG_NET_RX_BUSY_POLL 7321 .ndo_busy_poll = ixgbe_low_latency_recv, 7322 #endif 7323 #ifdef IXGBE_FCOE 7324 .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get, 7325 .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target, 7326 .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put, 7327 .ndo_fcoe_enable = ixgbe_fcoe_enable, 7328 .ndo_fcoe_disable = ixgbe_fcoe_disable, 7329 .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn, 7330 .ndo_fcoe_get_hbainfo = ixgbe_fcoe_get_hbainfo, 7331 #endif /* IXGBE_FCOE */ 7332 .ndo_set_features = ixgbe_set_features, 7333 .ndo_fix_features = ixgbe_fix_features, 7334 .ndo_fdb_add = ixgbe_ndo_fdb_add, 7335 .ndo_bridge_setlink = ixgbe_ndo_bridge_setlink, 7336 .ndo_bridge_getlink = ixgbe_ndo_bridge_getlink, 7337 }; 7338 7339 /** 7340 * ixgbe_enumerate_functions - Get the number of ports this device has 7341 * @adapter: adapter structure 7342 * 7343 * This function enumerates the phsyical functions co-located on a single slot, 7344 * in order to determine how many ports a device has. This is most useful in 7345 * determining the required GT/s of PCIe bandwidth necessary for optimal 7346 * performance. 7347 **/ 7348 static inline int ixgbe_enumerate_functions(struct ixgbe_adapter *adapter) 7349 { 7350 struct ixgbe_hw *hw = &adapter->hw; 7351 struct list_head *entry; 7352 int physfns = 0; 7353 7354 /* Some cards can not use the generic count PCIe functions method, and 7355 * so must be hardcoded to the correct value. 7356 */ 7357 switch (hw->device_id) { 7358 case IXGBE_DEV_ID_82599_SFP_SF_QP: 7359 case IXGBE_DEV_ID_82599_QSFP_SF_QP: 7360 physfns = 4; 7361 break; 7362 default: 7363 list_for_each(entry, &adapter->pdev->bus_list) { 7364 struct pci_dev *pdev = 7365 list_entry(entry, struct pci_dev, bus_list); 7366 /* don't count virtual functions */ 7367 if (!pdev->is_virtfn) 7368 physfns++; 7369 } 7370 } 7371 7372 return physfns; 7373 } 7374 7375 /** 7376 * ixgbe_wol_supported - Check whether device supports WoL 7377 * @hw: hw specific details 7378 * @device_id: the device ID 7379 * @subdev_id: the subsystem device ID 7380 * 7381 * This function is used by probe and ethtool to determine 7382 * which devices have WoL support 7383 * 7384 **/ 7385 int ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id, 7386 u16 subdevice_id) 7387 { 7388 struct ixgbe_hw *hw = &adapter->hw; 7389 u16 wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK; 7390 int is_wol_supported = 0; 7391 7392 switch (device_id) { 7393 case IXGBE_DEV_ID_82599_SFP: 7394 /* Only these subdevices could supports WOL */ 7395 switch (subdevice_id) { 7396 case IXGBE_SUBDEV_ID_82599_560FLR: 7397 /* only support first port */ 7398 if (hw->bus.func != 0) 7399 break; 7400 case IXGBE_SUBDEV_ID_82599_SP_560FLR: 7401 case IXGBE_SUBDEV_ID_82599_SFP: 7402 case IXGBE_SUBDEV_ID_82599_RNDC: 7403 case IXGBE_SUBDEV_ID_82599_ECNA_DP: 7404 case IXGBE_SUBDEV_ID_82599_LOM_SFP: 7405 is_wol_supported = 1; 7406 break; 7407 } 7408 break; 7409 case IXGBE_DEV_ID_82599EN_SFP: 7410 /* Only this subdevice supports WOL */ 7411 switch (subdevice_id) { 7412 case IXGBE_SUBDEV_ID_82599EN_SFP_OCP1: 7413 is_wol_supported = 1; 7414 break; 7415 } 7416 break; 7417 case IXGBE_DEV_ID_82599_COMBO_BACKPLANE: 7418 /* All except this subdevice support WOL */ 7419 if (subdevice_id != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ) 7420 is_wol_supported = 1; 7421 break; 7422 case IXGBE_DEV_ID_82599_KX4: 7423 is_wol_supported = 1; 7424 break; 7425 case IXGBE_DEV_ID_X540T: 7426 case IXGBE_DEV_ID_X540T1: 7427 /* check eeprom to see if enabled wol */ 7428 if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) || 7429 ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && 7430 (hw->bus.func == 0))) { 7431 is_wol_supported = 1; 7432 } 7433 break; 7434 } 7435 7436 return is_wol_supported; 7437 } 7438 7439 /** 7440 * ixgbe_probe - Device Initialization Routine 7441 * @pdev: PCI device information struct 7442 * @ent: entry in ixgbe_pci_tbl 7443 * 7444 * Returns 0 on success, negative on failure 7445 * 7446 * ixgbe_probe initializes an adapter identified by a pci_dev structure. 7447 * The OS initialization, configuring of the adapter private structure, 7448 * and a hardware reset occur. 7449 **/ 7450 static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 7451 { 7452 struct net_device *netdev; 7453 struct ixgbe_adapter *adapter = NULL; 7454 struct ixgbe_hw *hw; 7455 const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data]; 7456 static int cards_found; 7457 int i, err, pci_using_dac, expected_gts; 7458 unsigned int indices = MAX_TX_QUEUES; 7459 u8 part_str[IXGBE_PBANUM_LENGTH]; 7460 #ifdef IXGBE_FCOE 7461 u16 device_caps; 7462 #endif 7463 u32 eec; 7464 7465 /* Catch broken hardware that put the wrong VF device ID in 7466 * the PCIe SR-IOV capability. 7467 */ 7468 if (pdev->is_virtfn) { 7469 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n", 7470 pci_name(pdev), pdev->vendor, pdev->device); 7471 return -EINVAL; 7472 } 7473 7474 err = pci_enable_device_mem(pdev); 7475 if (err) 7476 return err; 7477 7478 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && 7479 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { 7480 pci_using_dac = 1; 7481 } else { 7482 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 7483 if (err) { 7484 err = dma_set_coherent_mask(&pdev->dev, 7485 DMA_BIT_MASK(32)); 7486 if (err) { 7487 dev_err(&pdev->dev, 7488 "No usable DMA configuration, aborting\n"); 7489 goto err_dma; 7490 } 7491 } 7492 pci_using_dac = 0; 7493 } 7494 7495 err = pci_request_selected_regions(pdev, pci_select_bars(pdev, 7496 IORESOURCE_MEM), ixgbe_driver_name); 7497 if (err) { 7498 dev_err(&pdev->dev, 7499 "pci_request_selected_regions failed 0x%x\n", err); 7500 goto err_pci_reg; 7501 } 7502 7503 pci_enable_pcie_error_reporting(pdev); 7504 7505 pci_set_master(pdev); 7506 pci_save_state(pdev); 7507 7508 if (ii->mac == ixgbe_mac_82598EB) { 7509 #ifdef CONFIG_IXGBE_DCB 7510 /* 8 TC w/ 4 queues per TC */ 7511 indices = 4 * MAX_TRAFFIC_CLASS; 7512 #else 7513 indices = IXGBE_MAX_RSS_INDICES; 7514 #endif 7515 } 7516 7517 netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices); 7518 if (!netdev) { 7519 err = -ENOMEM; 7520 goto err_alloc_etherdev; 7521 } 7522 7523 SET_NETDEV_DEV(netdev, &pdev->dev); 7524 7525 adapter = netdev_priv(netdev); 7526 pci_set_drvdata(pdev, adapter); 7527 7528 adapter->netdev = netdev; 7529 adapter->pdev = pdev; 7530 hw = &adapter->hw; 7531 hw->back = adapter; 7532 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 7533 7534 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 7535 pci_resource_len(pdev, 0)); 7536 if (!hw->hw_addr) { 7537 err = -EIO; 7538 goto err_ioremap; 7539 } 7540 7541 netdev->netdev_ops = &ixgbe_netdev_ops; 7542 ixgbe_set_ethtool_ops(netdev); 7543 netdev->watchdog_timeo = 5 * HZ; 7544 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1); 7545 7546 adapter->bd_number = cards_found; 7547 7548 /* Setup hw api */ 7549 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 7550 hw->mac.type = ii->mac; 7551 7552 /* EEPROM */ 7553 memcpy(&hw->eeprom.ops, ii->eeprom_ops, sizeof(hw->eeprom.ops)); 7554 eec = IXGBE_READ_REG(hw, IXGBE_EEC); 7555 /* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */ 7556 if (!(eec & (1 << 8))) 7557 hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic; 7558 7559 /* PHY */ 7560 memcpy(&hw->phy.ops, ii->phy_ops, sizeof(hw->phy.ops)); 7561 hw->phy.sfp_type = ixgbe_sfp_type_unknown; 7562 /* ixgbe_identify_phy_generic will set prtad and mmds properly */ 7563 hw->phy.mdio.prtad = MDIO_PRTAD_NONE; 7564 hw->phy.mdio.mmds = 0; 7565 hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22; 7566 hw->phy.mdio.dev = netdev; 7567 hw->phy.mdio.mdio_read = ixgbe_mdio_read; 7568 hw->phy.mdio.mdio_write = ixgbe_mdio_write; 7569 7570 ii->get_invariants(hw); 7571 7572 /* setup the private structure */ 7573 err = ixgbe_sw_init(adapter); 7574 if (err) 7575 goto err_sw_init; 7576 7577 /* Cache if MNG FW is up so we don't have to read the REG later */ 7578 if (hw->mac.ops.mng_fw_enabled) 7579 hw->mng_fw_enabled = hw->mac.ops.mng_fw_enabled(hw); 7580 7581 /* Make it possible the adapter to be woken up via WOL */ 7582 switch (adapter->hw.mac.type) { 7583 case ixgbe_mac_82599EB: 7584 case ixgbe_mac_X540: 7585 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); 7586 break; 7587 default: 7588 break; 7589 } 7590 7591 /* 7592 * If there is a fan on this device and it has failed log the 7593 * failure. 7594 */ 7595 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) { 7596 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 7597 if (esdp & IXGBE_ESDP_SDP1) 7598 e_crit(probe, "Fan has stopped, replace the adapter\n"); 7599 } 7600 7601 if (allow_unsupported_sfp) 7602 hw->allow_unsupported_sfp = allow_unsupported_sfp; 7603 7604 /* reset_hw fills in the perm_addr as well */ 7605 hw->phy.reset_if_overtemp = true; 7606 err = hw->mac.ops.reset_hw(hw); 7607 hw->phy.reset_if_overtemp = false; 7608 if (err == IXGBE_ERR_SFP_NOT_PRESENT && 7609 hw->mac.type == ixgbe_mac_82598EB) { 7610 err = 0; 7611 } else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 7612 e_dev_err("failed to load because an unsupported SFP+ or QSFP module type was detected.\n"); 7613 e_dev_err("Reload the driver after installing a supported module.\n"); 7614 goto err_sw_init; 7615 } else if (err) { 7616 e_dev_err("HW Init failed: %d\n", err); 7617 goto err_sw_init; 7618 } 7619 7620 #ifdef CONFIG_PCI_IOV 7621 /* SR-IOV not supported on the 82598 */ 7622 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 7623 goto skip_sriov; 7624 /* Mailbox */ 7625 ixgbe_init_mbx_params_pf(hw); 7626 memcpy(&hw->mbx.ops, ii->mbx_ops, sizeof(hw->mbx.ops)); 7627 ixgbe_enable_sriov(adapter); 7628 pci_sriov_set_totalvfs(pdev, 63); 7629 skip_sriov: 7630 7631 #endif 7632 netdev->features = NETIF_F_SG | 7633 NETIF_F_IP_CSUM | 7634 NETIF_F_IPV6_CSUM | 7635 NETIF_F_HW_VLAN_CTAG_TX | 7636 NETIF_F_HW_VLAN_CTAG_RX | 7637 NETIF_F_HW_VLAN_CTAG_FILTER | 7638 NETIF_F_TSO | 7639 NETIF_F_TSO6 | 7640 NETIF_F_RXHASH | 7641 NETIF_F_RXCSUM; 7642 7643 netdev->hw_features = netdev->features; 7644 7645 switch (adapter->hw.mac.type) { 7646 case ixgbe_mac_82599EB: 7647 case ixgbe_mac_X540: 7648 netdev->features |= NETIF_F_SCTP_CSUM; 7649 netdev->hw_features |= NETIF_F_SCTP_CSUM | 7650 NETIF_F_NTUPLE; 7651 break; 7652 default: 7653 break; 7654 } 7655 7656 netdev->hw_features |= NETIF_F_RXALL; 7657 7658 netdev->vlan_features |= NETIF_F_TSO; 7659 netdev->vlan_features |= NETIF_F_TSO6; 7660 netdev->vlan_features |= NETIF_F_IP_CSUM; 7661 netdev->vlan_features |= NETIF_F_IPV6_CSUM; 7662 netdev->vlan_features |= NETIF_F_SG; 7663 7664 netdev->priv_flags |= IFF_UNICAST_FLT; 7665 netdev->priv_flags |= IFF_SUPP_NOFCS; 7666 7667 #ifdef CONFIG_IXGBE_DCB 7668 netdev->dcbnl_ops = &dcbnl_ops; 7669 #endif 7670 7671 #ifdef IXGBE_FCOE 7672 if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) { 7673 unsigned int fcoe_l; 7674 7675 if (hw->mac.ops.get_device_caps) { 7676 hw->mac.ops.get_device_caps(hw, &device_caps); 7677 if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS) 7678 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE; 7679 } 7680 7681 7682 fcoe_l = min_t(int, IXGBE_FCRETA_SIZE, num_online_cpus()); 7683 adapter->ring_feature[RING_F_FCOE].limit = fcoe_l; 7684 7685 netdev->features |= NETIF_F_FSO | 7686 NETIF_F_FCOE_CRC; 7687 7688 netdev->vlan_features |= NETIF_F_FSO | 7689 NETIF_F_FCOE_CRC | 7690 NETIF_F_FCOE_MTU; 7691 } 7692 #endif /* IXGBE_FCOE */ 7693 if (pci_using_dac) { 7694 netdev->features |= NETIF_F_HIGHDMA; 7695 netdev->vlan_features |= NETIF_F_HIGHDMA; 7696 } 7697 7698 if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) 7699 netdev->hw_features |= NETIF_F_LRO; 7700 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) 7701 netdev->features |= NETIF_F_LRO; 7702 7703 /* make sure the EEPROM is good */ 7704 if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) { 7705 e_dev_err("The EEPROM Checksum Is Not Valid\n"); 7706 err = -EIO; 7707 goto err_sw_init; 7708 } 7709 7710 memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len); 7711 7712 if (!is_valid_ether_addr(netdev->dev_addr)) { 7713 e_dev_err("invalid MAC address\n"); 7714 err = -EIO; 7715 goto err_sw_init; 7716 } 7717 7718 setup_timer(&adapter->service_timer, &ixgbe_service_timer, 7719 (unsigned long) adapter); 7720 7721 INIT_WORK(&adapter->service_task, ixgbe_service_task); 7722 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state); 7723 7724 err = ixgbe_init_interrupt_scheme(adapter); 7725 if (err) 7726 goto err_sw_init; 7727 7728 /* WOL not supported for all devices */ 7729 adapter->wol = 0; 7730 hw->eeprom.ops.read(hw, 0x2c, &adapter->eeprom_cap); 7731 hw->wol_enabled = ixgbe_wol_supported(adapter, pdev->device, 7732 pdev->subsystem_device); 7733 if (hw->wol_enabled) 7734 adapter->wol = IXGBE_WUFC_MAG; 7735 7736 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol); 7737 7738 /* save off EEPROM version number */ 7739 hw->eeprom.ops.read(hw, 0x2e, &adapter->eeprom_verh); 7740 hw->eeprom.ops.read(hw, 0x2d, &adapter->eeprom_verl); 7741 7742 /* pick up the PCI bus settings for reporting later */ 7743 hw->mac.ops.get_bus_info(hw); 7744 if (ixgbe_pcie_from_parent(hw)) 7745 ixgbe_get_parent_bus_info(adapter); 7746 7747 /* print bus type/speed/width info */ 7748 e_dev_info("(PCI Express:%s:%s) %pM\n", 7749 (hw->bus.speed == ixgbe_bus_speed_8000 ? "8.0GT/s" : 7750 hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" : 7751 hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" : 7752 "Unknown"), 7753 (hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" : 7754 hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" : 7755 hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" : 7756 "Unknown"), 7757 netdev->dev_addr); 7758 7759 err = ixgbe_read_pba_string_generic(hw, part_str, IXGBE_PBANUM_LENGTH); 7760 if (err) 7761 strncpy(part_str, "Unknown", IXGBE_PBANUM_LENGTH); 7762 if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present) 7763 e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n", 7764 hw->mac.type, hw->phy.type, hw->phy.sfp_type, 7765 part_str); 7766 else 7767 e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n", 7768 hw->mac.type, hw->phy.type, part_str); 7769 7770 /* calculate the expected PCIe bandwidth required for optimal 7771 * performance. Note that some older parts will never have enough 7772 * bandwidth due to being older generation PCIe parts. We clamp these 7773 * parts to ensure no warning is displayed if it can't be fixed. 7774 */ 7775 switch (hw->mac.type) { 7776 case ixgbe_mac_82598EB: 7777 expected_gts = min(ixgbe_enumerate_functions(adapter) * 10, 16); 7778 break; 7779 default: 7780 expected_gts = ixgbe_enumerate_functions(adapter) * 10; 7781 break; 7782 } 7783 ixgbe_check_minimum_link(adapter, expected_gts); 7784 7785 /* reset the hardware with the new settings */ 7786 err = hw->mac.ops.start_hw(hw); 7787 if (err == IXGBE_ERR_EEPROM_VERSION) { 7788 /* We are running on a pre-production device, log a warning */ 7789 e_dev_warn("This device is a pre-production adapter/LOM. " 7790 "Please be aware there may be issues associated " 7791 "with your hardware. If you are experiencing " 7792 "problems please contact your Intel or hardware " 7793 "representative who provided you with this " 7794 "hardware.\n"); 7795 } 7796 strcpy(netdev->name, "eth%d"); 7797 err = register_netdev(netdev); 7798 if (err) 7799 goto err_register; 7800 7801 /* power down the optics for 82599 SFP+ fiber */ 7802 if (hw->mac.ops.disable_tx_laser) 7803 hw->mac.ops.disable_tx_laser(hw); 7804 7805 /* carrier off reporting is important to ethtool even BEFORE open */ 7806 netif_carrier_off(netdev); 7807 7808 #ifdef CONFIG_IXGBE_DCA 7809 if (dca_add_requester(&pdev->dev) == 0) { 7810 adapter->flags |= IXGBE_FLAG_DCA_ENABLED; 7811 ixgbe_setup_dca(adapter); 7812 } 7813 #endif 7814 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 7815 e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs); 7816 for (i = 0; i < adapter->num_vfs; i++) 7817 ixgbe_vf_configuration(pdev, (i | 0x10000000)); 7818 } 7819 7820 /* firmware requires driver version to be 0xFFFFFFFF 7821 * since os does not support feature 7822 */ 7823 if (hw->mac.ops.set_fw_drv_ver) 7824 hw->mac.ops.set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF, 7825 0xFF); 7826 7827 /* add san mac addr to netdev */ 7828 ixgbe_add_sanmac_netdev(netdev); 7829 7830 e_dev_info("%s\n", ixgbe_default_device_descr); 7831 cards_found++; 7832 7833 #ifdef CONFIG_IXGBE_HWMON 7834 if (ixgbe_sysfs_init(adapter)) 7835 e_err(probe, "failed to allocate sysfs resources\n"); 7836 #endif /* CONFIG_IXGBE_HWMON */ 7837 7838 ixgbe_dbg_adapter_init(adapter); 7839 7840 /* Need link setup for MNG FW, else wait for IXGBE_UP */ 7841 if (hw->mng_fw_enabled && hw->mac.ops.setup_link) 7842 hw->mac.ops.setup_link(hw, 7843 IXGBE_LINK_SPEED_10GB_FULL | IXGBE_LINK_SPEED_1GB_FULL, 7844 true); 7845 7846 return 0; 7847 7848 err_register: 7849 ixgbe_release_hw_control(adapter); 7850 ixgbe_clear_interrupt_scheme(adapter); 7851 err_sw_init: 7852 ixgbe_disable_sriov(adapter); 7853 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP; 7854 iounmap(hw->hw_addr); 7855 err_ioremap: 7856 free_netdev(netdev); 7857 err_alloc_etherdev: 7858 pci_release_selected_regions(pdev, 7859 pci_select_bars(pdev, IORESOURCE_MEM)); 7860 err_pci_reg: 7861 err_dma: 7862 pci_disable_device(pdev); 7863 return err; 7864 } 7865 7866 /** 7867 * ixgbe_remove - Device Removal Routine 7868 * @pdev: PCI device information struct 7869 * 7870 * ixgbe_remove is called by the PCI subsystem to alert the driver 7871 * that it should release a PCI device. The could be caused by a 7872 * Hot-Plug event, or because the driver is going to be removed from 7873 * memory. 7874 **/ 7875 static void ixgbe_remove(struct pci_dev *pdev) 7876 { 7877 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 7878 struct net_device *netdev = adapter->netdev; 7879 7880 ixgbe_dbg_adapter_exit(adapter); 7881 7882 set_bit(__IXGBE_DOWN, &adapter->state); 7883 cancel_work_sync(&adapter->service_task); 7884 7885 7886 #ifdef CONFIG_IXGBE_DCA 7887 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) { 7888 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED; 7889 dca_remove_requester(&pdev->dev); 7890 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1); 7891 } 7892 7893 #endif 7894 #ifdef CONFIG_IXGBE_HWMON 7895 ixgbe_sysfs_exit(adapter); 7896 #endif /* CONFIG_IXGBE_HWMON */ 7897 7898 /* remove the added san mac */ 7899 ixgbe_del_sanmac_netdev(netdev); 7900 7901 if (netdev->reg_state == NETREG_REGISTERED) 7902 unregister_netdev(netdev); 7903 7904 #ifdef CONFIG_PCI_IOV 7905 /* 7906 * Only disable SR-IOV on unload if the user specified the now 7907 * deprecated max_vfs module parameter. 7908 */ 7909 if (max_vfs) 7910 ixgbe_disable_sriov(adapter); 7911 #endif 7912 ixgbe_clear_interrupt_scheme(adapter); 7913 7914 ixgbe_release_hw_control(adapter); 7915 7916 #ifdef CONFIG_DCB 7917 kfree(adapter->ixgbe_ieee_pfc); 7918 kfree(adapter->ixgbe_ieee_ets); 7919 7920 #endif 7921 iounmap(adapter->hw.hw_addr); 7922 pci_release_selected_regions(pdev, pci_select_bars(pdev, 7923 IORESOURCE_MEM)); 7924 7925 e_dev_info("complete\n"); 7926 7927 free_netdev(netdev); 7928 7929 pci_disable_pcie_error_reporting(pdev); 7930 7931 pci_disable_device(pdev); 7932 } 7933 7934 /** 7935 * ixgbe_io_error_detected - called when PCI error is detected 7936 * @pdev: Pointer to PCI device 7937 * @state: The current pci connection state 7938 * 7939 * This function is called after a PCI bus error affecting 7940 * this device has been detected. 7941 */ 7942 static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, 7943 pci_channel_state_t state) 7944 { 7945 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 7946 struct net_device *netdev = adapter->netdev; 7947 7948 #ifdef CONFIG_PCI_IOV 7949 struct pci_dev *bdev, *vfdev; 7950 u32 dw0, dw1, dw2, dw3; 7951 int vf, pos; 7952 u16 req_id, pf_func; 7953 7954 if (adapter->hw.mac.type == ixgbe_mac_82598EB || 7955 adapter->num_vfs == 0) 7956 goto skip_bad_vf_detection; 7957 7958 bdev = pdev->bus->self; 7959 while (bdev && (pci_pcie_type(bdev) != PCI_EXP_TYPE_ROOT_PORT)) 7960 bdev = bdev->bus->self; 7961 7962 if (!bdev) 7963 goto skip_bad_vf_detection; 7964 7965 pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR); 7966 if (!pos) 7967 goto skip_bad_vf_detection; 7968 7969 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG, &dw0); 7970 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 4, &dw1); 7971 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 8, &dw2); 7972 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 12, &dw3); 7973 7974 req_id = dw1 >> 16; 7975 /* On the 82599 if bit 7 of the requestor ID is set then it's a VF */ 7976 if (!(req_id & 0x0080)) 7977 goto skip_bad_vf_detection; 7978 7979 pf_func = req_id & 0x01; 7980 if ((pf_func & 1) == (pdev->devfn & 1)) { 7981 unsigned int device_id; 7982 7983 vf = (req_id & 0x7F) >> 1; 7984 e_dev_err("VF %d has caused a PCIe error\n", vf); 7985 e_dev_err("TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: " 7986 "%8.8x\tdw3: %8.8x\n", 7987 dw0, dw1, dw2, dw3); 7988 switch (adapter->hw.mac.type) { 7989 case ixgbe_mac_82599EB: 7990 device_id = IXGBE_82599_VF_DEVICE_ID; 7991 break; 7992 case ixgbe_mac_X540: 7993 device_id = IXGBE_X540_VF_DEVICE_ID; 7994 break; 7995 default: 7996 device_id = 0; 7997 break; 7998 } 7999 8000 /* Find the pci device of the offending VF */ 8001 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, device_id, NULL); 8002 while (vfdev) { 8003 if (vfdev->devfn == (req_id & 0xFF)) 8004 break; 8005 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, 8006 device_id, vfdev); 8007 } 8008 /* 8009 * There's a slim chance the VF could have been hot plugged, 8010 * so if it is no longer present we don't need to issue the 8011 * VFLR. Just clean up the AER in that case. 8012 */ 8013 if (vfdev) { 8014 e_dev_err("Issuing VFLR to VF %d\n", vf); 8015 pci_write_config_dword(vfdev, 0xA8, 0x00008000); 8016 /* Free device reference count */ 8017 pci_dev_put(vfdev); 8018 } 8019 8020 pci_cleanup_aer_uncorrect_error_status(pdev); 8021 } 8022 8023 /* 8024 * Even though the error may have occurred on the other port 8025 * we still need to increment the vf error reference count for 8026 * both ports because the I/O resume function will be called 8027 * for both of them. 8028 */ 8029 adapter->vferr_refcount++; 8030 8031 return PCI_ERS_RESULT_RECOVERED; 8032 8033 skip_bad_vf_detection: 8034 #endif /* CONFIG_PCI_IOV */ 8035 netif_device_detach(netdev); 8036 8037 if (state == pci_channel_io_perm_failure) 8038 return PCI_ERS_RESULT_DISCONNECT; 8039 8040 if (netif_running(netdev)) 8041 ixgbe_down(adapter); 8042 pci_disable_device(pdev); 8043 8044 /* Request a slot reset. */ 8045 return PCI_ERS_RESULT_NEED_RESET; 8046 } 8047 8048 /** 8049 * ixgbe_io_slot_reset - called after the pci bus has been reset. 8050 * @pdev: Pointer to PCI device 8051 * 8052 * Restart the card from scratch, as if from a cold-boot. 8053 */ 8054 static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev) 8055 { 8056 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 8057 pci_ers_result_t result; 8058 int err; 8059 8060 if (pci_enable_device_mem(pdev)) { 8061 e_err(probe, "Cannot re-enable PCI device after reset.\n"); 8062 result = PCI_ERS_RESULT_DISCONNECT; 8063 } else { 8064 pci_set_master(pdev); 8065 pci_restore_state(pdev); 8066 pci_save_state(pdev); 8067 8068 pci_wake_from_d3(pdev, false); 8069 8070 ixgbe_reset(adapter); 8071 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); 8072 result = PCI_ERS_RESULT_RECOVERED; 8073 } 8074 8075 err = pci_cleanup_aer_uncorrect_error_status(pdev); 8076 if (err) { 8077 e_dev_err("pci_cleanup_aer_uncorrect_error_status " 8078 "failed 0x%0x\n", err); 8079 /* non-fatal, continue */ 8080 } 8081 8082 return result; 8083 } 8084 8085 /** 8086 * ixgbe_io_resume - called when traffic can start flowing again. 8087 * @pdev: Pointer to PCI device 8088 * 8089 * This callback is called when the error recovery driver tells us that 8090 * its OK to resume normal operation. 8091 */ 8092 static void ixgbe_io_resume(struct pci_dev *pdev) 8093 { 8094 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 8095 struct net_device *netdev = adapter->netdev; 8096 8097 #ifdef CONFIG_PCI_IOV 8098 if (adapter->vferr_refcount) { 8099 e_info(drv, "Resuming after VF err\n"); 8100 adapter->vferr_refcount--; 8101 return; 8102 } 8103 8104 #endif 8105 if (netif_running(netdev)) 8106 ixgbe_up(adapter); 8107 8108 netif_device_attach(netdev); 8109 } 8110 8111 static const struct pci_error_handlers ixgbe_err_handler = { 8112 .error_detected = ixgbe_io_error_detected, 8113 .slot_reset = ixgbe_io_slot_reset, 8114 .resume = ixgbe_io_resume, 8115 }; 8116 8117 static struct pci_driver ixgbe_driver = { 8118 .name = ixgbe_driver_name, 8119 .id_table = ixgbe_pci_tbl, 8120 .probe = ixgbe_probe, 8121 .remove = ixgbe_remove, 8122 #ifdef CONFIG_PM 8123 .suspend = ixgbe_suspend, 8124 .resume = ixgbe_resume, 8125 #endif 8126 .shutdown = ixgbe_shutdown, 8127 .sriov_configure = ixgbe_pci_sriov_configure, 8128 .err_handler = &ixgbe_err_handler 8129 }; 8130 8131 /** 8132 * ixgbe_init_module - Driver Registration Routine 8133 * 8134 * ixgbe_init_module is the first routine called when the driver is 8135 * loaded. All it does is register with the PCI subsystem. 8136 **/ 8137 static int __init ixgbe_init_module(void) 8138 { 8139 int ret; 8140 pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version); 8141 pr_info("%s\n", ixgbe_copyright); 8142 8143 ixgbe_dbg_init(); 8144 8145 ret = pci_register_driver(&ixgbe_driver); 8146 if (ret) { 8147 ixgbe_dbg_exit(); 8148 return ret; 8149 } 8150 8151 #ifdef CONFIG_IXGBE_DCA 8152 dca_register_notify(&dca_notifier); 8153 #endif 8154 8155 return 0; 8156 } 8157 8158 module_init(ixgbe_init_module); 8159 8160 /** 8161 * ixgbe_exit_module - Driver Exit Cleanup Routine 8162 * 8163 * ixgbe_exit_module is called just before the driver is removed 8164 * from memory. 8165 **/ 8166 static void __exit ixgbe_exit_module(void) 8167 { 8168 #ifdef CONFIG_IXGBE_DCA 8169 dca_unregister_notify(&dca_notifier); 8170 #endif 8171 pci_unregister_driver(&ixgbe_driver); 8172 8173 ixgbe_dbg_exit(); 8174 8175 rcu_barrier(); /* Wait for completion of call_rcu()'s */ 8176 } 8177 8178 #ifdef CONFIG_IXGBE_DCA 8179 static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event, 8180 void *p) 8181 { 8182 int ret_val; 8183 8184 ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event, 8185 __ixgbe_notify_dca); 8186 8187 return ret_val ? NOTIFY_BAD : NOTIFY_DONE; 8188 } 8189 8190 #endif /* CONFIG_IXGBE_DCA */ 8191 8192 module_exit(ixgbe_exit_module); 8193 8194 /* ixgbe_main.c */ 8195