1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2007 - 2018 Intel Corporation. */ 3 4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 5 6 #include <linux/module.h> 7 #include <linux/types.h> 8 #include <linux/init.h> 9 #include <linux/bitops.h> 10 #include <linux/vmalloc.h> 11 #include <linux/pagemap.h> 12 #include <linux/netdevice.h> 13 #include <linux/ipv6.h> 14 #include <linux/slab.h> 15 #include <net/checksum.h> 16 #include <net/ip6_checksum.h> 17 #include <net/pkt_sched.h> 18 #include <net/pkt_cls.h> 19 #include <linux/net_tstamp.h> 20 #include <linux/mii.h> 21 #include <linux/ethtool.h> 22 #include <linux/if.h> 23 #include <linux/if_vlan.h> 24 #include <linux/pci.h> 25 #include <linux/delay.h> 26 #include <linux/interrupt.h> 27 #include <linux/ip.h> 28 #include <linux/tcp.h> 29 #include <linux/sctp.h> 30 #include <linux/if_ether.h> 31 #include <linux/prefetch.h> 32 #include <linux/bpf.h> 33 #include <linux/bpf_trace.h> 34 #include <linux/pm_runtime.h> 35 #include <linux/etherdevice.h> 36 #include <linux/lockdep.h> 37 #ifdef CONFIG_IGB_DCA 38 #include <linux/dca.h> 39 #endif 40 #include <linux/i2c.h> 41 #include "igb.h" 42 43 enum queue_mode { 44 QUEUE_MODE_STRICT_PRIORITY, 45 QUEUE_MODE_STREAM_RESERVATION, 46 }; 47 48 enum tx_queue_prio { 49 TX_QUEUE_PRIO_HIGH, 50 TX_QUEUE_PRIO_LOW, 51 }; 52 53 char igb_driver_name[] = "igb"; 54 static const char igb_driver_string[] = 55 "Intel(R) Gigabit Ethernet Network Driver"; 56 static const char igb_copyright[] = 57 "Copyright (c) 2007-2014 Intel Corporation."; 58 59 static const struct e1000_info *igb_info_tbl[] = { 60 [board_82575] = &e1000_82575_info, 61 }; 62 63 static const struct pci_device_id igb_pci_tbl[] = { 64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) }, 66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 }, 68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 }, 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 }, 70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 }, 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 }, 72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 }, 73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 }, 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 }, 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 }, 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 }, 77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 }, 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 }, 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 }, 80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 }, 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 }, 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 }, 83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 }, 84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 }, 85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 }, 86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 }, 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 }, 88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 }, 89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 }, 90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 }, 91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 }, 92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 }, 93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 }, 94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 }, 95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 }, 96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 }, 97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 }, 98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 }, 99 /* required last entry */ 100 {0, } 101 }; 102 103 MODULE_DEVICE_TABLE(pci, igb_pci_tbl); 104 105 static int igb_setup_all_tx_resources(struct igb_adapter *); 106 static int igb_setup_all_rx_resources(struct igb_adapter *); 107 static void igb_free_all_tx_resources(struct igb_adapter *); 108 static void igb_free_all_rx_resources(struct igb_adapter *); 109 static void igb_setup_mrqc(struct igb_adapter *); 110 static int igb_probe(struct pci_dev *, const struct pci_device_id *); 111 static void igb_remove(struct pci_dev *pdev); 112 static void igb_init_queue_configuration(struct igb_adapter *adapter); 113 static int igb_sw_init(struct igb_adapter *); 114 int igb_open(struct net_device *); 115 int igb_close(struct net_device *); 116 static void igb_configure(struct igb_adapter *); 117 static void igb_configure_tx(struct igb_adapter *); 118 static void igb_configure_rx(struct igb_adapter *); 119 static void igb_clean_all_tx_rings(struct igb_adapter *); 120 static void igb_clean_all_rx_rings(struct igb_adapter *); 121 static void igb_clean_tx_ring(struct igb_ring *); 122 static void igb_clean_rx_ring(struct igb_ring *); 123 static void igb_set_rx_mode(struct net_device *); 124 static void igb_update_phy_info(struct timer_list *); 125 static void igb_watchdog(struct timer_list *); 126 static void igb_watchdog_task(struct work_struct *); 127 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *); 128 static void igb_get_stats64(struct net_device *dev, 129 struct rtnl_link_stats64 *stats); 130 static int igb_change_mtu(struct net_device *, int); 131 static int igb_set_mac(struct net_device *, void *); 132 static void igb_set_uta(struct igb_adapter *adapter, bool set); 133 static irqreturn_t igb_intr(int irq, void *); 134 static irqreturn_t igb_intr_msi(int irq, void *); 135 static irqreturn_t igb_msix_other(int irq, void *); 136 static irqreturn_t igb_msix_ring(int irq, void *); 137 #ifdef CONFIG_IGB_DCA 138 static void igb_update_dca(struct igb_q_vector *); 139 static void igb_setup_dca(struct igb_adapter *); 140 #endif /* CONFIG_IGB_DCA */ 141 static int igb_poll(struct napi_struct *, int); 142 static bool igb_clean_tx_irq(struct igb_q_vector *, int); 143 static int igb_clean_rx_irq(struct igb_q_vector *, int); 144 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd); 145 static void igb_tx_timeout(struct net_device *, unsigned int txqueue); 146 static void igb_reset_task(struct work_struct *); 147 static void igb_vlan_mode(struct net_device *netdev, 148 netdev_features_t features); 149 static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16); 150 static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16); 151 static void igb_restore_vlan(struct igb_adapter *); 152 static void igb_rar_set_index(struct igb_adapter *, u32); 153 static void igb_ping_all_vfs(struct igb_adapter *); 154 static void igb_msg_task(struct igb_adapter *); 155 static void igb_vmm_control(struct igb_adapter *); 156 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *); 157 static void igb_flush_mac_table(struct igb_adapter *); 158 static int igb_available_rars(struct igb_adapter *, u8); 159 static void igb_set_default_mac_filter(struct igb_adapter *); 160 static int igb_uc_sync(struct net_device *, const unsigned char *); 161 static int igb_uc_unsync(struct net_device *, const unsigned char *); 162 static void igb_restore_vf_multicasts(struct igb_adapter *adapter); 163 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac); 164 static int igb_ndo_set_vf_vlan(struct net_device *netdev, 165 int vf, u16 vlan, u8 qos, __be16 vlan_proto); 166 static int igb_ndo_set_vf_bw(struct net_device *, int, int, int); 167 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf, 168 bool setting); 169 static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, 170 bool setting); 171 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf, 172 struct ifla_vf_info *ivi); 173 static void igb_check_vf_rate_limit(struct igb_adapter *); 174 static void igb_nfc_filter_exit(struct igb_adapter *adapter); 175 static void igb_nfc_filter_restore(struct igb_adapter *adapter); 176 177 #ifdef CONFIG_PCI_IOV 178 static int igb_vf_configure(struct igb_adapter *adapter, int vf); 179 static int igb_disable_sriov(struct pci_dev *dev, bool reinit); 180 #endif 181 182 static int igb_suspend(struct device *); 183 static int igb_resume(struct device *); 184 static int igb_runtime_suspend(struct device *dev); 185 static int igb_runtime_resume(struct device *dev); 186 static int igb_runtime_idle(struct device *dev); 187 #ifdef CONFIG_PM 188 static const struct dev_pm_ops igb_pm_ops = { 189 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume) 190 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume, 191 igb_runtime_idle) 192 }; 193 #endif 194 static void igb_shutdown(struct pci_dev *); 195 static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs); 196 #ifdef CONFIG_IGB_DCA 197 static int igb_notify_dca(struct notifier_block *, unsigned long, void *); 198 static struct notifier_block dca_notifier = { 199 .notifier_call = igb_notify_dca, 200 .next = NULL, 201 .priority = 0 202 }; 203 #endif 204 #ifdef CONFIG_PCI_IOV 205 static unsigned int max_vfs; 206 module_param(max_vfs, uint, 0); 207 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function"); 208 #endif /* CONFIG_PCI_IOV */ 209 210 static pci_ers_result_t igb_io_error_detected(struct pci_dev *, 211 pci_channel_state_t); 212 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *); 213 static void igb_io_resume(struct pci_dev *); 214 215 static const struct pci_error_handlers igb_err_handler = { 216 .error_detected = igb_io_error_detected, 217 .slot_reset = igb_io_slot_reset, 218 .resume = igb_io_resume, 219 }; 220 221 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba); 222 223 static struct pci_driver igb_driver = { 224 .name = igb_driver_name, 225 .id_table = igb_pci_tbl, 226 .probe = igb_probe, 227 .remove = igb_remove, 228 #ifdef CONFIG_PM 229 .driver.pm = &igb_pm_ops, 230 #endif 231 .shutdown = igb_shutdown, 232 .sriov_configure = igb_pci_sriov_configure, 233 .err_handler = &igb_err_handler 234 }; 235 236 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 237 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver"); 238 MODULE_LICENSE("GPL v2"); 239 240 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 241 static int debug = -1; 242 module_param(debug, int, 0); 243 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 244 245 struct igb_reg_info { 246 u32 ofs; 247 char *name; 248 }; 249 250 static const struct igb_reg_info igb_reg_info_tbl[] = { 251 252 /* General Registers */ 253 {E1000_CTRL, "CTRL"}, 254 {E1000_STATUS, "STATUS"}, 255 {E1000_CTRL_EXT, "CTRL_EXT"}, 256 257 /* Interrupt Registers */ 258 {E1000_ICR, "ICR"}, 259 260 /* RX Registers */ 261 {E1000_RCTL, "RCTL"}, 262 {E1000_RDLEN(0), "RDLEN"}, 263 {E1000_RDH(0), "RDH"}, 264 {E1000_RDT(0), "RDT"}, 265 {E1000_RXDCTL(0), "RXDCTL"}, 266 {E1000_RDBAL(0), "RDBAL"}, 267 {E1000_RDBAH(0), "RDBAH"}, 268 269 /* TX Registers */ 270 {E1000_TCTL, "TCTL"}, 271 {E1000_TDBAL(0), "TDBAL"}, 272 {E1000_TDBAH(0), "TDBAH"}, 273 {E1000_TDLEN(0), "TDLEN"}, 274 {E1000_TDH(0), "TDH"}, 275 {E1000_TDT(0), "TDT"}, 276 {E1000_TXDCTL(0), "TXDCTL"}, 277 {E1000_TDFH, "TDFH"}, 278 {E1000_TDFT, "TDFT"}, 279 {E1000_TDFHS, "TDFHS"}, 280 {E1000_TDFPC, "TDFPC"}, 281 282 /* List Terminator */ 283 {} 284 }; 285 286 /* igb_regdump - register printout routine */ 287 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo) 288 { 289 int n = 0; 290 char rname[16]; 291 u32 regs[8]; 292 293 switch (reginfo->ofs) { 294 case E1000_RDLEN(0): 295 for (n = 0; n < 4; n++) 296 regs[n] = rd32(E1000_RDLEN(n)); 297 break; 298 case E1000_RDH(0): 299 for (n = 0; n < 4; n++) 300 regs[n] = rd32(E1000_RDH(n)); 301 break; 302 case E1000_RDT(0): 303 for (n = 0; n < 4; n++) 304 regs[n] = rd32(E1000_RDT(n)); 305 break; 306 case E1000_RXDCTL(0): 307 for (n = 0; n < 4; n++) 308 regs[n] = rd32(E1000_RXDCTL(n)); 309 break; 310 case E1000_RDBAL(0): 311 for (n = 0; n < 4; n++) 312 regs[n] = rd32(E1000_RDBAL(n)); 313 break; 314 case E1000_RDBAH(0): 315 for (n = 0; n < 4; n++) 316 regs[n] = rd32(E1000_RDBAH(n)); 317 break; 318 case E1000_TDBAL(0): 319 for (n = 0; n < 4; n++) 320 regs[n] = rd32(E1000_TDBAL(n)); 321 break; 322 case E1000_TDBAH(0): 323 for (n = 0; n < 4; n++) 324 regs[n] = rd32(E1000_TDBAH(n)); 325 break; 326 case E1000_TDLEN(0): 327 for (n = 0; n < 4; n++) 328 regs[n] = rd32(E1000_TDLEN(n)); 329 break; 330 case E1000_TDH(0): 331 for (n = 0; n < 4; n++) 332 regs[n] = rd32(E1000_TDH(n)); 333 break; 334 case E1000_TDT(0): 335 for (n = 0; n < 4; n++) 336 regs[n] = rd32(E1000_TDT(n)); 337 break; 338 case E1000_TXDCTL(0): 339 for (n = 0; n < 4; n++) 340 regs[n] = rd32(E1000_TXDCTL(n)); 341 break; 342 default: 343 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs)); 344 return; 345 } 346 347 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]"); 348 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1], 349 regs[2], regs[3]); 350 } 351 352 /* igb_dump - Print registers, Tx-rings and Rx-rings */ 353 static void igb_dump(struct igb_adapter *adapter) 354 { 355 struct net_device *netdev = adapter->netdev; 356 struct e1000_hw *hw = &adapter->hw; 357 struct igb_reg_info *reginfo; 358 struct igb_ring *tx_ring; 359 union e1000_adv_tx_desc *tx_desc; 360 struct my_u0 { __le64 a; __le64 b; } *u0; 361 struct igb_ring *rx_ring; 362 union e1000_adv_rx_desc *rx_desc; 363 u32 staterr; 364 u16 i, n; 365 366 if (!netif_msg_hw(adapter)) 367 return; 368 369 /* Print netdevice Info */ 370 if (netdev) { 371 dev_info(&adapter->pdev->dev, "Net device Info\n"); 372 pr_info("Device Name state trans_start\n"); 373 pr_info("%-15s %016lX %016lX\n", netdev->name, 374 netdev->state, dev_trans_start(netdev)); 375 } 376 377 /* Print Registers */ 378 dev_info(&adapter->pdev->dev, "Register Dump\n"); 379 pr_info(" Register Name Value\n"); 380 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl; 381 reginfo->name; reginfo++) { 382 igb_regdump(hw, reginfo); 383 } 384 385 /* Print TX Ring Summary */ 386 if (!netdev || !netif_running(netdev)) 387 goto exit; 388 389 dev_info(&adapter->pdev->dev, "TX Rings Summary\n"); 390 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); 391 for (n = 0; n < adapter->num_tx_queues; n++) { 392 struct igb_tx_buffer *buffer_info; 393 tx_ring = adapter->tx_ring[n]; 394 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean]; 395 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n", 396 n, tx_ring->next_to_use, tx_ring->next_to_clean, 397 (u64)dma_unmap_addr(buffer_info, dma), 398 dma_unmap_len(buffer_info, len), 399 buffer_info->next_to_watch, 400 (u64)buffer_info->time_stamp); 401 } 402 403 /* Print TX Rings */ 404 if (!netif_msg_tx_done(adapter)) 405 goto rx_ring_summary; 406 407 dev_info(&adapter->pdev->dev, "TX Rings Dump\n"); 408 409 /* Transmit Descriptor Formats 410 * 411 * Advanced Transmit Descriptor 412 * +--------------------------------------------------------------+ 413 * 0 | Buffer Address [63:0] | 414 * +--------------------------------------------------------------+ 415 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN | 416 * +--------------------------------------------------------------+ 417 * 63 46 45 40 39 38 36 35 32 31 24 15 0 418 */ 419 420 for (n = 0; n < adapter->num_tx_queues; n++) { 421 tx_ring = adapter->tx_ring[n]; 422 pr_info("------------------------------------\n"); 423 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index); 424 pr_info("------------------------------------\n"); 425 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n"); 426 427 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { 428 const char *next_desc; 429 struct igb_tx_buffer *buffer_info; 430 tx_desc = IGB_TX_DESC(tx_ring, i); 431 buffer_info = &tx_ring->tx_buffer_info[i]; 432 u0 = (struct my_u0 *)tx_desc; 433 if (i == tx_ring->next_to_use && 434 i == tx_ring->next_to_clean) 435 next_desc = " NTC/U"; 436 else if (i == tx_ring->next_to_use) 437 next_desc = " NTU"; 438 else if (i == tx_ring->next_to_clean) 439 next_desc = " NTC"; 440 else 441 next_desc = ""; 442 443 pr_info("T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n", 444 i, le64_to_cpu(u0->a), 445 le64_to_cpu(u0->b), 446 (u64)dma_unmap_addr(buffer_info, dma), 447 dma_unmap_len(buffer_info, len), 448 buffer_info->next_to_watch, 449 (u64)buffer_info->time_stamp, 450 buffer_info->skb, next_desc); 451 452 if (netif_msg_pktdata(adapter) && buffer_info->skb) 453 print_hex_dump(KERN_INFO, "", 454 DUMP_PREFIX_ADDRESS, 455 16, 1, buffer_info->skb->data, 456 dma_unmap_len(buffer_info, len), 457 true); 458 } 459 } 460 461 /* Print RX Rings Summary */ 462 rx_ring_summary: 463 dev_info(&adapter->pdev->dev, "RX Rings Summary\n"); 464 pr_info("Queue [NTU] [NTC]\n"); 465 for (n = 0; n < adapter->num_rx_queues; n++) { 466 rx_ring = adapter->rx_ring[n]; 467 pr_info(" %5d %5X %5X\n", 468 n, rx_ring->next_to_use, rx_ring->next_to_clean); 469 } 470 471 /* Print RX Rings */ 472 if (!netif_msg_rx_status(adapter)) 473 goto exit; 474 475 dev_info(&adapter->pdev->dev, "RX Rings Dump\n"); 476 477 /* Advanced Receive Descriptor (Read) Format 478 * 63 1 0 479 * +-----------------------------------------------------+ 480 * 0 | Packet Buffer Address [63:1] |A0/NSE| 481 * +----------------------------------------------+------+ 482 * 8 | Header Buffer Address [63:1] | DD | 483 * +-----------------------------------------------------+ 484 * 485 * 486 * Advanced Receive Descriptor (Write-Back) Format 487 * 488 * 63 48 47 32 31 30 21 20 17 16 4 3 0 489 * +------------------------------------------------------+ 490 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS | 491 * | Checksum Ident | | | | Type | Type | 492 * +------------------------------------------------------+ 493 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 494 * +------------------------------------------------------+ 495 * 63 48 47 32 31 20 19 0 496 */ 497 498 for (n = 0; n < adapter->num_rx_queues; n++) { 499 rx_ring = adapter->rx_ring[n]; 500 pr_info("------------------------------------\n"); 501 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); 502 pr_info("------------------------------------\n"); 503 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n"); 504 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n"); 505 506 for (i = 0; i < rx_ring->count; i++) { 507 const char *next_desc; 508 struct igb_rx_buffer *buffer_info; 509 buffer_info = &rx_ring->rx_buffer_info[i]; 510 rx_desc = IGB_RX_DESC(rx_ring, i); 511 u0 = (struct my_u0 *)rx_desc; 512 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 513 514 if (i == rx_ring->next_to_use) 515 next_desc = " NTU"; 516 else if (i == rx_ring->next_to_clean) 517 next_desc = " NTC"; 518 else 519 next_desc = ""; 520 521 if (staterr & E1000_RXD_STAT_DD) { 522 /* Descriptor Done */ 523 pr_info("%s[0x%03X] %016llX %016llX ---------------- %s\n", 524 "RWB", i, 525 le64_to_cpu(u0->a), 526 le64_to_cpu(u0->b), 527 next_desc); 528 } else { 529 pr_info("%s[0x%03X] %016llX %016llX %016llX %s\n", 530 "R ", i, 531 le64_to_cpu(u0->a), 532 le64_to_cpu(u0->b), 533 (u64)buffer_info->dma, 534 next_desc); 535 536 if (netif_msg_pktdata(adapter) && 537 buffer_info->dma && buffer_info->page) { 538 print_hex_dump(KERN_INFO, "", 539 DUMP_PREFIX_ADDRESS, 540 16, 1, 541 page_address(buffer_info->page) + 542 buffer_info->page_offset, 543 igb_rx_bufsz(rx_ring), true); 544 } 545 } 546 } 547 } 548 549 exit: 550 return; 551 } 552 553 /** 554 * igb_get_i2c_data - Reads the I2C SDA data bit 555 * @data: opaque pointer to adapter struct 556 * 557 * Returns the I2C data bit value 558 **/ 559 static int igb_get_i2c_data(void *data) 560 { 561 struct igb_adapter *adapter = (struct igb_adapter *)data; 562 struct e1000_hw *hw = &adapter->hw; 563 s32 i2cctl = rd32(E1000_I2CPARAMS); 564 565 return !!(i2cctl & E1000_I2C_DATA_IN); 566 } 567 568 /** 569 * igb_set_i2c_data - Sets the I2C data bit 570 * @data: pointer to hardware structure 571 * @state: I2C data value (0 or 1) to set 572 * 573 * Sets the I2C data bit 574 **/ 575 static void igb_set_i2c_data(void *data, int state) 576 { 577 struct igb_adapter *adapter = (struct igb_adapter *)data; 578 struct e1000_hw *hw = &adapter->hw; 579 s32 i2cctl = rd32(E1000_I2CPARAMS); 580 581 if (state) { 582 i2cctl |= E1000_I2C_DATA_OUT | E1000_I2C_DATA_OE_N; 583 } else { 584 i2cctl &= ~E1000_I2C_DATA_OE_N; 585 i2cctl &= ~E1000_I2C_DATA_OUT; 586 } 587 588 wr32(E1000_I2CPARAMS, i2cctl); 589 wrfl(); 590 } 591 592 /** 593 * igb_set_i2c_clk - Sets the I2C SCL clock 594 * @data: pointer to hardware structure 595 * @state: state to set clock 596 * 597 * Sets the I2C clock line to state 598 **/ 599 static void igb_set_i2c_clk(void *data, int state) 600 { 601 struct igb_adapter *adapter = (struct igb_adapter *)data; 602 struct e1000_hw *hw = &adapter->hw; 603 s32 i2cctl = rd32(E1000_I2CPARAMS); 604 605 if (state) { 606 i2cctl |= E1000_I2C_CLK_OUT | E1000_I2C_CLK_OE_N; 607 } else { 608 i2cctl &= ~E1000_I2C_CLK_OUT; 609 i2cctl &= ~E1000_I2C_CLK_OE_N; 610 } 611 wr32(E1000_I2CPARAMS, i2cctl); 612 wrfl(); 613 } 614 615 /** 616 * igb_get_i2c_clk - Gets the I2C SCL clock state 617 * @data: pointer to hardware structure 618 * 619 * Gets the I2C clock state 620 **/ 621 static int igb_get_i2c_clk(void *data) 622 { 623 struct igb_adapter *adapter = (struct igb_adapter *)data; 624 struct e1000_hw *hw = &adapter->hw; 625 s32 i2cctl = rd32(E1000_I2CPARAMS); 626 627 return !!(i2cctl & E1000_I2C_CLK_IN); 628 } 629 630 static const struct i2c_algo_bit_data igb_i2c_algo = { 631 .setsda = igb_set_i2c_data, 632 .setscl = igb_set_i2c_clk, 633 .getsda = igb_get_i2c_data, 634 .getscl = igb_get_i2c_clk, 635 .udelay = 5, 636 .timeout = 20, 637 }; 638 639 /** 640 * igb_get_hw_dev - return device 641 * @hw: pointer to hardware structure 642 * 643 * used by hardware layer to print debugging information 644 **/ 645 struct net_device *igb_get_hw_dev(struct e1000_hw *hw) 646 { 647 struct igb_adapter *adapter = hw->back; 648 return adapter->netdev; 649 } 650 651 /** 652 * igb_init_module - Driver Registration Routine 653 * 654 * igb_init_module is the first routine called when the driver is 655 * loaded. All it does is register with the PCI subsystem. 656 **/ 657 static int __init igb_init_module(void) 658 { 659 int ret; 660 661 pr_info("%s\n", igb_driver_string); 662 pr_info("%s\n", igb_copyright); 663 664 #ifdef CONFIG_IGB_DCA 665 dca_register_notify(&dca_notifier); 666 #endif 667 ret = pci_register_driver(&igb_driver); 668 #ifdef CONFIG_IGB_DCA 669 if (ret) 670 dca_unregister_notify(&dca_notifier); 671 #endif 672 return ret; 673 } 674 675 module_init(igb_init_module); 676 677 /** 678 * igb_exit_module - Driver Exit Cleanup Routine 679 * 680 * igb_exit_module is called just before the driver is removed 681 * from memory. 682 **/ 683 static void __exit igb_exit_module(void) 684 { 685 #ifdef CONFIG_IGB_DCA 686 dca_unregister_notify(&dca_notifier); 687 #endif 688 pci_unregister_driver(&igb_driver); 689 } 690 691 module_exit(igb_exit_module); 692 693 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1)) 694 /** 695 * igb_cache_ring_register - Descriptor ring to register mapping 696 * @adapter: board private structure to initialize 697 * 698 * Once we know the feature-set enabled for the device, we'll cache 699 * the register offset the descriptor ring is assigned to. 700 **/ 701 static void igb_cache_ring_register(struct igb_adapter *adapter) 702 { 703 int i = 0, j = 0; 704 u32 rbase_offset = adapter->vfs_allocated_count; 705 706 switch (adapter->hw.mac.type) { 707 case e1000_82576: 708 /* The queues are allocated for virtualization such that VF 0 709 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc. 710 * In order to avoid collision we start at the first free queue 711 * and continue consuming queues in the same sequence 712 */ 713 if (adapter->vfs_allocated_count) { 714 for (; i < adapter->rss_queues; i++) 715 adapter->rx_ring[i]->reg_idx = rbase_offset + 716 Q_IDX_82576(i); 717 } 718 fallthrough; 719 case e1000_82575: 720 case e1000_82580: 721 case e1000_i350: 722 case e1000_i354: 723 case e1000_i210: 724 case e1000_i211: 725 default: 726 for (; i < adapter->num_rx_queues; i++) 727 adapter->rx_ring[i]->reg_idx = rbase_offset + i; 728 for (; j < adapter->num_tx_queues; j++) 729 adapter->tx_ring[j]->reg_idx = rbase_offset + j; 730 break; 731 } 732 } 733 734 u32 igb_rd32(struct e1000_hw *hw, u32 reg) 735 { 736 struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw); 737 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 738 u32 value = 0; 739 740 if (E1000_REMOVED(hw_addr)) 741 return ~value; 742 743 value = readl(&hw_addr[reg]); 744 745 /* reads should not return all F's */ 746 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 747 struct net_device *netdev = igb->netdev; 748 hw->hw_addr = NULL; 749 netdev_err(netdev, "PCIe link lost\n"); 750 WARN(pci_device_is_present(igb->pdev), 751 "igb: Failed to read reg 0x%x!\n", reg); 752 } 753 754 return value; 755 } 756 757 /** 758 * igb_write_ivar - configure ivar for given MSI-X vector 759 * @hw: pointer to the HW structure 760 * @msix_vector: vector number we are allocating to a given ring 761 * @index: row index of IVAR register to write within IVAR table 762 * @offset: column offset of in IVAR, should be multiple of 8 763 * 764 * This function is intended to handle the writing of the IVAR register 765 * for adapters 82576 and newer. The IVAR table consists of 2 columns, 766 * each containing an cause allocation for an Rx and Tx ring, and a 767 * variable number of rows depending on the number of queues supported. 768 **/ 769 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector, 770 int index, int offset) 771 { 772 u32 ivar = array_rd32(E1000_IVAR0, index); 773 774 /* clear any bits that are currently set */ 775 ivar &= ~((u32)0xFF << offset); 776 777 /* write vector and valid bit */ 778 ivar |= (msix_vector | E1000_IVAR_VALID) << offset; 779 780 array_wr32(E1000_IVAR0, index, ivar); 781 } 782 783 #define IGB_N0_QUEUE -1 784 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector) 785 { 786 struct igb_adapter *adapter = q_vector->adapter; 787 struct e1000_hw *hw = &adapter->hw; 788 int rx_queue = IGB_N0_QUEUE; 789 int tx_queue = IGB_N0_QUEUE; 790 u32 msixbm = 0; 791 792 if (q_vector->rx.ring) 793 rx_queue = q_vector->rx.ring->reg_idx; 794 if (q_vector->tx.ring) 795 tx_queue = q_vector->tx.ring->reg_idx; 796 797 switch (hw->mac.type) { 798 case e1000_82575: 799 /* The 82575 assigns vectors using a bitmask, which matches the 800 * bitmask for the EICR/EIMS/EIMC registers. To assign one 801 * or more queues to a vector, we write the appropriate bits 802 * into the MSIXBM register for that vector. 803 */ 804 if (rx_queue > IGB_N0_QUEUE) 805 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue; 806 if (tx_queue > IGB_N0_QUEUE) 807 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue; 808 if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0) 809 msixbm |= E1000_EIMS_OTHER; 810 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm); 811 q_vector->eims_value = msixbm; 812 break; 813 case e1000_82576: 814 /* 82576 uses a table that essentially consists of 2 columns 815 * with 8 rows. The ordering is column-major so we use the 816 * lower 3 bits as the row index, and the 4th bit as the 817 * column offset. 818 */ 819 if (rx_queue > IGB_N0_QUEUE) 820 igb_write_ivar(hw, msix_vector, 821 rx_queue & 0x7, 822 (rx_queue & 0x8) << 1); 823 if (tx_queue > IGB_N0_QUEUE) 824 igb_write_ivar(hw, msix_vector, 825 tx_queue & 0x7, 826 ((tx_queue & 0x8) << 1) + 8); 827 q_vector->eims_value = BIT(msix_vector); 828 break; 829 case e1000_82580: 830 case e1000_i350: 831 case e1000_i354: 832 case e1000_i210: 833 case e1000_i211: 834 /* On 82580 and newer adapters the scheme is similar to 82576 835 * however instead of ordering column-major we have things 836 * ordered row-major. So we traverse the table by using 837 * bit 0 as the column offset, and the remaining bits as the 838 * row index. 839 */ 840 if (rx_queue > IGB_N0_QUEUE) 841 igb_write_ivar(hw, msix_vector, 842 rx_queue >> 1, 843 (rx_queue & 0x1) << 4); 844 if (tx_queue > IGB_N0_QUEUE) 845 igb_write_ivar(hw, msix_vector, 846 tx_queue >> 1, 847 ((tx_queue & 0x1) << 4) + 8); 848 q_vector->eims_value = BIT(msix_vector); 849 break; 850 default: 851 BUG(); 852 break; 853 } 854 855 /* add q_vector eims value to global eims_enable_mask */ 856 adapter->eims_enable_mask |= q_vector->eims_value; 857 858 /* configure q_vector to set itr on first interrupt */ 859 q_vector->set_itr = 1; 860 } 861 862 /** 863 * igb_configure_msix - Configure MSI-X hardware 864 * @adapter: board private structure to initialize 865 * 866 * igb_configure_msix sets up the hardware to properly 867 * generate MSI-X interrupts. 868 **/ 869 static void igb_configure_msix(struct igb_adapter *adapter) 870 { 871 u32 tmp; 872 int i, vector = 0; 873 struct e1000_hw *hw = &adapter->hw; 874 875 adapter->eims_enable_mask = 0; 876 877 /* set vector for other causes, i.e. link changes */ 878 switch (hw->mac.type) { 879 case e1000_82575: 880 tmp = rd32(E1000_CTRL_EXT); 881 /* enable MSI-X PBA support*/ 882 tmp |= E1000_CTRL_EXT_PBA_CLR; 883 884 /* Auto-Mask interrupts upon ICR read. */ 885 tmp |= E1000_CTRL_EXT_EIAME; 886 tmp |= E1000_CTRL_EXT_IRCA; 887 888 wr32(E1000_CTRL_EXT, tmp); 889 890 /* enable msix_other interrupt */ 891 array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER); 892 adapter->eims_other = E1000_EIMS_OTHER; 893 894 break; 895 896 case e1000_82576: 897 case e1000_82580: 898 case e1000_i350: 899 case e1000_i354: 900 case e1000_i210: 901 case e1000_i211: 902 /* Turn on MSI-X capability first, or our settings 903 * won't stick. And it will take days to debug. 904 */ 905 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE | 906 E1000_GPIE_PBA | E1000_GPIE_EIAME | 907 E1000_GPIE_NSICR); 908 909 /* enable msix_other interrupt */ 910 adapter->eims_other = BIT(vector); 911 tmp = (vector++ | E1000_IVAR_VALID) << 8; 912 913 wr32(E1000_IVAR_MISC, tmp); 914 break; 915 default: 916 /* do nothing, since nothing else supports MSI-X */ 917 break; 918 } /* switch (hw->mac.type) */ 919 920 adapter->eims_enable_mask |= adapter->eims_other; 921 922 for (i = 0; i < adapter->num_q_vectors; i++) 923 igb_assign_vector(adapter->q_vector[i], vector++); 924 925 wrfl(); 926 } 927 928 /** 929 * igb_request_msix - Initialize MSI-X interrupts 930 * @adapter: board private structure to initialize 931 * 932 * igb_request_msix allocates MSI-X vectors and requests interrupts from the 933 * kernel. 934 **/ 935 static int igb_request_msix(struct igb_adapter *adapter) 936 { 937 unsigned int num_q_vectors = adapter->num_q_vectors; 938 struct net_device *netdev = adapter->netdev; 939 int i, err = 0, vector = 0, free_vector = 0; 940 941 err = request_irq(adapter->msix_entries[vector].vector, 942 igb_msix_other, 0, netdev->name, adapter); 943 if (err) 944 goto err_out; 945 946 if (num_q_vectors > MAX_Q_VECTORS) { 947 num_q_vectors = MAX_Q_VECTORS; 948 dev_warn(&adapter->pdev->dev, 949 "The number of queue vectors (%d) is higher than max allowed (%d)\n", 950 adapter->num_q_vectors, MAX_Q_VECTORS); 951 } 952 for (i = 0; i < num_q_vectors; i++) { 953 struct igb_q_vector *q_vector = adapter->q_vector[i]; 954 955 vector++; 956 957 q_vector->itr_register = adapter->io_addr + E1000_EITR(vector); 958 959 if (q_vector->rx.ring && q_vector->tx.ring) 960 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 961 q_vector->rx.ring->queue_index); 962 else if (q_vector->tx.ring) 963 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 964 q_vector->tx.ring->queue_index); 965 else if (q_vector->rx.ring) 966 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 967 q_vector->rx.ring->queue_index); 968 else 969 sprintf(q_vector->name, "%s-unused", netdev->name); 970 971 err = request_irq(adapter->msix_entries[vector].vector, 972 igb_msix_ring, 0, q_vector->name, 973 q_vector); 974 if (err) 975 goto err_free; 976 } 977 978 igb_configure_msix(adapter); 979 return 0; 980 981 err_free: 982 /* free already assigned IRQs */ 983 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 984 985 vector--; 986 for (i = 0; i < vector; i++) { 987 free_irq(adapter->msix_entries[free_vector++].vector, 988 adapter->q_vector[i]); 989 } 990 err_out: 991 return err; 992 } 993 994 /** 995 * igb_free_q_vector - Free memory allocated for specific interrupt vector 996 * @adapter: board private structure to initialize 997 * @v_idx: Index of vector to be freed 998 * 999 * This function frees the memory allocated to the q_vector. 1000 **/ 1001 static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx) 1002 { 1003 struct igb_q_vector *q_vector = adapter->q_vector[v_idx]; 1004 1005 adapter->q_vector[v_idx] = NULL; 1006 1007 /* igb_get_stats64() might access the rings on this vector, 1008 * we must wait a grace period before freeing it. 1009 */ 1010 if (q_vector) 1011 kfree_rcu(q_vector, rcu); 1012 } 1013 1014 /** 1015 * igb_reset_q_vector - Reset config for interrupt vector 1016 * @adapter: board private structure to initialize 1017 * @v_idx: Index of vector to be reset 1018 * 1019 * If NAPI is enabled it will delete any references to the 1020 * NAPI struct. This is preparation for igb_free_q_vector. 1021 **/ 1022 static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx) 1023 { 1024 struct igb_q_vector *q_vector = adapter->q_vector[v_idx]; 1025 1026 /* Coming from igb_set_interrupt_capability, the vectors are not yet 1027 * allocated. So, q_vector is NULL so we should stop here. 1028 */ 1029 if (!q_vector) 1030 return; 1031 1032 if (q_vector->tx.ring) 1033 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 1034 1035 if (q_vector->rx.ring) 1036 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 1037 1038 netif_napi_del(&q_vector->napi); 1039 1040 } 1041 1042 static void igb_reset_interrupt_capability(struct igb_adapter *adapter) 1043 { 1044 int v_idx = adapter->num_q_vectors; 1045 1046 if (adapter->flags & IGB_FLAG_HAS_MSIX) 1047 pci_disable_msix(adapter->pdev); 1048 else if (adapter->flags & IGB_FLAG_HAS_MSI) 1049 pci_disable_msi(adapter->pdev); 1050 1051 while (v_idx--) 1052 igb_reset_q_vector(adapter, v_idx); 1053 } 1054 1055 /** 1056 * igb_free_q_vectors - Free memory allocated for interrupt vectors 1057 * @adapter: board private structure to initialize 1058 * 1059 * This function frees the memory allocated to the q_vectors. In addition if 1060 * NAPI is enabled it will delete any references to the NAPI struct prior 1061 * to freeing the q_vector. 1062 **/ 1063 static void igb_free_q_vectors(struct igb_adapter *adapter) 1064 { 1065 int v_idx = adapter->num_q_vectors; 1066 1067 adapter->num_tx_queues = 0; 1068 adapter->num_rx_queues = 0; 1069 adapter->num_q_vectors = 0; 1070 1071 while (v_idx--) { 1072 igb_reset_q_vector(adapter, v_idx); 1073 igb_free_q_vector(adapter, v_idx); 1074 } 1075 } 1076 1077 /** 1078 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts 1079 * @adapter: board private structure to initialize 1080 * 1081 * This function resets the device so that it has 0 Rx queues, Tx queues, and 1082 * MSI-X interrupts allocated. 1083 */ 1084 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter) 1085 { 1086 igb_free_q_vectors(adapter); 1087 igb_reset_interrupt_capability(adapter); 1088 } 1089 1090 /** 1091 * igb_set_interrupt_capability - set MSI or MSI-X if supported 1092 * @adapter: board private structure to initialize 1093 * @msix: boolean value of MSIX capability 1094 * 1095 * Attempt to configure interrupts using the best available 1096 * capabilities of the hardware and kernel. 1097 **/ 1098 static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix) 1099 { 1100 int err; 1101 int numvecs, i; 1102 1103 if (!msix) 1104 goto msi_only; 1105 adapter->flags |= IGB_FLAG_HAS_MSIX; 1106 1107 /* Number of supported queues. */ 1108 adapter->num_rx_queues = adapter->rss_queues; 1109 if (adapter->vfs_allocated_count) 1110 adapter->num_tx_queues = 1; 1111 else 1112 adapter->num_tx_queues = adapter->rss_queues; 1113 1114 /* start with one vector for every Rx queue */ 1115 numvecs = adapter->num_rx_queues; 1116 1117 /* if Tx handler is separate add 1 for every Tx queue */ 1118 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS)) 1119 numvecs += adapter->num_tx_queues; 1120 1121 /* store the number of vectors reserved for queues */ 1122 adapter->num_q_vectors = numvecs; 1123 1124 /* add 1 vector for link status interrupts */ 1125 numvecs++; 1126 for (i = 0; i < numvecs; i++) 1127 adapter->msix_entries[i].entry = i; 1128 1129 err = pci_enable_msix_range(adapter->pdev, 1130 adapter->msix_entries, 1131 numvecs, 1132 numvecs); 1133 if (err > 0) 1134 return; 1135 1136 igb_reset_interrupt_capability(adapter); 1137 1138 /* If we can't do MSI-X, try MSI */ 1139 msi_only: 1140 adapter->flags &= ~IGB_FLAG_HAS_MSIX; 1141 #ifdef CONFIG_PCI_IOV 1142 /* disable SR-IOV for non MSI-X configurations */ 1143 if (adapter->vf_data) { 1144 struct e1000_hw *hw = &adapter->hw; 1145 /* disable iov and allow time for transactions to clear */ 1146 pci_disable_sriov(adapter->pdev); 1147 msleep(500); 1148 1149 kfree(adapter->vf_mac_list); 1150 adapter->vf_mac_list = NULL; 1151 kfree(adapter->vf_data); 1152 adapter->vf_data = NULL; 1153 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ); 1154 wrfl(); 1155 msleep(100); 1156 dev_info(&adapter->pdev->dev, "IOV Disabled\n"); 1157 } 1158 #endif 1159 adapter->vfs_allocated_count = 0; 1160 adapter->rss_queues = 1; 1161 adapter->flags |= IGB_FLAG_QUEUE_PAIRS; 1162 adapter->num_rx_queues = 1; 1163 adapter->num_tx_queues = 1; 1164 adapter->num_q_vectors = 1; 1165 if (!pci_enable_msi(adapter->pdev)) 1166 adapter->flags |= IGB_FLAG_HAS_MSI; 1167 } 1168 1169 static void igb_add_ring(struct igb_ring *ring, 1170 struct igb_ring_container *head) 1171 { 1172 head->ring = ring; 1173 head->count++; 1174 } 1175 1176 /** 1177 * igb_alloc_q_vector - Allocate memory for a single interrupt vector 1178 * @adapter: board private structure to initialize 1179 * @v_count: q_vectors allocated on adapter, used for ring interleaving 1180 * @v_idx: index of vector in adapter struct 1181 * @txr_count: total number of Tx rings to allocate 1182 * @txr_idx: index of first Tx ring to allocate 1183 * @rxr_count: total number of Rx rings to allocate 1184 * @rxr_idx: index of first Rx ring to allocate 1185 * 1186 * We allocate one q_vector. If allocation fails we return -ENOMEM. 1187 **/ 1188 static int igb_alloc_q_vector(struct igb_adapter *adapter, 1189 int v_count, int v_idx, 1190 int txr_count, int txr_idx, 1191 int rxr_count, int rxr_idx) 1192 { 1193 struct igb_q_vector *q_vector; 1194 struct igb_ring *ring; 1195 int ring_count; 1196 size_t size; 1197 1198 /* igb only supports 1 Tx and/or 1 Rx queue per vector */ 1199 if (txr_count > 1 || rxr_count > 1) 1200 return -ENOMEM; 1201 1202 ring_count = txr_count + rxr_count; 1203 size = kmalloc_size_roundup(struct_size(q_vector, ring, ring_count)); 1204 1205 /* allocate q_vector and rings */ 1206 q_vector = adapter->q_vector[v_idx]; 1207 if (!q_vector) { 1208 q_vector = kzalloc(size, GFP_KERNEL); 1209 } else if (size > ksize(q_vector)) { 1210 struct igb_q_vector *new_q_vector; 1211 1212 new_q_vector = kzalloc(size, GFP_KERNEL); 1213 if (new_q_vector) 1214 kfree_rcu(q_vector, rcu); 1215 q_vector = new_q_vector; 1216 } else { 1217 memset(q_vector, 0, size); 1218 } 1219 if (!q_vector) 1220 return -ENOMEM; 1221 1222 /* initialize NAPI */ 1223 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll); 1224 1225 /* tie q_vector and adapter together */ 1226 adapter->q_vector[v_idx] = q_vector; 1227 q_vector->adapter = adapter; 1228 1229 /* initialize work limits */ 1230 q_vector->tx.work_limit = adapter->tx_work_limit; 1231 1232 /* initialize ITR configuration */ 1233 q_vector->itr_register = adapter->io_addr + E1000_EITR(0); 1234 q_vector->itr_val = IGB_START_ITR; 1235 1236 /* initialize pointer to rings */ 1237 ring = q_vector->ring; 1238 1239 /* intialize ITR */ 1240 if (rxr_count) { 1241 /* rx or rx/tx vector */ 1242 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 1243 q_vector->itr_val = adapter->rx_itr_setting; 1244 } else { 1245 /* tx only vector */ 1246 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 1247 q_vector->itr_val = adapter->tx_itr_setting; 1248 } 1249 1250 if (txr_count) { 1251 /* assign generic ring traits */ 1252 ring->dev = &adapter->pdev->dev; 1253 ring->netdev = adapter->netdev; 1254 1255 /* configure backlink on ring */ 1256 ring->q_vector = q_vector; 1257 1258 /* update q_vector Tx values */ 1259 igb_add_ring(ring, &q_vector->tx); 1260 1261 /* For 82575, context index must be unique per ring. */ 1262 if (adapter->hw.mac.type == e1000_82575) 1263 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags); 1264 1265 /* apply Tx specific ring traits */ 1266 ring->count = adapter->tx_ring_count; 1267 ring->queue_index = txr_idx; 1268 1269 ring->cbs_enable = false; 1270 ring->idleslope = 0; 1271 ring->sendslope = 0; 1272 ring->hicredit = 0; 1273 ring->locredit = 0; 1274 1275 u64_stats_init(&ring->tx_syncp); 1276 u64_stats_init(&ring->tx_syncp2); 1277 1278 /* assign ring to adapter */ 1279 adapter->tx_ring[txr_idx] = ring; 1280 1281 /* push pointer to next ring */ 1282 ring++; 1283 } 1284 1285 if (rxr_count) { 1286 /* assign generic ring traits */ 1287 ring->dev = &adapter->pdev->dev; 1288 ring->netdev = adapter->netdev; 1289 1290 /* configure backlink on ring */ 1291 ring->q_vector = q_vector; 1292 1293 /* update q_vector Rx values */ 1294 igb_add_ring(ring, &q_vector->rx); 1295 1296 /* set flag indicating ring supports SCTP checksum offload */ 1297 if (adapter->hw.mac.type >= e1000_82576) 1298 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags); 1299 1300 /* On i350, i354, i210, and i211, loopback VLAN packets 1301 * have the tag byte-swapped. 1302 */ 1303 if (adapter->hw.mac.type >= e1000_i350) 1304 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags); 1305 1306 /* apply Rx specific ring traits */ 1307 ring->count = adapter->rx_ring_count; 1308 ring->queue_index = rxr_idx; 1309 1310 u64_stats_init(&ring->rx_syncp); 1311 1312 /* assign ring to adapter */ 1313 adapter->rx_ring[rxr_idx] = ring; 1314 } 1315 1316 return 0; 1317 } 1318 1319 1320 /** 1321 * igb_alloc_q_vectors - Allocate memory for interrupt vectors 1322 * @adapter: board private structure to initialize 1323 * 1324 * We allocate one q_vector per queue interrupt. If allocation fails we 1325 * return -ENOMEM. 1326 **/ 1327 static int igb_alloc_q_vectors(struct igb_adapter *adapter) 1328 { 1329 int q_vectors = adapter->num_q_vectors; 1330 int rxr_remaining = adapter->num_rx_queues; 1331 int txr_remaining = adapter->num_tx_queues; 1332 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 1333 int err; 1334 1335 if (q_vectors >= (rxr_remaining + txr_remaining)) { 1336 for (; rxr_remaining; v_idx++) { 1337 err = igb_alloc_q_vector(adapter, q_vectors, v_idx, 1338 0, 0, 1, rxr_idx); 1339 1340 if (err) 1341 goto err_out; 1342 1343 /* update counts and index */ 1344 rxr_remaining--; 1345 rxr_idx++; 1346 } 1347 } 1348 1349 for (; v_idx < q_vectors; v_idx++) { 1350 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 1351 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 1352 1353 err = igb_alloc_q_vector(adapter, q_vectors, v_idx, 1354 tqpv, txr_idx, rqpv, rxr_idx); 1355 1356 if (err) 1357 goto err_out; 1358 1359 /* update counts and index */ 1360 rxr_remaining -= rqpv; 1361 txr_remaining -= tqpv; 1362 rxr_idx++; 1363 txr_idx++; 1364 } 1365 1366 return 0; 1367 1368 err_out: 1369 adapter->num_tx_queues = 0; 1370 adapter->num_rx_queues = 0; 1371 adapter->num_q_vectors = 0; 1372 1373 while (v_idx--) 1374 igb_free_q_vector(adapter, v_idx); 1375 1376 return -ENOMEM; 1377 } 1378 1379 /** 1380 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 1381 * @adapter: board private structure to initialize 1382 * @msix: boolean value of MSIX capability 1383 * 1384 * This function initializes the interrupts and allocates all of the queues. 1385 **/ 1386 static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix) 1387 { 1388 struct pci_dev *pdev = adapter->pdev; 1389 int err; 1390 1391 igb_set_interrupt_capability(adapter, msix); 1392 1393 err = igb_alloc_q_vectors(adapter); 1394 if (err) { 1395 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); 1396 goto err_alloc_q_vectors; 1397 } 1398 1399 igb_cache_ring_register(adapter); 1400 1401 return 0; 1402 1403 err_alloc_q_vectors: 1404 igb_reset_interrupt_capability(adapter); 1405 return err; 1406 } 1407 1408 /** 1409 * igb_request_irq - initialize interrupts 1410 * @adapter: board private structure to initialize 1411 * 1412 * Attempts to configure interrupts using the best available 1413 * capabilities of the hardware and kernel. 1414 **/ 1415 static int igb_request_irq(struct igb_adapter *adapter) 1416 { 1417 struct net_device *netdev = adapter->netdev; 1418 struct pci_dev *pdev = adapter->pdev; 1419 int err = 0; 1420 1421 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1422 err = igb_request_msix(adapter); 1423 if (!err) 1424 goto request_done; 1425 /* fall back to MSI */ 1426 igb_free_all_tx_resources(adapter); 1427 igb_free_all_rx_resources(adapter); 1428 1429 igb_clear_interrupt_scheme(adapter); 1430 err = igb_init_interrupt_scheme(adapter, false); 1431 if (err) 1432 goto request_done; 1433 1434 igb_setup_all_tx_resources(adapter); 1435 igb_setup_all_rx_resources(adapter); 1436 igb_configure(adapter); 1437 } 1438 1439 igb_assign_vector(adapter->q_vector[0], 0); 1440 1441 if (adapter->flags & IGB_FLAG_HAS_MSI) { 1442 err = request_irq(pdev->irq, igb_intr_msi, 0, 1443 netdev->name, adapter); 1444 if (!err) 1445 goto request_done; 1446 1447 /* fall back to legacy interrupts */ 1448 igb_reset_interrupt_capability(adapter); 1449 adapter->flags &= ~IGB_FLAG_HAS_MSI; 1450 } 1451 1452 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED, 1453 netdev->name, adapter); 1454 1455 if (err) 1456 dev_err(&pdev->dev, "Error %d getting interrupt\n", 1457 err); 1458 1459 request_done: 1460 return err; 1461 } 1462 1463 static void igb_free_irq(struct igb_adapter *adapter) 1464 { 1465 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1466 int vector = 0, i; 1467 1468 free_irq(adapter->msix_entries[vector++].vector, adapter); 1469 1470 for (i = 0; i < adapter->num_q_vectors; i++) 1471 free_irq(adapter->msix_entries[vector++].vector, 1472 adapter->q_vector[i]); 1473 } else { 1474 free_irq(adapter->pdev->irq, adapter); 1475 } 1476 } 1477 1478 /** 1479 * igb_irq_disable - Mask off interrupt generation on the NIC 1480 * @adapter: board private structure 1481 **/ 1482 static void igb_irq_disable(struct igb_adapter *adapter) 1483 { 1484 struct e1000_hw *hw = &adapter->hw; 1485 1486 /* we need to be careful when disabling interrupts. The VFs are also 1487 * mapped into these registers and so clearing the bits can cause 1488 * issues on the VF drivers so we only need to clear what we set 1489 */ 1490 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1491 u32 regval = rd32(E1000_EIAM); 1492 1493 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask); 1494 wr32(E1000_EIMC, adapter->eims_enable_mask); 1495 regval = rd32(E1000_EIAC); 1496 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask); 1497 } 1498 1499 wr32(E1000_IAM, 0); 1500 wr32(E1000_IMC, ~0); 1501 wrfl(); 1502 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1503 int i; 1504 1505 for (i = 0; i < adapter->num_q_vectors; i++) 1506 synchronize_irq(adapter->msix_entries[i].vector); 1507 } else { 1508 synchronize_irq(adapter->pdev->irq); 1509 } 1510 } 1511 1512 /** 1513 * igb_irq_enable - Enable default interrupt generation settings 1514 * @adapter: board private structure 1515 **/ 1516 static void igb_irq_enable(struct igb_adapter *adapter) 1517 { 1518 struct e1000_hw *hw = &adapter->hw; 1519 1520 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1521 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA; 1522 u32 regval = rd32(E1000_EIAC); 1523 1524 wr32(E1000_EIAC, regval | adapter->eims_enable_mask); 1525 regval = rd32(E1000_EIAM); 1526 wr32(E1000_EIAM, regval | adapter->eims_enable_mask); 1527 wr32(E1000_EIMS, adapter->eims_enable_mask); 1528 if (adapter->vfs_allocated_count) { 1529 wr32(E1000_MBVFIMR, 0xFF); 1530 ims |= E1000_IMS_VMMB; 1531 } 1532 wr32(E1000_IMS, ims); 1533 } else { 1534 wr32(E1000_IMS, IMS_ENABLE_MASK | 1535 E1000_IMS_DRSTA); 1536 wr32(E1000_IAM, IMS_ENABLE_MASK | 1537 E1000_IMS_DRSTA); 1538 } 1539 } 1540 1541 static void igb_update_mng_vlan(struct igb_adapter *adapter) 1542 { 1543 struct e1000_hw *hw = &adapter->hw; 1544 u16 pf_id = adapter->vfs_allocated_count; 1545 u16 vid = adapter->hw.mng_cookie.vlan_id; 1546 u16 old_vid = adapter->mng_vlan_id; 1547 1548 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) { 1549 /* add VID to filter table */ 1550 igb_vfta_set(hw, vid, pf_id, true, true); 1551 adapter->mng_vlan_id = vid; 1552 } else { 1553 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE; 1554 } 1555 1556 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) && 1557 (vid != old_vid) && 1558 !test_bit(old_vid, adapter->active_vlans)) { 1559 /* remove VID from filter table */ 1560 igb_vfta_set(hw, vid, pf_id, false, true); 1561 } 1562 } 1563 1564 /** 1565 * igb_release_hw_control - release control of the h/w to f/w 1566 * @adapter: address of board private structure 1567 * 1568 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 1569 * For ASF and Pass Through versions of f/w this means that the 1570 * driver is no longer loaded. 1571 **/ 1572 static void igb_release_hw_control(struct igb_adapter *adapter) 1573 { 1574 struct e1000_hw *hw = &adapter->hw; 1575 u32 ctrl_ext; 1576 1577 /* Let firmware take over control of h/w */ 1578 ctrl_ext = rd32(E1000_CTRL_EXT); 1579 wr32(E1000_CTRL_EXT, 1580 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 1581 } 1582 1583 /** 1584 * igb_get_hw_control - get control of the h/w from f/w 1585 * @adapter: address of board private structure 1586 * 1587 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 1588 * For ASF and Pass Through versions of f/w this means that 1589 * the driver is loaded. 1590 **/ 1591 static void igb_get_hw_control(struct igb_adapter *adapter) 1592 { 1593 struct e1000_hw *hw = &adapter->hw; 1594 u32 ctrl_ext; 1595 1596 /* Let firmware know the driver has taken over */ 1597 ctrl_ext = rd32(E1000_CTRL_EXT); 1598 wr32(E1000_CTRL_EXT, 1599 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 1600 } 1601 1602 static void enable_fqtss(struct igb_adapter *adapter, bool enable) 1603 { 1604 struct net_device *netdev = adapter->netdev; 1605 struct e1000_hw *hw = &adapter->hw; 1606 1607 WARN_ON(hw->mac.type != e1000_i210); 1608 1609 if (enable) 1610 adapter->flags |= IGB_FLAG_FQTSS; 1611 else 1612 adapter->flags &= ~IGB_FLAG_FQTSS; 1613 1614 if (netif_running(netdev)) 1615 schedule_work(&adapter->reset_task); 1616 } 1617 1618 static bool is_fqtss_enabled(struct igb_adapter *adapter) 1619 { 1620 return (adapter->flags & IGB_FLAG_FQTSS) ? true : false; 1621 } 1622 1623 static void set_tx_desc_fetch_prio(struct e1000_hw *hw, int queue, 1624 enum tx_queue_prio prio) 1625 { 1626 u32 val; 1627 1628 WARN_ON(hw->mac.type != e1000_i210); 1629 WARN_ON(queue < 0 || queue > 4); 1630 1631 val = rd32(E1000_I210_TXDCTL(queue)); 1632 1633 if (prio == TX_QUEUE_PRIO_HIGH) 1634 val |= E1000_TXDCTL_PRIORITY; 1635 else 1636 val &= ~E1000_TXDCTL_PRIORITY; 1637 1638 wr32(E1000_I210_TXDCTL(queue), val); 1639 } 1640 1641 static void set_queue_mode(struct e1000_hw *hw, int queue, enum queue_mode mode) 1642 { 1643 u32 val; 1644 1645 WARN_ON(hw->mac.type != e1000_i210); 1646 WARN_ON(queue < 0 || queue > 1); 1647 1648 val = rd32(E1000_I210_TQAVCC(queue)); 1649 1650 if (mode == QUEUE_MODE_STREAM_RESERVATION) 1651 val |= E1000_TQAVCC_QUEUEMODE; 1652 else 1653 val &= ~E1000_TQAVCC_QUEUEMODE; 1654 1655 wr32(E1000_I210_TQAVCC(queue), val); 1656 } 1657 1658 static bool is_any_cbs_enabled(struct igb_adapter *adapter) 1659 { 1660 int i; 1661 1662 for (i = 0; i < adapter->num_tx_queues; i++) { 1663 if (adapter->tx_ring[i]->cbs_enable) 1664 return true; 1665 } 1666 1667 return false; 1668 } 1669 1670 static bool is_any_txtime_enabled(struct igb_adapter *adapter) 1671 { 1672 int i; 1673 1674 for (i = 0; i < adapter->num_tx_queues; i++) { 1675 if (adapter->tx_ring[i]->launchtime_enable) 1676 return true; 1677 } 1678 1679 return false; 1680 } 1681 1682 /** 1683 * igb_config_tx_modes - Configure "Qav Tx mode" features on igb 1684 * @adapter: pointer to adapter struct 1685 * @queue: queue number 1686 * 1687 * Configure CBS and Launchtime for a given hardware queue. 1688 * Parameters are retrieved from the correct Tx ring, so 1689 * igb_save_cbs_params() and igb_save_txtime_params() should be used 1690 * for setting those correctly prior to this function being called. 1691 **/ 1692 static void igb_config_tx_modes(struct igb_adapter *adapter, int queue) 1693 { 1694 struct net_device *netdev = adapter->netdev; 1695 struct e1000_hw *hw = &adapter->hw; 1696 struct igb_ring *ring; 1697 u32 tqavcc, tqavctrl; 1698 u16 value; 1699 1700 WARN_ON(hw->mac.type != e1000_i210); 1701 WARN_ON(queue < 0 || queue > 1); 1702 ring = adapter->tx_ring[queue]; 1703 1704 /* If any of the Qav features is enabled, configure queues as SR and 1705 * with HIGH PRIO. If none is, then configure them with LOW PRIO and 1706 * as SP. 1707 */ 1708 if (ring->cbs_enable || ring->launchtime_enable) { 1709 set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_HIGH); 1710 set_queue_mode(hw, queue, QUEUE_MODE_STREAM_RESERVATION); 1711 } else { 1712 set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_LOW); 1713 set_queue_mode(hw, queue, QUEUE_MODE_STRICT_PRIORITY); 1714 } 1715 1716 /* If CBS is enabled, set DataTranARB and config its parameters. */ 1717 if (ring->cbs_enable || queue == 0) { 1718 /* i210 does not allow the queue 0 to be in the Strict 1719 * Priority mode while the Qav mode is enabled, so, 1720 * instead of disabling strict priority mode, we give 1721 * queue 0 the maximum of credits possible. 1722 * 1723 * See section 8.12.19 of the i210 datasheet, "Note: 1724 * Queue0 QueueMode must be set to 1b when 1725 * TransmitMode is set to Qav." 1726 */ 1727 if (queue == 0 && !ring->cbs_enable) { 1728 /* max "linkspeed" idleslope in kbps */ 1729 ring->idleslope = 1000000; 1730 ring->hicredit = ETH_FRAME_LEN; 1731 } 1732 1733 /* Always set data transfer arbitration to credit-based 1734 * shaper algorithm on TQAVCTRL if CBS is enabled for any of 1735 * the queues. 1736 */ 1737 tqavctrl = rd32(E1000_I210_TQAVCTRL); 1738 tqavctrl |= E1000_TQAVCTRL_DATATRANARB; 1739 wr32(E1000_I210_TQAVCTRL, tqavctrl); 1740 1741 /* According to i210 datasheet section 7.2.7.7, we should set 1742 * the 'idleSlope' field from TQAVCC register following the 1743 * equation: 1744 * 1745 * For 100 Mbps link speed: 1746 * 1747 * value = BW * 0x7735 * 0.2 (E1) 1748 * 1749 * For 1000Mbps link speed: 1750 * 1751 * value = BW * 0x7735 * 2 (E2) 1752 * 1753 * E1 and E2 can be merged into one equation as shown below. 1754 * Note that 'link-speed' is in Mbps. 1755 * 1756 * value = BW * 0x7735 * 2 * link-speed 1757 * -------------- (E3) 1758 * 1000 1759 * 1760 * 'BW' is the percentage bandwidth out of full link speed 1761 * which can be found with the following equation. Note that 1762 * idleSlope here is the parameter from this function which 1763 * is in kbps. 1764 * 1765 * BW = idleSlope 1766 * ----------------- (E4) 1767 * link-speed * 1000 1768 * 1769 * That said, we can come up with a generic equation to 1770 * calculate the value we should set it TQAVCC register by 1771 * replacing 'BW' in E3 by E4. The resulting equation is: 1772 * 1773 * value = idleSlope * 0x7735 * 2 * link-speed 1774 * ----------------- -------------- (E5) 1775 * link-speed * 1000 1000 1776 * 1777 * 'link-speed' is present in both sides of the fraction so 1778 * it is canceled out. The final equation is the following: 1779 * 1780 * value = idleSlope * 61034 1781 * ----------------- (E6) 1782 * 1000000 1783 * 1784 * NOTE: For i210, given the above, we can see that idleslope 1785 * is represented in 16.38431 kbps units by the value at 1786 * the TQAVCC register (1Gbps / 61034), which reduces 1787 * the granularity for idleslope increments. 1788 * For instance, if you want to configure a 2576kbps 1789 * idleslope, the value to be written on the register 1790 * would have to be 157.23. If rounded down, you end 1791 * up with less bandwidth available than originally 1792 * required (~2572 kbps). If rounded up, you end up 1793 * with a higher bandwidth (~2589 kbps). Below the 1794 * approach we take is to always round up the 1795 * calculated value, so the resulting bandwidth might 1796 * be slightly higher for some configurations. 1797 */ 1798 value = DIV_ROUND_UP_ULL(ring->idleslope * 61034ULL, 1000000); 1799 1800 tqavcc = rd32(E1000_I210_TQAVCC(queue)); 1801 tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK; 1802 tqavcc |= value; 1803 wr32(E1000_I210_TQAVCC(queue), tqavcc); 1804 1805 wr32(E1000_I210_TQAVHC(queue), 1806 0x80000000 + ring->hicredit * 0x7735); 1807 } else { 1808 1809 /* Set idleSlope to zero. */ 1810 tqavcc = rd32(E1000_I210_TQAVCC(queue)); 1811 tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK; 1812 wr32(E1000_I210_TQAVCC(queue), tqavcc); 1813 1814 /* Set hiCredit to zero. */ 1815 wr32(E1000_I210_TQAVHC(queue), 0); 1816 1817 /* If CBS is not enabled for any queues anymore, then return to 1818 * the default state of Data Transmission Arbitration on 1819 * TQAVCTRL. 1820 */ 1821 if (!is_any_cbs_enabled(adapter)) { 1822 tqavctrl = rd32(E1000_I210_TQAVCTRL); 1823 tqavctrl &= ~E1000_TQAVCTRL_DATATRANARB; 1824 wr32(E1000_I210_TQAVCTRL, tqavctrl); 1825 } 1826 } 1827 1828 /* If LaunchTime is enabled, set DataTranTIM. */ 1829 if (ring->launchtime_enable) { 1830 /* Always set DataTranTIM on TQAVCTRL if LaunchTime is enabled 1831 * for any of the SR queues, and configure fetchtime delta. 1832 * XXX NOTE: 1833 * - LaunchTime will be enabled for all SR queues. 1834 * - A fixed offset can be added relative to the launch 1835 * time of all packets if configured at reg LAUNCH_OS0. 1836 * We are keeping it as 0 for now (default value). 1837 */ 1838 tqavctrl = rd32(E1000_I210_TQAVCTRL); 1839 tqavctrl |= E1000_TQAVCTRL_DATATRANTIM | 1840 E1000_TQAVCTRL_FETCHTIME_DELTA; 1841 wr32(E1000_I210_TQAVCTRL, tqavctrl); 1842 } else { 1843 /* If Launchtime is not enabled for any SR queues anymore, 1844 * then clear DataTranTIM on TQAVCTRL and clear fetchtime delta, 1845 * effectively disabling Launchtime. 1846 */ 1847 if (!is_any_txtime_enabled(adapter)) { 1848 tqavctrl = rd32(E1000_I210_TQAVCTRL); 1849 tqavctrl &= ~E1000_TQAVCTRL_DATATRANTIM; 1850 tqavctrl &= ~E1000_TQAVCTRL_FETCHTIME_DELTA; 1851 wr32(E1000_I210_TQAVCTRL, tqavctrl); 1852 } 1853 } 1854 1855 /* XXX: In i210 controller the sendSlope and loCredit parameters from 1856 * CBS are not configurable by software so we don't do any 'controller 1857 * configuration' in respect to these parameters. 1858 */ 1859 1860 netdev_dbg(netdev, "Qav Tx mode: cbs %s, launchtime %s, queue %d idleslope %d sendslope %d hiCredit %d locredit %d\n", 1861 ring->cbs_enable ? "enabled" : "disabled", 1862 ring->launchtime_enable ? "enabled" : "disabled", 1863 queue, 1864 ring->idleslope, ring->sendslope, 1865 ring->hicredit, ring->locredit); 1866 } 1867 1868 static int igb_save_txtime_params(struct igb_adapter *adapter, int queue, 1869 bool enable) 1870 { 1871 struct igb_ring *ring; 1872 1873 if (queue < 0 || queue > adapter->num_tx_queues) 1874 return -EINVAL; 1875 1876 ring = adapter->tx_ring[queue]; 1877 ring->launchtime_enable = enable; 1878 1879 return 0; 1880 } 1881 1882 static int igb_save_cbs_params(struct igb_adapter *adapter, int queue, 1883 bool enable, int idleslope, int sendslope, 1884 int hicredit, int locredit) 1885 { 1886 struct igb_ring *ring; 1887 1888 if (queue < 0 || queue > adapter->num_tx_queues) 1889 return -EINVAL; 1890 1891 ring = adapter->tx_ring[queue]; 1892 1893 ring->cbs_enable = enable; 1894 ring->idleslope = idleslope; 1895 ring->sendslope = sendslope; 1896 ring->hicredit = hicredit; 1897 ring->locredit = locredit; 1898 1899 return 0; 1900 } 1901 1902 /** 1903 * igb_setup_tx_mode - Switch to/from Qav Tx mode when applicable 1904 * @adapter: pointer to adapter struct 1905 * 1906 * Configure TQAVCTRL register switching the controller's Tx mode 1907 * if FQTSS mode is enabled or disabled. Additionally, will issue 1908 * a call to igb_config_tx_modes() per queue so any previously saved 1909 * Tx parameters are applied. 1910 **/ 1911 static void igb_setup_tx_mode(struct igb_adapter *adapter) 1912 { 1913 struct net_device *netdev = adapter->netdev; 1914 struct e1000_hw *hw = &adapter->hw; 1915 u32 val; 1916 1917 /* Only i210 controller supports changing the transmission mode. */ 1918 if (hw->mac.type != e1000_i210) 1919 return; 1920 1921 if (is_fqtss_enabled(adapter)) { 1922 int i, max_queue; 1923 1924 /* Configure TQAVCTRL register: set transmit mode to 'Qav', 1925 * set data fetch arbitration to 'round robin', set SP_WAIT_SR 1926 * so SP queues wait for SR ones. 1927 */ 1928 val = rd32(E1000_I210_TQAVCTRL); 1929 val |= E1000_TQAVCTRL_XMIT_MODE | E1000_TQAVCTRL_SP_WAIT_SR; 1930 val &= ~E1000_TQAVCTRL_DATAFETCHARB; 1931 wr32(E1000_I210_TQAVCTRL, val); 1932 1933 /* Configure Tx and Rx packet buffers sizes as described in 1934 * i210 datasheet section 7.2.7.7. 1935 */ 1936 val = rd32(E1000_TXPBS); 1937 val &= ~I210_TXPBSIZE_MASK; 1938 val |= I210_TXPBSIZE_PB0_6KB | I210_TXPBSIZE_PB1_6KB | 1939 I210_TXPBSIZE_PB2_6KB | I210_TXPBSIZE_PB3_6KB; 1940 wr32(E1000_TXPBS, val); 1941 1942 val = rd32(E1000_RXPBS); 1943 val &= ~I210_RXPBSIZE_MASK; 1944 val |= I210_RXPBSIZE_PB_30KB; 1945 wr32(E1000_RXPBS, val); 1946 1947 /* Section 8.12.9 states that MAX_TPKT_SIZE from DTXMXPKTSZ 1948 * register should not exceed the buffer size programmed in 1949 * TXPBS. The smallest buffer size programmed in TXPBS is 4kB 1950 * so according to the datasheet we should set MAX_TPKT_SIZE to 1951 * 4kB / 64. 1952 * 1953 * However, when we do so, no frame from queue 2 and 3 are 1954 * transmitted. It seems the MAX_TPKT_SIZE should not be great 1955 * or _equal_ to the buffer size programmed in TXPBS. For this 1956 * reason, we set MAX_ TPKT_SIZE to (4kB - 1) / 64. 1957 */ 1958 val = (4096 - 1) / 64; 1959 wr32(E1000_I210_DTXMXPKTSZ, val); 1960 1961 /* Since FQTSS mode is enabled, apply any CBS configuration 1962 * previously set. If no previous CBS configuration has been 1963 * done, then the initial configuration is applied, which means 1964 * CBS is disabled. 1965 */ 1966 max_queue = (adapter->num_tx_queues < I210_SR_QUEUES_NUM) ? 1967 adapter->num_tx_queues : I210_SR_QUEUES_NUM; 1968 1969 for (i = 0; i < max_queue; i++) { 1970 igb_config_tx_modes(adapter, i); 1971 } 1972 } else { 1973 wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT); 1974 wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT); 1975 wr32(E1000_I210_DTXMXPKTSZ, I210_DTXMXPKTSZ_DEFAULT); 1976 1977 val = rd32(E1000_I210_TQAVCTRL); 1978 /* According to Section 8.12.21, the other flags we've set when 1979 * enabling FQTSS are not relevant when disabling FQTSS so we 1980 * don't set they here. 1981 */ 1982 val &= ~E1000_TQAVCTRL_XMIT_MODE; 1983 wr32(E1000_I210_TQAVCTRL, val); 1984 } 1985 1986 netdev_dbg(netdev, "FQTSS %s\n", (is_fqtss_enabled(adapter)) ? 1987 "enabled" : "disabled"); 1988 } 1989 1990 /** 1991 * igb_configure - configure the hardware for RX and TX 1992 * @adapter: private board structure 1993 **/ 1994 static void igb_configure(struct igb_adapter *adapter) 1995 { 1996 struct net_device *netdev = adapter->netdev; 1997 int i; 1998 1999 igb_get_hw_control(adapter); 2000 igb_set_rx_mode(netdev); 2001 igb_setup_tx_mode(adapter); 2002 2003 igb_restore_vlan(adapter); 2004 2005 igb_setup_tctl(adapter); 2006 igb_setup_mrqc(adapter); 2007 igb_setup_rctl(adapter); 2008 2009 igb_nfc_filter_restore(adapter); 2010 igb_configure_tx(adapter); 2011 igb_configure_rx(adapter); 2012 2013 igb_rx_fifo_flush_82575(&adapter->hw); 2014 2015 /* call igb_desc_unused which always leaves 2016 * at least 1 descriptor unused to make sure 2017 * next_to_use != next_to_clean 2018 */ 2019 for (i = 0; i < adapter->num_rx_queues; i++) { 2020 struct igb_ring *ring = adapter->rx_ring[i]; 2021 igb_alloc_rx_buffers(ring, igb_desc_unused(ring)); 2022 } 2023 } 2024 2025 /** 2026 * igb_power_up_link - Power up the phy/serdes link 2027 * @adapter: address of board private structure 2028 **/ 2029 void igb_power_up_link(struct igb_adapter *adapter) 2030 { 2031 igb_reset_phy(&adapter->hw); 2032 2033 if (adapter->hw.phy.media_type == e1000_media_type_copper) 2034 igb_power_up_phy_copper(&adapter->hw); 2035 else 2036 igb_power_up_serdes_link_82575(&adapter->hw); 2037 2038 igb_setup_link(&adapter->hw); 2039 } 2040 2041 /** 2042 * igb_power_down_link - Power down the phy/serdes link 2043 * @adapter: address of board private structure 2044 */ 2045 static void igb_power_down_link(struct igb_adapter *adapter) 2046 { 2047 if (adapter->hw.phy.media_type == e1000_media_type_copper) 2048 igb_power_down_phy_copper_82575(&adapter->hw); 2049 else 2050 igb_shutdown_serdes_link_82575(&adapter->hw); 2051 } 2052 2053 /** 2054 * igb_check_swap_media - Detect and switch function for Media Auto Sense 2055 * @adapter: address of the board private structure 2056 **/ 2057 static void igb_check_swap_media(struct igb_adapter *adapter) 2058 { 2059 struct e1000_hw *hw = &adapter->hw; 2060 u32 ctrl_ext, connsw; 2061 bool swap_now = false; 2062 2063 ctrl_ext = rd32(E1000_CTRL_EXT); 2064 connsw = rd32(E1000_CONNSW); 2065 2066 /* need to live swap if current media is copper and we have fiber/serdes 2067 * to go to. 2068 */ 2069 2070 if ((hw->phy.media_type == e1000_media_type_copper) && 2071 (!(connsw & E1000_CONNSW_AUTOSENSE_EN))) { 2072 swap_now = true; 2073 } else if ((hw->phy.media_type != e1000_media_type_copper) && 2074 !(connsw & E1000_CONNSW_SERDESD)) { 2075 /* copper signal takes time to appear */ 2076 if (adapter->copper_tries < 4) { 2077 adapter->copper_tries++; 2078 connsw |= E1000_CONNSW_AUTOSENSE_CONF; 2079 wr32(E1000_CONNSW, connsw); 2080 return; 2081 } else { 2082 adapter->copper_tries = 0; 2083 if ((connsw & E1000_CONNSW_PHYSD) && 2084 (!(connsw & E1000_CONNSW_PHY_PDN))) { 2085 swap_now = true; 2086 connsw &= ~E1000_CONNSW_AUTOSENSE_CONF; 2087 wr32(E1000_CONNSW, connsw); 2088 } 2089 } 2090 } 2091 2092 if (!swap_now) 2093 return; 2094 2095 switch (hw->phy.media_type) { 2096 case e1000_media_type_copper: 2097 netdev_info(adapter->netdev, 2098 "MAS: changing media to fiber/serdes\n"); 2099 ctrl_ext |= 2100 E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES; 2101 adapter->flags |= IGB_FLAG_MEDIA_RESET; 2102 adapter->copper_tries = 0; 2103 break; 2104 case e1000_media_type_internal_serdes: 2105 case e1000_media_type_fiber: 2106 netdev_info(adapter->netdev, 2107 "MAS: changing media to copper\n"); 2108 ctrl_ext &= 2109 ~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES; 2110 adapter->flags |= IGB_FLAG_MEDIA_RESET; 2111 break; 2112 default: 2113 /* shouldn't get here during regular operation */ 2114 netdev_err(adapter->netdev, 2115 "AMS: Invalid media type found, returning\n"); 2116 break; 2117 } 2118 wr32(E1000_CTRL_EXT, ctrl_ext); 2119 } 2120 2121 /** 2122 * igb_up - Open the interface and prepare it to handle traffic 2123 * @adapter: board private structure 2124 **/ 2125 int igb_up(struct igb_adapter *adapter) 2126 { 2127 struct e1000_hw *hw = &adapter->hw; 2128 int i; 2129 2130 /* hardware has been reset, we need to reload some things */ 2131 igb_configure(adapter); 2132 2133 clear_bit(__IGB_DOWN, &adapter->state); 2134 2135 for (i = 0; i < adapter->num_q_vectors; i++) 2136 napi_enable(&(adapter->q_vector[i]->napi)); 2137 2138 if (adapter->flags & IGB_FLAG_HAS_MSIX) 2139 igb_configure_msix(adapter); 2140 else 2141 igb_assign_vector(adapter->q_vector[0], 0); 2142 2143 /* Clear any pending interrupts. */ 2144 rd32(E1000_TSICR); 2145 rd32(E1000_ICR); 2146 igb_irq_enable(adapter); 2147 2148 /* notify VFs that reset has been completed */ 2149 if (adapter->vfs_allocated_count) { 2150 u32 reg_data = rd32(E1000_CTRL_EXT); 2151 2152 reg_data |= E1000_CTRL_EXT_PFRSTD; 2153 wr32(E1000_CTRL_EXT, reg_data); 2154 } 2155 2156 netif_tx_start_all_queues(adapter->netdev); 2157 2158 /* start the watchdog. */ 2159 hw->mac.get_link_status = 1; 2160 schedule_work(&adapter->watchdog_task); 2161 2162 if ((adapter->flags & IGB_FLAG_EEE) && 2163 (!hw->dev_spec._82575.eee_disable)) 2164 adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T; 2165 2166 return 0; 2167 } 2168 2169 void igb_down(struct igb_adapter *adapter) 2170 { 2171 struct net_device *netdev = adapter->netdev; 2172 struct e1000_hw *hw = &adapter->hw; 2173 u32 tctl, rctl; 2174 int i; 2175 2176 /* signal that we're down so the interrupt handler does not 2177 * reschedule our watchdog timer 2178 */ 2179 set_bit(__IGB_DOWN, &adapter->state); 2180 2181 /* disable receives in the hardware */ 2182 rctl = rd32(E1000_RCTL); 2183 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN); 2184 /* flush and sleep below */ 2185 2186 igb_nfc_filter_exit(adapter); 2187 2188 netif_carrier_off(netdev); 2189 netif_tx_stop_all_queues(netdev); 2190 2191 /* disable transmits in the hardware */ 2192 tctl = rd32(E1000_TCTL); 2193 tctl &= ~E1000_TCTL_EN; 2194 wr32(E1000_TCTL, tctl); 2195 /* flush both disables and wait for them to finish */ 2196 wrfl(); 2197 usleep_range(10000, 11000); 2198 2199 igb_irq_disable(adapter); 2200 2201 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE; 2202 2203 for (i = 0; i < adapter->num_q_vectors; i++) { 2204 if (adapter->q_vector[i]) { 2205 napi_synchronize(&adapter->q_vector[i]->napi); 2206 napi_disable(&adapter->q_vector[i]->napi); 2207 } 2208 } 2209 2210 del_timer_sync(&adapter->watchdog_timer); 2211 del_timer_sync(&adapter->phy_info_timer); 2212 2213 /* record the stats before reset*/ 2214 spin_lock(&adapter->stats64_lock); 2215 igb_update_stats(adapter); 2216 spin_unlock(&adapter->stats64_lock); 2217 2218 adapter->link_speed = 0; 2219 adapter->link_duplex = 0; 2220 2221 if (!pci_channel_offline(adapter->pdev)) 2222 igb_reset(adapter); 2223 2224 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 2225 adapter->flags &= ~IGB_FLAG_VLAN_PROMISC; 2226 2227 igb_clean_all_tx_rings(adapter); 2228 igb_clean_all_rx_rings(adapter); 2229 #ifdef CONFIG_IGB_DCA 2230 2231 /* since we reset the hardware DCA settings were cleared */ 2232 igb_setup_dca(adapter); 2233 #endif 2234 } 2235 2236 void igb_reinit_locked(struct igb_adapter *adapter) 2237 { 2238 while (test_and_set_bit(__IGB_RESETTING, &adapter->state)) 2239 usleep_range(1000, 2000); 2240 igb_down(adapter); 2241 igb_up(adapter); 2242 clear_bit(__IGB_RESETTING, &adapter->state); 2243 } 2244 2245 /** igb_enable_mas - Media Autosense re-enable after swap 2246 * 2247 * @adapter: adapter struct 2248 **/ 2249 static void igb_enable_mas(struct igb_adapter *adapter) 2250 { 2251 struct e1000_hw *hw = &adapter->hw; 2252 u32 connsw = rd32(E1000_CONNSW); 2253 2254 /* configure for SerDes media detect */ 2255 if ((hw->phy.media_type == e1000_media_type_copper) && 2256 (!(connsw & E1000_CONNSW_SERDESD))) { 2257 connsw |= E1000_CONNSW_ENRGSRC; 2258 connsw |= E1000_CONNSW_AUTOSENSE_EN; 2259 wr32(E1000_CONNSW, connsw); 2260 wrfl(); 2261 } 2262 } 2263 2264 #ifdef CONFIG_IGB_HWMON 2265 /** 2266 * igb_set_i2c_bb - Init I2C interface 2267 * @hw: pointer to hardware structure 2268 **/ 2269 static void igb_set_i2c_bb(struct e1000_hw *hw) 2270 { 2271 u32 ctrl_ext; 2272 s32 i2cctl; 2273 2274 ctrl_ext = rd32(E1000_CTRL_EXT); 2275 ctrl_ext |= E1000_CTRL_I2C_ENA; 2276 wr32(E1000_CTRL_EXT, ctrl_ext); 2277 wrfl(); 2278 2279 i2cctl = rd32(E1000_I2CPARAMS); 2280 i2cctl |= E1000_I2CBB_EN 2281 | E1000_I2C_CLK_OE_N 2282 | E1000_I2C_DATA_OE_N; 2283 wr32(E1000_I2CPARAMS, i2cctl); 2284 wrfl(); 2285 } 2286 #endif 2287 2288 void igb_reset(struct igb_adapter *adapter) 2289 { 2290 struct pci_dev *pdev = adapter->pdev; 2291 struct e1000_hw *hw = &adapter->hw; 2292 struct e1000_mac_info *mac = &hw->mac; 2293 struct e1000_fc_info *fc = &hw->fc; 2294 u32 pba, hwm; 2295 2296 /* Repartition Pba for greater than 9k mtu 2297 * To take effect CTRL.RST is required. 2298 */ 2299 switch (mac->type) { 2300 case e1000_i350: 2301 case e1000_i354: 2302 case e1000_82580: 2303 pba = rd32(E1000_RXPBS); 2304 pba = igb_rxpbs_adjust_82580(pba); 2305 break; 2306 case e1000_82576: 2307 pba = rd32(E1000_RXPBS); 2308 pba &= E1000_RXPBS_SIZE_MASK_82576; 2309 break; 2310 case e1000_82575: 2311 case e1000_i210: 2312 case e1000_i211: 2313 default: 2314 pba = E1000_PBA_34K; 2315 break; 2316 } 2317 2318 if (mac->type == e1000_82575) { 2319 u32 min_rx_space, min_tx_space, needed_tx_space; 2320 2321 /* write Rx PBA so that hardware can report correct Tx PBA */ 2322 wr32(E1000_PBA, pba); 2323 2324 /* To maintain wire speed transmits, the Tx FIFO should be 2325 * large enough to accommodate two full transmit packets, 2326 * rounded up to the next 1KB and expressed in KB. Likewise, 2327 * the Rx FIFO should be large enough to accommodate at least 2328 * one full receive packet and is similarly rounded up and 2329 * expressed in KB. 2330 */ 2331 min_rx_space = DIV_ROUND_UP(MAX_JUMBO_FRAME_SIZE, 1024); 2332 2333 /* The Tx FIFO also stores 16 bytes of information about the Tx 2334 * but don't include Ethernet FCS because hardware appends it. 2335 * We only need to round down to the nearest 512 byte block 2336 * count since the value we care about is 2 frames, not 1. 2337 */ 2338 min_tx_space = adapter->max_frame_size; 2339 min_tx_space += sizeof(union e1000_adv_tx_desc) - ETH_FCS_LEN; 2340 min_tx_space = DIV_ROUND_UP(min_tx_space, 512); 2341 2342 /* upper 16 bits has Tx packet buffer allocation size in KB */ 2343 needed_tx_space = min_tx_space - (rd32(E1000_PBA) >> 16); 2344 2345 /* If current Tx allocation is less than the min Tx FIFO size, 2346 * and the min Tx FIFO size is less than the current Rx FIFO 2347 * allocation, take space away from current Rx allocation. 2348 */ 2349 if (needed_tx_space < pba) { 2350 pba -= needed_tx_space; 2351 2352 /* if short on Rx space, Rx wins and must trump Tx 2353 * adjustment 2354 */ 2355 if (pba < min_rx_space) 2356 pba = min_rx_space; 2357 } 2358 2359 /* adjust PBA for jumbo frames */ 2360 wr32(E1000_PBA, pba); 2361 } 2362 2363 /* flow control settings 2364 * The high water mark must be low enough to fit one full frame 2365 * after transmitting the pause frame. As such we must have enough 2366 * space to allow for us to complete our current transmit and then 2367 * receive the frame that is in progress from the link partner. 2368 * Set it to: 2369 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 2370 */ 2371 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 2372 2373 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 2374 fc->low_water = fc->high_water - 16; 2375 fc->pause_time = 0xFFFF; 2376 fc->send_xon = 1; 2377 fc->current_mode = fc->requested_mode; 2378 2379 /* disable receive for all VFs and wait one second */ 2380 if (adapter->vfs_allocated_count) { 2381 int i; 2382 2383 for (i = 0 ; i < adapter->vfs_allocated_count; i++) 2384 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC; 2385 2386 /* ping all the active vfs to let them know we are going down */ 2387 igb_ping_all_vfs(adapter); 2388 2389 /* disable transmits and receives */ 2390 wr32(E1000_VFRE, 0); 2391 wr32(E1000_VFTE, 0); 2392 } 2393 2394 /* Allow time for pending master requests to run */ 2395 hw->mac.ops.reset_hw(hw); 2396 wr32(E1000_WUC, 0); 2397 2398 if (adapter->flags & IGB_FLAG_MEDIA_RESET) { 2399 /* need to resetup here after media swap */ 2400 adapter->ei.get_invariants(hw); 2401 adapter->flags &= ~IGB_FLAG_MEDIA_RESET; 2402 } 2403 if ((mac->type == e1000_82575 || mac->type == e1000_i350) && 2404 (adapter->flags & IGB_FLAG_MAS_ENABLE)) { 2405 igb_enable_mas(adapter); 2406 } 2407 if (hw->mac.ops.init_hw(hw)) 2408 dev_err(&pdev->dev, "Hardware Error\n"); 2409 2410 /* RAR registers were cleared during init_hw, clear mac table */ 2411 igb_flush_mac_table(adapter); 2412 __dev_uc_unsync(adapter->netdev, NULL); 2413 2414 /* Recover default RAR entry */ 2415 igb_set_default_mac_filter(adapter); 2416 2417 /* Flow control settings reset on hardware reset, so guarantee flow 2418 * control is off when forcing speed. 2419 */ 2420 if (!hw->mac.autoneg) 2421 igb_force_mac_fc(hw); 2422 2423 igb_init_dmac(adapter, pba); 2424 #ifdef CONFIG_IGB_HWMON 2425 /* Re-initialize the thermal sensor on i350 devices. */ 2426 if (!test_bit(__IGB_DOWN, &adapter->state)) { 2427 if (mac->type == e1000_i350 && hw->bus.func == 0) { 2428 /* If present, re-initialize the external thermal sensor 2429 * interface. 2430 */ 2431 if (adapter->ets) 2432 igb_set_i2c_bb(hw); 2433 mac->ops.init_thermal_sensor_thresh(hw); 2434 } 2435 } 2436 #endif 2437 /* Re-establish EEE setting */ 2438 if (hw->phy.media_type == e1000_media_type_copper) { 2439 switch (mac->type) { 2440 case e1000_i350: 2441 case e1000_i210: 2442 case e1000_i211: 2443 igb_set_eee_i350(hw, true, true); 2444 break; 2445 case e1000_i354: 2446 igb_set_eee_i354(hw, true, true); 2447 break; 2448 default: 2449 break; 2450 } 2451 } 2452 if (!netif_running(adapter->netdev)) 2453 igb_power_down_link(adapter); 2454 2455 igb_update_mng_vlan(adapter); 2456 2457 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */ 2458 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE); 2459 2460 /* Re-enable PTP, where applicable. */ 2461 if (adapter->ptp_flags & IGB_PTP_ENABLED) 2462 igb_ptp_reset(adapter); 2463 2464 igb_get_phy_info(hw); 2465 } 2466 2467 static netdev_features_t igb_fix_features(struct net_device *netdev, 2468 netdev_features_t features) 2469 { 2470 /* Since there is no support for separate Rx/Tx vlan accel 2471 * enable/disable make sure Tx flag is always in same state as Rx. 2472 */ 2473 if (features & NETIF_F_HW_VLAN_CTAG_RX) 2474 features |= NETIF_F_HW_VLAN_CTAG_TX; 2475 else 2476 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 2477 2478 return features; 2479 } 2480 2481 static int igb_set_features(struct net_device *netdev, 2482 netdev_features_t features) 2483 { 2484 netdev_features_t changed = netdev->features ^ features; 2485 struct igb_adapter *adapter = netdev_priv(netdev); 2486 2487 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 2488 igb_vlan_mode(netdev, features); 2489 2490 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 2491 return 0; 2492 2493 if (!(features & NETIF_F_NTUPLE)) { 2494 struct hlist_node *node2; 2495 struct igb_nfc_filter *rule; 2496 2497 spin_lock(&adapter->nfc_lock); 2498 hlist_for_each_entry_safe(rule, node2, 2499 &adapter->nfc_filter_list, nfc_node) { 2500 igb_erase_filter(adapter, rule); 2501 hlist_del(&rule->nfc_node); 2502 kfree(rule); 2503 } 2504 spin_unlock(&adapter->nfc_lock); 2505 adapter->nfc_filter_count = 0; 2506 } 2507 2508 netdev->features = features; 2509 2510 if (netif_running(netdev)) 2511 igb_reinit_locked(adapter); 2512 else 2513 igb_reset(adapter); 2514 2515 return 1; 2516 } 2517 2518 static int igb_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 2519 struct net_device *dev, 2520 const unsigned char *addr, u16 vid, 2521 u16 flags, 2522 struct netlink_ext_ack *extack) 2523 { 2524 /* guarantee we can provide a unique filter for the unicast address */ 2525 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) { 2526 struct igb_adapter *adapter = netdev_priv(dev); 2527 int vfn = adapter->vfs_allocated_count; 2528 2529 if (netdev_uc_count(dev) >= igb_available_rars(adapter, vfn)) 2530 return -ENOMEM; 2531 } 2532 2533 return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags); 2534 } 2535 2536 #define IGB_MAX_MAC_HDR_LEN 127 2537 #define IGB_MAX_NETWORK_HDR_LEN 511 2538 2539 static netdev_features_t 2540 igb_features_check(struct sk_buff *skb, struct net_device *dev, 2541 netdev_features_t features) 2542 { 2543 unsigned int network_hdr_len, mac_hdr_len; 2544 2545 /* Make certain the headers can be described by a context descriptor */ 2546 mac_hdr_len = skb_network_header(skb) - skb->data; 2547 if (unlikely(mac_hdr_len > IGB_MAX_MAC_HDR_LEN)) 2548 return features & ~(NETIF_F_HW_CSUM | 2549 NETIF_F_SCTP_CRC | 2550 NETIF_F_GSO_UDP_L4 | 2551 NETIF_F_HW_VLAN_CTAG_TX | 2552 NETIF_F_TSO | 2553 NETIF_F_TSO6); 2554 2555 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 2556 if (unlikely(network_hdr_len > IGB_MAX_NETWORK_HDR_LEN)) 2557 return features & ~(NETIF_F_HW_CSUM | 2558 NETIF_F_SCTP_CRC | 2559 NETIF_F_GSO_UDP_L4 | 2560 NETIF_F_TSO | 2561 NETIF_F_TSO6); 2562 2563 /* We can only support IPV4 TSO in tunnels if we can mangle the 2564 * inner IP ID field, so strip TSO if MANGLEID is not supported. 2565 */ 2566 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 2567 features &= ~NETIF_F_TSO; 2568 2569 return features; 2570 } 2571 2572 static void igb_offload_apply(struct igb_adapter *adapter, s32 queue) 2573 { 2574 if (!is_fqtss_enabled(adapter)) { 2575 enable_fqtss(adapter, true); 2576 return; 2577 } 2578 2579 igb_config_tx_modes(adapter, queue); 2580 2581 if (!is_any_cbs_enabled(adapter) && !is_any_txtime_enabled(adapter)) 2582 enable_fqtss(adapter, false); 2583 } 2584 2585 static int igb_offload_cbs(struct igb_adapter *adapter, 2586 struct tc_cbs_qopt_offload *qopt) 2587 { 2588 struct e1000_hw *hw = &adapter->hw; 2589 int err; 2590 2591 /* CBS offloading is only supported by i210 controller. */ 2592 if (hw->mac.type != e1000_i210) 2593 return -EOPNOTSUPP; 2594 2595 /* CBS offloading is only supported by queue 0 and queue 1. */ 2596 if (qopt->queue < 0 || qopt->queue > 1) 2597 return -EINVAL; 2598 2599 err = igb_save_cbs_params(adapter, qopt->queue, qopt->enable, 2600 qopt->idleslope, qopt->sendslope, 2601 qopt->hicredit, qopt->locredit); 2602 if (err) 2603 return err; 2604 2605 igb_offload_apply(adapter, qopt->queue); 2606 2607 return 0; 2608 } 2609 2610 #define ETHER_TYPE_FULL_MASK ((__force __be16)~0) 2611 #define VLAN_PRIO_FULL_MASK (0x07) 2612 2613 static int igb_parse_cls_flower(struct igb_adapter *adapter, 2614 struct flow_cls_offload *f, 2615 int traffic_class, 2616 struct igb_nfc_filter *input) 2617 { 2618 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 2619 struct flow_dissector *dissector = rule->match.dissector; 2620 struct netlink_ext_ack *extack = f->common.extack; 2621 2622 if (dissector->used_keys & 2623 ~(BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | 2624 BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | 2625 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 2626 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN))) { 2627 NL_SET_ERR_MSG_MOD(extack, 2628 "Unsupported key used, only BASIC, CONTROL, ETH_ADDRS and VLAN are supported"); 2629 return -EOPNOTSUPP; 2630 } 2631 2632 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 2633 struct flow_match_eth_addrs match; 2634 2635 flow_rule_match_eth_addrs(rule, &match); 2636 if (!is_zero_ether_addr(match.mask->dst)) { 2637 if (!is_broadcast_ether_addr(match.mask->dst)) { 2638 NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for destination MAC address"); 2639 return -EINVAL; 2640 } 2641 2642 input->filter.match_flags |= 2643 IGB_FILTER_FLAG_DST_MAC_ADDR; 2644 ether_addr_copy(input->filter.dst_addr, match.key->dst); 2645 } 2646 2647 if (!is_zero_ether_addr(match.mask->src)) { 2648 if (!is_broadcast_ether_addr(match.mask->src)) { 2649 NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for source MAC address"); 2650 return -EINVAL; 2651 } 2652 2653 input->filter.match_flags |= 2654 IGB_FILTER_FLAG_SRC_MAC_ADDR; 2655 ether_addr_copy(input->filter.src_addr, match.key->src); 2656 } 2657 } 2658 2659 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 2660 struct flow_match_basic match; 2661 2662 flow_rule_match_basic(rule, &match); 2663 if (match.mask->n_proto) { 2664 if (match.mask->n_proto != ETHER_TYPE_FULL_MASK) { 2665 NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for EtherType filter"); 2666 return -EINVAL; 2667 } 2668 2669 input->filter.match_flags |= IGB_FILTER_FLAG_ETHER_TYPE; 2670 input->filter.etype = match.key->n_proto; 2671 } 2672 } 2673 2674 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 2675 struct flow_match_vlan match; 2676 2677 flow_rule_match_vlan(rule, &match); 2678 if (match.mask->vlan_priority) { 2679 if (match.mask->vlan_priority != VLAN_PRIO_FULL_MASK) { 2680 NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for VLAN priority"); 2681 return -EINVAL; 2682 } 2683 2684 input->filter.match_flags |= IGB_FILTER_FLAG_VLAN_TCI; 2685 input->filter.vlan_tci = 2686 (__force __be16)match.key->vlan_priority; 2687 } 2688 } 2689 2690 input->action = traffic_class; 2691 input->cookie = f->cookie; 2692 2693 return 0; 2694 } 2695 2696 static int igb_configure_clsflower(struct igb_adapter *adapter, 2697 struct flow_cls_offload *cls_flower) 2698 { 2699 struct netlink_ext_ack *extack = cls_flower->common.extack; 2700 struct igb_nfc_filter *filter, *f; 2701 int err, tc; 2702 2703 tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid); 2704 if (tc < 0) { 2705 NL_SET_ERR_MSG_MOD(extack, "Invalid traffic class"); 2706 return -EINVAL; 2707 } 2708 2709 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 2710 if (!filter) 2711 return -ENOMEM; 2712 2713 err = igb_parse_cls_flower(adapter, cls_flower, tc, filter); 2714 if (err < 0) 2715 goto err_parse; 2716 2717 spin_lock(&adapter->nfc_lock); 2718 2719 hlist_for_each_entry(f, &adapter->nfc_filter_list, nfc_node) { 2720 if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) { 2721 err = -EEXIST; 2722 NL_SET_ERR_MSG_MOD(extack, 2723 "This filter is already set in ethtool"); 2724 goto err_locked; 2725 } 2726 } 2727 2728 hlist_for_each_entry(f, &adapter->cls_flower_list, nfc_node) { 2729 if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) { 2730 err = -EEXIST; 2731 NL_SET_ERR_MSG_MOD(extack, 2732 "This filter is already set in cls_flower"); 2733 goto err_locked; 2734 } 2735 } 2736 2737 err = igb_add_filter(adapter, filter); 2738 if (err < 0) { 2739 NL_SET_ERR_MSG_MOD(extack, "Could not add filter to the adapter"); 2740 goto err_locked; 2741 } 2742 2743 hlist_add_head(&filter->nfc_node, &adapter->cls_flower_list); 2744 2745 spin_unlock(&adapter->nfc_lock); 2746 2747 return 0; 2748 2749 err_locked: 2750 spin_unlock(&adapter->nfc_lock); 2751 2752 err_parse: 2753 kfree(filter); 2754 2755 return err; 2756 } 2757 2758 static int igb_delete_clsflower(struct igb_adapter *adapter, 2759 struct flow_cls_offload *cls_flower) 2760 { 2761 struct igb_nfc_filter *filter; 2762 int err; 2763 2764 spin_lock(&adapter->nfc_lock); 2765 2766 hlist_for_each_entry(filter, &adapter->cls_flower_list, nfc_node) 2767 if (filter->cookie == cls_flower->cookie) 2768 break; 2769 2770 if (!filter) { 2771 err = -ENOENT; 2772 goto out; 2773 } 2774 2775 err = igb_erase_filter(adapter, filter); 2776 if (err < 0) 2777 goto out; 2778 2779 hlist_del(&filter->nfc_node); 2780 kfree(filter); 2781 2782 out: 2783 spin_unlock(&adapter->nfc_lock); 2784 2785 return err; 2786 } 2787 2788 static int igb_setup_tc_cls_flower(struct igb_adapter *adapter, 2789 struct flow_cls_offload *cls_flower) 2790 { 2791 switch (cls_flower->command) { 2792 case FLOW_CLS_REPLACE: 2793 return igb_configure_clsflower(adapter, cls_flower); 2794 case FLOW_CLS_DESTROY: 2795 return igb_delete_clsflower(adapter, cls_flower); 2796 case FLOW_CLS_STATS: 2797 return -EOPNOTSUPP; 2798 default: 2799 return -EOPNOTSUPP; 2800 } 2801 } 2802 2803 static int igb_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 2804 void *cb_priv) 2805 { 2806 struct igb_adapter *adapter = cb_priv; 2807 2808 if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data)) 2809 return -EOPNOTSUPP; 2810 2811 switch (type) { 2812 case TC_SETUP_CLSFLOWER: 2813 return igb_setup_tc_cls_flower(adapter, type_data); 2814 2815 default: 2816 return -EOPNOTSUPP; 2817 } 2818 } 2819 2820 static int igb_offload_txtime(struct igb_adapter *adapter, 2821 struct tc_etf_qopt_offload *qopt) 2822 { 2823 struct e1000_hw *hw = &adapter->hw; 2824 int err; 2825 2826 /* Launchtime offloading is only supported by i210 controller. */ 2827 if (hw->mac.type != e1000_i210) 2828 return -EOPNOTSUPP; 2829 2830 /* Launchtime offloading is only supported by queues 0 and 1. */ 2831 if (qopt->queue < 0 || qopt->queue > 1) 2832 return -EINVAL; 2833 2834 err = igb_save_txtime_params(adapter, qopt->queue, qopt->enable); 2835 if (err) 2836 return err; 2837 2838 igb_offload_apply(adapter, qopt->queue); 2839 2840 return 0; 2841 } 2842 2843 static int igb_tc_query_caps(struct igb_adapter *adapter, 2844 struct tc_query_caps_base *base) 2845 { 2846 switch (base->type) { 2847 case TC_SETUP_QDISC_TAPRIO: { 2848 struct tc_taprio_caps *caps = base->caps; 2849 2850 caps->broken_mqprio = true; 2851 2852 return 0; 2853 } 2854 default: 2855 return -EOPNOTSUPP; 2856 } 2857 } 2858 2859 static LIST_HEAD(igb_block_cb_list); 2860 2861 static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type, 2862 void *type_data) 2863 { 2864 struct igb_adapter *adapter = netdev_priv(dev); 2865 2866 switch (type) { 2867 case TC_QUERY_CAPS: 2868 return igb_tc_query_caps(adapter, type_data); 2869 case TC_SETUP_QDISC_CBS: 2870 return igb_offload_cbs(adapter, type_data); 2871 case TC_SETUP_BLOCK: 2872 return flow_block_cb_setup_simple(type_data, 2873 &igb_block_cb_list, 2874 igb_setup_tc_block_cb, 2875 adapter, adapter, true); 2876 2877 case TC_SETUP_QDISC_ETF: 2878 return igb_offload_txtime(adapter, type_data); 2879 2880 default: 2881 return -EOPNOTSUPP; 2882 } 2883 } 2884 2885 static int igb_xdp_setup(struct net_device *dev, struct netdev_bpf *bpf) 2886 { 2887 int i, frame_size = dev->mtu + IGB_ETH_PKT_HDR_PAD; 2888 struct igb_adapter *adapter = netdev_priv(dev); 2889 struct bpf_prog *prog = bpf->prog, *old_prog; 2890 bool running = netif_running(dev); 2891 bool need_reset; 2892 2893 /* verify igb ring attributes are sufficient for XDP */ 2894 for (i = 0; i < adapter->num_rx_queues; i++) { 2895 struct igb_ring *ring = adapter->rx_ring[i]; 2896 2897 if (frame_size > igb_rx_bufsz(ring)) { 2898 NL_SET_ERR_MSG_MOD(bpf->extack, 2899 "The RX buffer size is too small for the frame size"); 2900 netdev_warn(dev, "XDP RX buffer size %d is too small for the frame size %d\n", 2901 igb_rx_bufsz(ring), frame_size); 2902 return -EINVAL; 2903 } 2904 } 2905 2906 old_prog = xchg(&adapter->xdp_prog, prog); 2907 need_reset = (!!prog != !!old_prog); 2908 2909 /* device is up and bpf is added/removed, must setup the RX queues */ 2910 if (need_reset && running) { 2911 igb_close(dev); 2912 } else { 2913 for (i = 0; i < adapter->num_rx_queues; i++) 2914 (void)xchg(&adapter->rx_ring[i]->xdp_prog, 2915 adapter->xdp_prog); 2916 } 2917 2918 if (old_prog) 2919 bpf_prog_put(old_prog); 2920 2921 /* bpf is just replaced, RXQ and MTU are already setup */ 2922 if (!need_reset) { 2923 return 0; 2924 } else { 2925 if (prog) 2926 xdp_features_set_redirect_target(dev, true); 2927 else 2928 xdp_features_clear_redirect_target(dev); 2929 } 2930 2931 if (running) 2932 igb_open(dev); 2933 2934 return 0; 2935 } 2936 2937 static int igb_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2938 { 2939 switch (xdp->command) { 2940 case XDP_SETUP_PROG: 2941 return igb_xdp_setup(dev, xdp); 2942 default: 2943 return -EINVAL; 2944 } 2945 } 2946 2947 /* This function assumes __netif_tx_lock is held by the caller. */ 2948 static void igb_xdp_ring_update_tail(struct igb_ring *ring) 2949 { 2950 lockdep_assert_held(&txring_txq(ring)->_xmit_lock); 2951 2952 /* Force memory writes to complete before letting h/w know there 2953 * are new descriptors to fetch. 2954 */ 2955 wmb(); 2956 writel(ring->next_to_use, ring->tail); 2957 } 2958 2959 static struct igb_ring *igb_xdp_tx_queue_mapping(struct igb_adapter *adapter) 2960 { 2961 unsigned int r_idx = smp_processor_id(); 2962 2963 if (r_idx >= adapter->num_tx_queues) 2964 r_idx = r_idx % adapter->num_tx_queues; 2965 2966 return adapter->tx_ring[r_idx]; 2967 } 2968 2969 static int igb_xdp_xmit_back(struct igb_adapter *adapter, struct xdp_buff *xdp) 2970 { 2971 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 2972 int cpu = smp_processor_id(); 2973 struct igb_ring *tx_ring; 2974 struct netdev_queue *nq; 2975 u32 ret; 2976 2977 if (unlikely(!xdpf)) 2978 return IGB_XDP_CONSUMED; 2979 2980 /* During program transitions its possible adapter->xdp_prog is assigned 2981 * but ring has not been configured yet. In this case simply abort xmit. 2982 */ 2983 tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL; 2984 if (unlikely(!tx_ring)) 2985 return IGB_XDP_CONSUMED; 2986 2987 nq = txring_txq(tx_ring); 2988 __netif_tx_lock(nq, cpu); 2989 /* Avoid transmit queue timeout since we share it with the slow path */ 2990 txq_trans_cond_update(nq); 2991 ret = igb_xmit_xdp_ring(adapter, tx_ring, xdpf); 2992 __netif_tx_unlock(nq); 2993 2994 return ret; 2995 } 2996 2997 static int igb_xdp_xmit(struct net_device *dev, int n, 2998 struct xdp_frame **frames, u32 flags) 2999 { 3000 struct igb_adapter *adapter = netdev_priv(dev); 3001 int cpu = smp_processor_id(); 3002 struct igb_ring *tx_ring; 3003 struct netdev_queue *nq; 3004 int nxmit = 0; 3005 int i; 3006 3007 if (unlikely(test_bit(__IGB_DOWN, &adapter->state))) 3008 return -ENETDOWN; 3009 3010 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 3011 return -EINVAL; 3012 3013 /* During program transitions its possible adapter->xdp_prog is assigned 3014 * but ring has not been configured yet. In this case simply abort xmit. 3015 */ 3016 tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL; 3017 if (unlikely(!tx_ring)) 3018 return -ENXIO; 3019 3020 nq = txring_txq(tx_ring); 3021 __netif_tx_lock(nq, cpu); 3022 3023 /* Avoid transmit queue timeout since we share it with the slow path */ 3024 txq_trans_cond_update(nq); 3025 3026 for (i = 0; i < n; i++) { 3027 struct xdp_frame *xdpf = frames[i]; 3028 int err; 3029 3030 err = igb_xmit_xdp_ring(adapter, tx_ring, xdpf); 3031 if (err != IGB_XDP_TX) 3032 break; 3033 nxmit++; 3034 } 3035 3036 if (unlikely(flags & XDP_XMIT_FLUSH)) 3037 igb_xdp_ring_update_tail(tx_ring); 3038 3039 __netif_tx_unlock(nq); 3040 3041 return nxmit; 3042 } 3043 3044 static const struct net_device_ops igb_netdev_ops = { 3045 .ndo_open = igb_open, 3046 .ndo_stop = igb_close, 3047 .ndo_start_xmit = igb_xmit_frame, 3048 .ndo_get_stats64 = igb_get_stats64, 3049 .ndo_set_rx_mode = igb_set_rx_mode, 3050 .ndo_set_mac_address = igb_set_mac, 3051 .ndo_change_mtu = igb_change_mtu, 3052 .ndo_eth_ioctl = igb_ioctl, 3053 .ndo_tx_timeout = igb_tx_timeout, 3054 .ndo_validate_addr = eth_validate_addr, 3055 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid, 3056 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid, 3057 .ndo_set_vf_mac = igb_ndo_set_vf_mac, 3058 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan, 3059 .ndo_set_vf_rate = igb_ndo_set_vf_bw, 3060 .ndo_set_vf_spoofchk = igb_ndo_set_vf_spoofchk, 3061 .ndo_set_vf_trust = igb_ndo_set_vf_trust, 3062 .ndo_get_vf_config = igb_ndo_get_vf_config, 3063 .ndo_fix_features = igb_fix_features, 3064 .ndo_set_features = igb_set_features, 3065 .ndo_fdb_add = igb_ndo_fdb_add, 3066 .ndo_features_check = igb_features_check, 3067 .ndo_setup_tc = igb_setup_tc, 3068 .ndo_bpf = igb_xdp, 3069 .ndo_xdp_xmit = igb_xdp_xmit, 3070 }; 3071 3072 /** 3073 * igb_set_fw_version - Configure version string for ethtool 3074 * @adapter: adapter struct 3075 **/ 3076 void igb_set_fw_version(struct igb_adapter *adapter) 3077 { 3078 struct e1000_hw *hw = &adapter->hw; 3079 struct e1000_fw_version fw; 3080 3081 igb_get_fw_version(hw, &fw); 3082 3083 switch (hw->mac.type) { 3084 case e1000_i210: 3085 case e1000_i211: 3086 if (!(igb_get_flash_presence_i210(hw))) { 3087 snprintf(adapter->fw_version, 3088 sizeof(adapter->fw_version), 3089 "%2d.%2d-%d", 3090 fw.invm_major, fw.invm_minor, 3091 fw.invm_img_type); 3092 break; 3093 } 3094 fallthrough; 3095 default: 3096 /* if option is rom valid, display its version too */ 3097 if (fw.or_valid) { 3098 snprintf(adapter->fw_version, 3099 sizeof(adapter->fw_version), 3100 "%d.%d, 0x%08x, %d.%d.%d", 3101 fw.eep_major, fw.eep_minor, fw.etrack_id, 3102 fw.or_major, fw.or_build, fw.or_patch); 3103 /* no option rom */ 3104 } else if (fw.etrack_id != 0X0000) { 3105 snprintf(adapter->fw_version, 3106 sizeof(adapter->fw_version), 3107 "%d.%d, 0x%08x", 3108 fw.eep_major, fw.eep_minor, fw.etrack_id); 3109 } else { 3110 snprintf(adapter->fw_version, 3111 sizeof(adapter->fw_version), 3112 "%d.%d.%d", 3113 fw.eep_major, fw.eep_minor, fw.eep_build); 3114 } 3115 break; 3116 } 3117 } 3118 3119 /** 3120 * igb_init_mas - init Media Autosense feature if enabled in the NVM 3121 * 3122 * @adapter: adapter struct 3123 **/ 3124 static void igb_init_mas(struct igb_adapter *adapter) 3125 { 3126 struct e1000_hw *hw = &adapter->hw; 3127 u16 eeprom_data; 3128 3129 hw->nvm.ops.read(hw, NVM_COMPAT, 1, &eeprom_data); 3130 switch (hw->bus.func) { 3131 case E1000_FUNC_0: 3132 if (eeprom_data & IGB_MAS_ENABLE_0) { 3133 adapter->flags |= IGB_FLAG_MAS_ENABLE; 3134 netdev_info(adapter->netdev, 3135 "MAS: Enabling Media Autosense for port %d\n", 3136 hw->bus.func); 3137 } 3138 break; 3139 case E1000_FUNC_1: 3140 if (eeprom_data & IGB_MAS_ENABLE_1) { 3141 adapter->flags |= IGB_FLAG_MAS_ENABLE; 3142 netdev_info(adapter->netdev, 3143 "MAS: Enabling Media Autosense for port %d\n", 3144 hw->bus.func); 3145 } 3146 break; 3147 case E1000_FUNC_2: 3148 if (eeprom_data & IGB_MAS_ENABLE_2) { 3149 adapter->flags |= IGB_FLAG_MAS_ENABLE; 3150 netdev_info(adapter->netdev, 3151 "MAS: Enabling Media Autosense for port %d\n", 3152 hw->bus.func); 3153 } 3154 break; 3155 case E1000_FUNC_3: 3156 if (eeprom_data & IGB_MAS_ENABLE_3) { 3157 adapter->flags |= IGB_FLAG_MAS_ENABLE; 3158 netdev_info(adapter->netdev, 3159 "MAS: Enabling Media Autosense for port %d\n", 3160 hw->bus.func); 3161 } 3162 break; 3163 default: 3164 /* Shouldn't get here */ 3165 netdev_err(adapter->netdev, 3166 "MAS: Invalid port configuration, returning\n"); 3167 break; 3168 } 3169 } 3170 3171 /** 3172 * igb_init_i2c - Init I2C interface 3173 * @adapter: pointer to adapter structure 3174 **/ 3175 static s32 igb_init_i2c(struct igb_adapter *adapter) 3176 { 3177 s32 status = 0; 3178 3179 /* I2C interface supported on i350 devices */ 3180 if (adapter->hw.mac.type != e1000_i350) 3181 return 0; 3182 3183 /* Initialize the i2c bus which is controlled by the registers. 3184 * This bus will use the i2c_algo_bit structure that implements 3185 * the protocol through toggling of the 4 bits in the register. 3186 */ 3187 adapter->i2c_adap.owner = THIS_MODULE; 3188 adapter->i2c_algo = igb_i2c_algo; 3189 adapter->i2c_algo.data = adapter; 3190 adapter->i2c_adap.algo_data = &adapter->i2c_algo; 3191 adapter->i2c_adap.dev.parent = &adapter->pdev->dev; 3192 strscpy(adapter->i2c_adap.name, "igb BB", 3193 sizeof(adapter->i2c_adap.name)); 3194 status = i2c_bit_add_bus(&adapter->i2c_adap); 3195 return status; 3196 } 3197 3198 /** 3199 * igb_probe - Device Initialization Routine 3200 * @pdev: PCI device information struct 3201 * @ent: entry in igb_pci_tbl 3202 * 3203 * Returns 0 on success, negative on failure 3204 * 3205 * igb_probe initializes an adapter identified by a pci_dev structure. 3206 * The OS initialization, configuring of the adapter private structure, 3207 * and a hardware reset occur. 3208 **/ 3209 static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3210 { 3211 struct net_device *netdev; 3212 struct igb_adapter *adapter; 3213 struct e1000_hw *hw; 3214 u16 eeprom_data = 0; 3215 s32 ret_val; 3216 static int global_quad_port_a; /* global quad port a indication */ 3217 const struct e1000_info *ei = igb_info_tbl[ent->driver_data]; 3218 u8 part_str[E1000_PBANUM_LENGTH]; 3219 int err; 3220 3221 /* Catch broken hardware that put the wrong VF device ID in 3222 * the PCIe SR-IOV capability. 3223 */ 3224 if (pdev->is_virtfn) { 3225 WARN(1, KERN_ERR "%s (%x:%x) should not be a VF!\n", 3226 pci_name(pdev), pdev->vendor, pdev->device); 3227 return -EINVAL; 3228 } 3229 3230 err = pci_enable_device_mem(pdev); 3231 if (err) 3232 return err; 3233 3234 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3235 if (err) { 3236 dev_err(&pdev->dev, 3237 "No usable DMA configuration, aborting\n"); 3238 goto err_dma; 3239 } 3240 3241 err = pci_request_mem_regions(pdev, igb_driver_name); 3242 if (err) 3243 goto err_pci_reg; 3244 3245 pci_set_master(pdev); 3246 pci_save_state(pdev); 3247 3248 err = -ENOMEM; 3249 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter), 3250 IGB_MAX_TX_QUEUES); 3251 if (!netdev) 3252 goto err_alloc_etherdev; 3253 3254 SET_NETDEV_DEV(netdev, &pdev->dev); 3255 3256 pci_set_drvdata(pdev, netdev); 3257 adapter = netdev_priv(netdev); 3258 adapter->netdev = netdev; 3259 adapter->pdev = pdev; 3260 hw = &adapter->hw; 3261 hw->back = adapter; 3262 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 3263 3264 err = -EIO; 3265 adapter->io_addr = pci_iomap(pdev, 0, 0); 3266 if (!adapter->io_addr) 3267 goto err_ioremap; 3268 /* hw->hw_addr can be altered, we'll use adapter->io_addr for unmap */ 3269 hw->hw_addr = adapter->io_addr; 3270 3271 netdev->netdev_ops = &igb_netdev_ops; 3272 igb_set_ethtool_ops(netdev); 3273 netdev->watchdog_timeo = 5 * HZ; 3274 3275 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1); 3276 3277 netdev->mem_start = pci_resource_start(pdev, 0); 3278 netdev->mem_end = pci_resource_end(pdev, 0); 3279 3280 /* PCI config space info */ 3281 hw->vendor_id = pdev->vendor; 3282 hw->device_id = pdev->device; 3283 hw->revision_id = pdev->revision; 3284 hw->subsystem_vendor_id = pdev->subsystem_vendor; 3285 hw->subsystem_device_id = pdev->subsystem_device; 3286 3287 /* Copy the default MAC, PHY and NVM function pointers */ 3288 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 3289 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 3290 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops)); 3291 /* Initialize skew-specific constants */ 3292 err = ei->get_invariants(hw); 3293 if (err) 3294 goto err_sw_init; 3295 3296 /* setup the private structure */ 3297 err = igb_sw_init(adapter); 3298 if (err) 3299 goto err_sw_init; 3300 3301 igb_get_bus_info_pcie(hw); 3302 3303 hw->phy.autoneg_wait_to_complete = false; 3304 3305 /* Copper options */ 3306 if (hw->phy.media_type == e1000_media_type_copper) { 3307 hw->phy.mdix = AUTO_ALL_MODES; 3308 hw->phy.disable_polarity_correction = false; 3309 hw->phy.ms_type = e1000_ms_hw_default; 3310 } 3311 3312 if (igb_check_reset_block(hw)) 3313 dev_info(&pdev->dev, 3314 "PHY reset is blocked due to SOL/IDER session.\n"); 3315 3316 /* features is initialized to 0 in allocation, it might have bits 3317 * set by igb_sw_init so we should use an or instead of an 3318 * assignment. 3319 */ 3320 netdev->features |= NETIF_F_SG | 3321 NETIF_F_TSO | 3322 NETIF_F_TSO6 | 3323 NETIF_F_RXHASH | 3324 NETIF_F_RXCSUM | 3325 NETIF_F_HW_CSUM; 3326 3327 if (hw->mac.type >= e1000_82576) 3328 netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4; 3329 3330 if (hw->mac.type >= e1000_i350) 3331 netdev->features |= NETIF_F_HW_TC; 3332 3333 #define IGB_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 3334 NETIF_F_GSO_GRE_CSUM | \ 3335 NETIF_F_GSO_IPXIP4 | \ 3336 NETIF_F_GSO_IPXIP6 | \ 3337 NETIF_F_GSO_UDP_TUNNEL | \ 3338 NETIF_F_GSO_UDP_TUNNEL_CSUM) 3339 3340 netdev->gso_partial_features = IGB_GSO_PARTIAL_FEATURES; 3341 netdev->features |= NETIF_F_GSO_PARTIAL | IGB_GSO_PARTIAL_FEATURES; 3342 3343 /* copy netdev features into list of user selectable features */ 3344 netdev->hw_features |= netdev->features | 3345 NETIF_F_HW_VLAN_CTAG_RX | 3346 NETIF_F_HW_VLAN_CTAG_TX | 3347 NETIF_F_RXALL; 3348 3349 if (hw->mac.type >= e1000_i350) 3350 netdev->hw_features |= NETIF_F_NTUPLE; 3351 3352 netdev->features |= NETIF_F_HIGHDMA; 3353 3354 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; 3355 netdev->mpls_features |= NETIF_F_HW_CSUM; 3356 netdev->hw_enc_features |= netdev->vlan_features; 3357 3358 /* set this bit last since it cannot be part of vlan_features */ 3359 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | 3360 NETIF_F_HW_VLAN_CTAG_RX | 3361 NETIF_F_HW_VLAN_CTAG_TX; 3362 3363 netdev->priv_flags |= IFF_SUPP_NOFCS; 3364 3365 netdev->priv_flags |= IFF_UNICAST_FLT; 3366 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT; 3367 3368 /* MTU range: 68 - 9216 */ 3369 netdev->min_mtu = ETH_MIN_MTU; 3370 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 3371 3372 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw); 3373 3374 /* before reading the NVM, reset the controller to put the device in a 3375 * known good starting state 3376 */ 3377 hw->mac.ops.reset_hw(hw); 3378 3379 /* make sure the NVM is good , i211/i210 parts can have special NVM 3380 * that doesn't contain a checksum 3381 */ 3382 switch (hw->mac.type) { 3383 case e1000_i210: 3384 case e1000_i211: 3385 if (igb_get_flash_presence_i210(hw)) { 3386 if (hw->nvm.ops.validate(hw) < 0) { 3387 dev_err(&pdev->dev, 3388 "The NVM Checksum Is Not Valid\n"); 3389 err = -EIO; 3390 goto err_eeprom; 3391 } 3392 } 3393 break; 3394 default: 3395 if (hw->nvm.ops.validate(hw) < 0) { 3396 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 3397 err = -EIO; 3398 goto err_eeprom; 3399 } 3400 break; 3401 } 3402 3403 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 3404 /* copy the MAC address out of the NVM */ 3405 if (hw->mac.ops.read_mac_addr(hw)) 3406 dev_err(&pdev->dev, "NVM Read Error\n"); 3407 } 3408 3409 eth_hw_addr_set(netdev, hw->mac.addr); 3410 3411 if (!is_valid_ether_addr(netdev->dev_addr)) { 3412 dev_err(&pdev->dev, "Invalid MAC Address\n"); 3413 err = -EIO; 3414 goto err_eeprom; 3415 } 3416 3417 igb_set_default_mac_filter(adapter); 3418 3419 /* get firmware version for ethtool -i */ 3420 igb_set_fw_version(adapter); 3421 3422 /* configure RXPBSIZE and TXPBSIZE */ 3423 if (hw->mac.type == e1000_i210) { 3424 wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT); 3425 wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT); 3426 } 3427 3428 timer_setup(&adapter->watchdog_timer, igb_watchdog, 0); 3429 timer_setup(&adapter->phy_info_timer, igb_update_phy_info, 0); 3430 3431 INIT_WORK(&adapter->reset_task, igb_reset_task); 3432 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task); 3433 3434 /* Initialize link properties that are user-changeable */ 3435 adapter->fc_autoneg = true; 3436 hw->mac.autoneg = true; 3437 hw->phy.autoneg_advertised = 0x2f; 3438 3439 hw->fc.requested_mode = e1000_fc_default; 3440 hw->fc.current_mode = e1000_fc_default; 3441 3442 igb_validate_mdi_setting(hw); 3443 3444 /* By default, support wake on port A */ 3445 if (hw->bus.func == 0) 3446 adapter->flags |= IGB_FLAG_WOL_SUPPORTED; 3447 3448 /* Check the NVM for wake support on non-port A ports */ 3449 if (hw->mac.type >= e1000_82580) 3450 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A + 3451 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1, 3452 &eeprom_data); 3453 else if (hw->bus.func == 1) 3454 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 3455 3456 if (eeprom_data & IGB_EEPROM_APME) 3457 adapter->flags |= IGB_FLAG_WOL_SUPPORTED; 3458 3459 /* now that we have the eeprom settings, apply the special cases where 3460 * the eeprom may be wrong or the board simply won't support wake on 3461 * lan on a particular port 3462 */ 3463 switch (pdev->device) { 3464 case E1000_DEV_ID_82575GB_QUAD_COPPER: 3465 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED; 3466 break; 3467 case E1000_DEV_ID_82575EB_FIBER_SERDES: 3468 case E1000_DEV_ID_82576_FIBER: 3469 case E1000_DEV_ID_82576_SERDES: 3470 /* Wake events only supported on port A for dual fiber 3471 * regardless of eeprom setting 3472 */ 3473 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1) 3474 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED; 3475 break; 3476 case E1000_DEV_ID_82576_QUAD_COPPER: 3477 case E1000_DEV_ID_82576_QUAD_COPPER_ET2: 3478 /* if quad port adapter, disable WoL on all but port A */ 3479 if (global_quad_port_a != 0) 3480 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED; 3481 else 3482 adapter->flags |= IGB_FLAG_QUAD_PORT_A; 3483 /* Reset for multiple quad port adapters */ 3484 if (++global_quad_port_a == 4) 3485 global_quad_port_a = 0; 3486 break; 3487 default: 3488 /* If the device can't wake, don't set software support */ 3489 if (!device_can_wakeup(&adapter->pdev->dev)) 3490 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED; 3491 } 3492 3493 /* initialize the wol settings based on the eeprom settings */ 3494 if (adapter->flags & IGB_FLAG_WOL_SUPPORTED) 3495 adapter->wol |= E1000_WUFC_MAG; 3496 3497 /* Some vendors want WoL disabled by default, but still supported */ 3498 if ((hw->mac.type == e1000_i350) && 3499 (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) { 3500 adapter->flags |= IGB_FLAG_WOL_SUPPORTED; 3501 adapter->wol = 0; 3502 } 3503 3504 /* Some vendors want the ability to Use the EEPROM setting as 3505 * enable/disable only, and not for capability 3506 */ 3507 if (((hw->mac.type == e1000_i350) || 3508 (hw->mac.type == e1000_i354)) && 3509 (pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)) { 3510 adapter->flags |= IGB_FLAG_WOL_SUPPORTED; 3511 adapter->wol = 0; 3512 } 3513 if (hw->mac.type == e1000_i350) { 3514 if (((pdev->subsystem_device == 0x5001) || 3515 (pdev->subsystem_device == 0x5002)) && 3516 (hw->bus.func == 0)) { 3517 adapter->flags |= IGB_FLAG_WOL_SUPPORTED; 3518 adapter->wol = 0; 3519 } 3520 if (pdev->subsystem_device == 0x1F52) 3521 adapter->flags |= IGB_FLAG_WOL_SUPPORTED; 3522 } 3523 3524 device_set_wakeup_enable(&adapter->pdev->dev, 3525 adapter->flags & IGB_FLAG_WOL_SUPPORTED); 3526 3527 /* reset the hardware with the new settings */ 3528 igb_reset(adapter); 3529 3530 /* Init the I2C interface */ 3531 err = igb_init_i2c(adapter); 3532 if (err) { 3533 dev_err(&pdev->dev, "failed to init i2c interface\n"); 3534 goto err_eeprom; 3535 } 3536 3537 /* let the f/w know that the h/w is now under the control of the 3538 * driver. 3539 */ 3540 igb_get_hw_control(adapter); 3541 3542 strcpy(netdev->name, "eth%d"); 3543 err = register_netdev(netdev); 3544 if (err) 3545 goto err_register; 3546 3547 /* carrier off reporting is important to ethtool even BEFORE open */ 3548 netif_carrier_off(netdev); 3549 3550 #ifdef CONFIG_IGB_DCA 3551 if (dca_add_requester(&pdev->dev) == 0) { 3552 adapter->flags |= IGB_FLAG_DCA_ENABLED; 3553 dev_info(&pdev->dev, "DCA enabled\n"); 3554 igb_setup_dca(adapter); 3555 } 3556 3557 #endif 3558 #ifdef CONFIG_IGB_HWMON 3559 /* Initialize the thermal sensor on i350 devices. */ 3560 if (hw->mac.type == e1000_i350 && hw->bus.func == 0) { 3561 u16 ets_word; 3562 3563 /* Read the NVM to determine if this i350 device supports an 3564 * external thermal sensor. 3565 */ 3566 hw->nvm.ops.read(hw, NVM_ETS_CFG, 1, &ets_word); 3567 if (ets_word != 0x0000 && ets_word != 0xFFFF) 3568 adapter->ets = true; 3569 else 3570 adapter->ets = false; 3571 /* Only enable I2C bit banging if an external thermal 3572 * sensor is supported. 3573 */ 3574 if (adapter->ets) 3575 igb_set_i2c_bb(hw); 3576 hw->mac.ops.init_thermal_sensor_thresh(hw); 3577 if (igb_sysfs_init(adapter)) 3578 dev_err(&pdev->dev, 3579 "failed to allocate sysfs resources\n"); 3580 } else { 3581 adapter->ets = false; 3582 } 3583 #endif 3584 /* Check if Media Autosense is enabled */ 3585 adapter->ei = *ei; 3586 if (hw->dev_spec._82575.mas_capable) 3587 igb_init_mas(adapter); 3588 3589 /* do hw tstamp init after resetting */ 3590 igb_ptp_init(adapter); 3591 3592 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n"); 3593 /* print bus type/speed/width info, not applicable to i354 */ 3594 if (hw->mac.type != e1000_i354) { 3595 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n", 3596 netdev->name, 3597 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" : 3598 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" : 3599 "unknown"), 3600 ((hw->bus.width == e1000_bus_width_pcie_x4) ? 3601 "Width x4" : 3602 (hw->bus.width == e1000_bus_width_pcie_x2) ? 3603 "Width x2" : 3604 (hw->bus.width == e1000_bus_width_pcie_x1) ? 3605 "Width x1" : "unknown"), netdev->dev_addr); 3606 } 3607 3608 if ((hw->mac.type == e1000_82576 && 3609 rd32(E1000_EECD) & E1000_EECD_PRES) || 3610 (hw->mac.type >= e1000_i210 || 3611 igb_get_flash_presence_i210(hw))) { 3612 ret_val = igb_read_part_string(hw, part_str, 3613 E1000_PBANUM_LENGTH); 3614 } else { 3615 ret_val = -E1000_ERR_INVM_VALUE_NOT_FOUND; 3616 } 3617 3618 if (ret_val) 3619 strcpy(part_str, "Unknown"); 3620 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str); 3621 dev_info(&pdev->dev, 3622 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n", 3623 (adapter->flags & IGB_FLAG_HAS_MSIX) ? "MSI-X" : 3624 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy", 3625 adapter->num_rx_queues, adapter->num_tx_queues); 3626 if (hw->phy.media_type == e1000_media_type_copper) { 3627 switch (hw->mac.type) { 3628 case e1000_i350: 3629 case e1000_i210: 3630 case e1000_i211: 3631 /* Enable EEE for internal copper PHY devices */ 3632 err = igb_set_eee_i350(hw, true, true); 3633 if ((!err) && 3634 (!hw->dev_spec._82575.eee_disable)) { 3635 adapter->eee_advert = 3636 MDIO_EEE_100TX | MDIO_EEE_1000T; 3637 adapter->flags |= IGB_FLAG_EEE; 3638 } 3639 break; 3640 case e1000_i354: 3641 if ((rd32(E1000_CTRL_EXT) & 3642 E1000_CTRL_EXT_LINK_MODE_SGMII)) { 3643 err = igb_set_eee_i354(hw, true, true); 3644 if ((!err) && 3645 (!hw->dev_spec._82575.eee_disable)) { 3646 adapter->eee_advert = 3647 MDIO_EEE_100TX | MDIO_EEE_1000T; 3648 adapter->flags |= IGB_FLAG_EEE; 3649 } 3650 } 3651 break; 3652 default: 3653 break; 3654 } 3655 } 3656 3657 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); 3658 3659 pm_runtime_put_noidle(&pdev->dev); 3660 return 0; 3661 3662 err_register: 3663 igb_release_hw_control(adapter); 3664 memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap)); 3665 err_eeprom: 3666 if (!igb_check_reset_block(hw)) 3667 igb_reset_phy(hw); 3668 3669 if (hw->flash_address) 3670 iounmap(hw->flash_address); 3671 err_sw_init: 3672 kfree(adapter->mac_table); 3673 kfree(adapter->shadow_vfta); 3674 igb_clear_interrupt_scheme(adapter); 3675 #ifdef CONFIG_PCI_IOV 3676 igb_disable_sriov(pdev, false); 3677 #endif 3678 pci_iounmap(pdev, adapter->io_addr); 3679 err_ioremap: 3680 free_netdev(netdev); 3681 err_alloc_etherdev: 3682 pci_release_mem_regions(pdev); 3683 err_pci_reg: 3684 err_dma: 3685 pci_disable_device(pdev); 3686 return err; 3687 } 3688 3689 #ifdef CONFIG_PCI_IOV 3690 static int igb_sriov_reinit(struct pci_dev *dev) 3691 { 3692 struct net_device *netdev = pci_get_drvdata(dev); 3693 struct igb_adapter *adapter = netdev_priv(netdev); 3694 struct pci_dev *pdev = adapter->pdev; 3695 3696 rtnl_lock(); 3697 3698 if (netif_running(netdev)) 3699 igb_close(netdev); 3700 else 3701 igb_reset(adapter); 3702 3703 igb_clear_interrupt_scheme(adapter); 3704 3705 igb_init_queue_configuration(adapter); 3706 3707 if (igb_init_interrupt_scheme(adapter, true)) { 3708 rtnl_unlock(); 3709 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 3710 return -ENOMEM; 3711 } 3712 3713 if (netif_running(netdev)) 3714 igb_open(netdev); 3715 3716 rtnl_unlock(); 3717 3718 return 0; 3719 } 3720 3721 static int igb_disable_sriov(struct pci_dev *pdev, bool reinit) 3722 { 3723 struct net_device *netdev = pci_get_drvdata(pdev); 3724 struct igb_adapter *adapter = netdev_priv(netdev); 3725 struct e1000_hw *hw = &adapter->hw; 3726 unsigned long flags; 3727 3728 /* reclaim resources allocated to VFs */ 3729 if (adapter->vf_data) { 3730 /* disable iov and allow time for transactions to clear */ 3731 if (pci_vfs_assigned(pdev)) { 3732 dev_warn(&pdev->dev, 3733 "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n"); 3734 return -EPERM; 3735 } else { 3736 pci_disable_sriov(pdev); 3737 msleep(500); 3738 } 3739 spin_lock_irqsave(&adapter->vfs_lock, flags); 3740 kfree(adapter->vf_mac_list); 3741 adapter->vf_mac_list = NULL; 3742 kfree(adapter->vf_data); 3743 adapter->vf_data = NULL; 3744 adapter->vfs_allocated_count = 0; 3745 spin_unlock_irqrestore(&adapter->vfs_lock, flags); 3746 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ); 3747 wrfl(); 3748 msleep(100); 3749 dev_info(&pdev->dev, "IOV Disabled\n"); 3750 3751 /* Re-enable DMA Coalescing flag since IOV is turned off */ 3752 adapter->flags |= IGB_FLAG_DMAC; 3753 } 3754 3755 return reinit ? igb_sriov_reinit(pdev) : 0; 3756 } 3757 3758 static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs, bool reinit) 3759 { 3760 struct net_device *netdev = pci_get_drvdata(pdev); 3761 struct igb_adapter *adapter = netdev_priv(netdev); 3762 int old_vfs = pci_num_vf(pdev); 3763 struct vf_mac_filter *mac_list; 3764 int err = 0; 3765 int num_vf_mac_filters, i; 3766 3767 if (!(adapter->flags & IGB_FLAG_HAS_MSIX) || num_vfs > 7) { 3768 err = -EPERM; 3769 goto out; 3770 } 3771 if (!num_vfs) 3772 goto out; 3773 3774 if (old_vfs) { 3775 dev_info(&pdev->dev, "%d pre-allocated VFs found - override max_vfs setting of %d\n", 3776 old_vfs, max_vfs); 3777 adapter->vfs_allocated_count = old_vfs; 3778 } else 3779 adapter->vfs_allocated_count = num_vfs; 3780 3781 adapter->vf_data = kcalloc(adapter->vfs_allocated_count, 3782 sizeof(struct vf_data_storage), GFP_KERNEL); 3783 3784 /* if allocation failed then we do not support SR-IOV */ 3785 if (!adapter->vf_data) { 3786 adapter->vfs_allocated_count = 0; 3787 err = -ENOMEM; 3788 goto out; 3789 } 3790 3791 /* Due to the limited number of RAR entries calculate potential 3792 * number of MAC filters available for the VFs. Reserve entries 3793 * for PF default MAC, PF MAC filters and at least one RAR entry 3794 * for each VF for VF MAC. 3795 */ 3796 num_vf_mac_filters = adapter->hw.mac.rar_entry_count - 3797 (1 + IGB_PF_MAC_FILTERS_RESERVED + 3798 adapter->vfs_allocated_count); 3799 3800 adapter->vf_mac_list = kcalloc(num_vf_mac_filters, 3801 sizeof(struct vf_mac_filter), 3802 GFP_KERNEL); 3803 3804 mac_list = adapter->vf_mac_list; 3805 INIT_LIST_HEAD(&adapter->vf_macs.l); 3806 3807 if (adapter->vf_mac_list) { 3808 /* Initialize list of VF MAC filters */ 3809 for (i = 0; i < num_vf_mac_filters; i++) { 3810 mac_list->vf = -1; 3811 mac_list->free = true; 3812 list_add(&mac_list->l, &adapter->vf_macs.l); 3813 mac_list++; 3814 } 3815 } else { 3816 /* If we could not allocate memory for the VF MAC filters 3817 * we can continue without this feature but warn user. 3818 */ 3819 dev_err(&pdev->dev, 3820 "Unable to allocate memory for VF MAC filter list\n"); 3821 } 3822 3823 dev_info(&pdev->dev, "%d VFs allocated\n", 3824 adapter->vfs_allocated_count); 3825 for (i = 0; i < adapter->vfs_allocated_count; i++) 3826 igb_vf_configure(adapter, i); 3827 3828 /* DMA Coalescing is not supported in IOV mode. */ 3829 adapter->flags &= ~IGB_FLAG_DMAC; 3830 3831 if (reinit) { 3832 err = igb_sriov_reinit(pdev); 3833 if (err) 3834 goto err_out; 3835 } 3836 3837 /* only call pci_enable_sriov() if no VFs are allocated already */ 3838 if (!old_vfs) { 3839 err = pci_enable_sriov(pdev, adapter->vfs_allocated_count); 3840 if (err) 3841 goto err_out; 3842 } 3843 3844 goto out; 3845 3846 err_out: 3847 kfree(adapter->vf_mac_list); 3848 adapter->vf_mac_list = NULL; 3849 kfree(adapter->vf_data); 3850 adapter->vf_data = NULL; 3851 adapter->vfs_allocated_count = 0; 3852 out: 3853 return err; 3854 } 3855 3856 #endif 3857 /** 3858 * igb_remove_i2c - Cleanup I2C interface 3859 * @adapter: pointer to adapter structure 3860 **/ 3861 static void igb_remove_i2c(struct igb_adapter *adapter) 3862 { 3863 /* free the adapter bus structure */ 3864 i2c_del_adapter(&adapter->i2c_adap); 3865 } 3866 3867 /** 3868 * igb_remove - Device Removal Routine 3869 * @pdev: PCI device information struct 3870 * 3871 * igb_remove is called by the PCI subsystem to alert the driver 3872 * that it should release a PCI device. The could be caused by a 3873 * Hot-Plug event, or because the driver is going to be removed from 3874 * memory. 3875 **/ 3876 static void igb_remove(struct pci_dev *pdev) 3877 { 3878 struct net_device *netdev = pci_get_drvdata(pdev); 3879 struct igb_adapter *adapter = netdev_priv(netdev); 3880 struct e1000_hw *hw = &adapter->hw; 3881 3882 pm_runtime_get_noresume(&pdev->dev); 3883 #ifdef CONFIG_IGB_HWMON 3884 igb_sysfs_exit(adapter); 3885 #endif 3886 igb_remove_i2c(adapter); 3887 igb_ptp_stop(adapter); 3888 /* The watchdog timer may be rescheduled, so explicitly 3889 * disable watchdog from being rescheduled. 3890 */ 3891 set_bit(__IGB_DOWN, &adapter->state); 3892 del_timer_sync(&adapter->watchdog_timer); 3893 del_timer_sync(&adapter->phy_info_timer); 3894 3895 cancel_work_sync(&adapter->reset_task); 3896 cancel_work_sync(&adapter->watchdog_task); 3897 3898 #ifdef CONFIG_IGB_DCA 3899 if (adapter->flags & IGB_FLAG_DCA_ENABLED) { 3900 dev_info(&pdev->dev, "DCA disabled\n"); 3901 dca_remove_requester(&pdev->dev); 3902 adapter->flags &= ~IGB_FLAG_DCA_ENABLED; 3903 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE); 3904 } 3905 #endif 3906 3907 /* Release control of h/w to f/w. If f/w is AMT enabled, this 3908 * would have already happened in close and is redundant. 3909 */ 3910 igb_release_hw_control(adapter); 3911 3912 #ifdef CONFIG_PCI_IOV 3913 igb_disable_sriov(pdev, false); 3914 #endif 3915 3916 unregister_netdev(netdev); 3917 3918 igb_clear_interrupt_scheme(adapter); 3919 3920 pci_iounmap(pdev, adapter->io_addr); 3921 if (hw->flash_address) 3922 iounmap(hw->flash_address); 3923 pci_release_mem_regions(pdev); 3924 3925 kfree(adapter->mac_table); 3926 kfree(adapter->shadow_vfta); 3927 free_netdev(netdev); 3928 3929 pci_disable_device(pdev); 3930 } 3931 3932 /** 3933 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space 3934 * @adapter: board private structure to initialize 3935 * 3936 * This function initializes the vf specific data storage and then attempts to 3937 * allocate the VFs. The reason for ordering it this way is because it is much 3938 * mor expensive time wise to disable SR-IOV than it is to allocate and free 3939 * the memory for the VFs. 3940 **/ 3941 static void igb_probe_vfs(struct igb_adapter *adapter) 3942 { 3943 #ifdef CONFIG_PCI_IOV 3944 struct pci_dev *pdev = adapter->pdev; 3945 struct e1000_hw *hw = &adapter->hw; 3946 3947 /* Virtualization features not supported on i210 and 82580 family. */ 3948 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211) || 3949 (hw->mac.type == e1000_82580)) 3950 return; 3951 3952 /* Of the below we really only want the effect of getting 3953 * IGB_FLAG_HAS_MSIX set (if available), without which 3954 * igb_enable_sriov() has no effect. 3955 */ 3956 igb_set_interrupt_capability(adapter, true); 3957 igb_reset_interrupt_capability(adapter); 3958 3959 pci_sriov_set_totalvfs(pdev, 7); 3960 igb_enable_sriov(pdev, max_vfs, false); 3961 3962 #endif /* CONFIG_PCI_IOV */ 3963 } 3964 3965 unsigned int igb_get_max_rss_queues(struct igb_adapter *adapter) 3966 { 3967 struct e1000_hw *hw = &adapter->hw; 3968 unsigned int max_rss_queues; 3969 3970 /* Determine the maximum number of RSS queues supported. */ 3971 switch (hw->mac.type) { 3972 case e1000_i211: 3973 max_rss_queues = IGB_MAX_RX_QUEUES_I211; 3974 break; 3975 case e1000_82575: 3976 case e1000_i210: 3977 max_rss_queues = IGB_MAX_RX_QUEUES_82575; 3978 break; 3979 case e1000_i350: 3980 /* I350 cannot do RSS and SR-IOV at the same time */ 3981 if (!!adapter->vfs_allocated_count) { 3982 max_rss_queues = 1; 3983 break; 3984 } 3985 fallthrough; 3986 case e1000_82576: 3987 if (!!adapter->vfs_allocated_count) { 3988 max_rss_queues = 2; 3989 break; 3990 } 3991 fallthrough; 3992 case e1000_82580: 3993 case e1000_i354: 3994 default: 3995 max_rss_queues = IGB_MAX_RX_QUEUES; 3996 break; 3997 } 3998 3999 return max_rss_queues; 4000 } 4001 4002 static void igb_init_queue_configuration(struct igb_adapter *adapter) 4003 { 4004 u32 max_rss_queues; 4005 4006 max_rss_queues = igb_get_max_rss_queues(adapter); 4007 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 4008 4009 igb_set_flag_queue_pairs(adapter, max_rss_queues); 4010 } 4011 4012 void igb_set_flag_queue_pairs(struct igb_adapter *adapter, 4013 const u32 max_rss_queues) 4014 { 4015 struct e1000_hw *hw = &adapter->hw; 4016 4017 /* Determine if we need to pair queues. */ 4018 switch (hw->mac.type) { 4019 case e1000_82575: 4020 case e1000_i211: 4021 /* Device supports enough interrupts without queue pairing. */ 4022 break; 4023 case e1000_82576: 4024 case e1000_82580: 4025 case e1000_i350: 4026 case e1000_i354: 4027 case e1000_i210: 4028 default: 4029 /* If rss_queues > half of max_rss_queues, pair the queues in 4030 * order to conserve interrupts due to limited supply. 4031 */ 4032 if (adapter->rss_queues > (max_rss_queues / 2)) 4033 adapter->flags |= IGB_FLAG_QUEUE_PAIRS; 4034 else 4035 adapter->flags &= ~IGB_FLAG_QUEUE_PAIRS; 4036 break; 4037 } 4038 } 4039 4040 /** 4041 * igb_sw_init - Initialize general software structures (struct igb_adapter) 4042 * @adapter: board private structure to initialize 4043 * 4044 * igb_sw_init initializes the Adapter private data structure. 4045 * Fields are initialized based on PCI device information and 4046 * OS network device settings (MTU size). 4047 **/ 4048 static int igb_sw_init(struct igb_adapter *adapter) 4049 { 4050 struct e1000_hw *hw = &adapter->hw; 4051 struct net_device *netdev = adapter->netdev; 4052 struct pci_dev *pdev = adapter->pdev; 4053 4054 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 4055 4056 /* set default ring sizes */ 4057 adapter->tx_ring_count = IGB_DEFAULT_TXD; 4058 adapter->rx_ring_count = IGB_DEFAULT_RXD; 4059 4060 /* set default ITR values */ 4061 adapter->rx_itr_setting = IGB_DEFAULT_ITR; 4062 adapter->tx_itr_setting = IGB_DEFAULT_ITR; 4063 4064 /* set default work limits */ 4065 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK; 4066 4067 adapter->max_frame_size = netdev->mtu + IGB_ETH_PKT_HDR_PAD; 4068 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 4069 4070 spin_lock_init(&adapter->nfc_lock); 4071 spin_lock_init(&adapter->stats64_lock); 4072 4073 /* init spinlock to avoid concurrency of VF resources */ 4074 spin_lock_init(&adapter->vfs_lock); 4075 #ifdef CONFIG_PCI_IOV 4076 switch (hw->mac.type) { 4077 case e1000_82576: 4078 case e1000_i350: 4079 if (max_vfs > 7) { 4080 dev_warn(&pdev->dev, 4081 "Maximum of 7 VFs per PF, using max\n"); 4082 max_vfs = adapter->vfs_allocated_count = 7; 4083 } else 4084 adapter->vfs_allocated_count = max_vfs; 4085 if (adapter->vfs_allocated_count) 4086 dev_warn(&pdev->dev, 4087 "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n"); 4088 break; 4089 default: 4090 break; 4091 } 4092 #endif /* CONFIG_PCI_IOV */ 4093 4094 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 4095 adapter->flags |= IGB_FLAG_HAS_MSIX; 4096 4097 adapter->mac_table = kcalloc(hw->mac.rar_entry_count, 4098 sizeof(struct igb_mac_addr), 4099 GFP_KERNEL); 4100 if (!adapter->mac_table) 4101 return -ENOMEM; 4102 4103 igb_probe_vfs(adapter); 4104 4105 igb_init_queue_configuration(adapter); 4106 4107 /* Setup and initialize a copy of the hw vlan table array */ 4108 adapter->shadow_vfta = kcalloc(E1000_VLAN_FILTER_TBL_SIZE, sizeof(u32), 4109 GFP_KERNEL); 4110 if (!adapter->shadow_vfta) 4111 return -ENOMEM; 4112 4113 /* This call may decrease the number of queues */ 4114 if (igb_init_interrupt_scheme(adapter, true)) { 4115 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 4116 return -ENOMEM; 4117 } 4118 4119 /* Explicitly disable IRQ since the NIC can be in any state. */ 4120 igb_irq_disable(adapter); 4121 4122 if (hw->mac.type >= e1000_i350) 4123 adapter->flags &= ~IGB_FLAG_DMAC; 4124 4125 set_bit(__IGB_DOWN, &adapter->state); 4126 return 0; 4127 } 4128 4129 /** 4130 * __igb_open - Called when a network interface is made active 4131 * @netdev: network interface device structure 4132 * @resuming: indicates whether we are in a resume call 4133 * 4134 * Returns 0 on success, negative value on failure 4135 * 4136 * The open entry point is called when a network interface is made 4137 * active by the system (IFF_UP). At this point all resources needed 4138 * for transmit and receive operations are allocated, the interrupt 4139 * handler is registered with the OS, the watchdog timer is started, 4140 * and the stack is notified that the interface is ready. 4141 **/ 4142 static int __igb_open(struct net_device *netdev, bool resuming) 4143 { 4144 struct igb_adapter *adapter = netdev_priv(netdev); 4145 struct e1000_hw *hw = &adapter->hw; 4146 struct pci_dev *pdev = adapter->pdev; 4147 int err; 4148 int i; 4149 4150 /* disallow open during test */ 4151 if (test_bit(__IGB_TESTING, &adapter->state)) { 4152 WARN_ON(resuming); 4153 return -EBUSY; 4154 } 4155 4156 if (!resuming) 4157 pm_runtime_get_sync(&pdev->dev); 4158 4159 netif_carrier_off(netdev); 4160 4161 /* allocate transmit descriptors */ 4162 err = igb_setup_all_tx_resources(adapter); 4163 if (err) 4164 goto err_setup_tx; 4165 4166 /* allocate receive descriptors */ 4167 err = igb_setup_all_rx_resources(adapter); 4168 if (err) 4169 goto err_setup_rx; 4170 4171 igb_power_up_link(adapter); 4172 4173 /* before we allocate an interrupt, we must be ready to handle it. 4174 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt 4175 * as soon as we call pci_request_irq, so we have to setup our 4176 * clean_rx handler before we do so. 4177 */ 4178 igb_configure(adapter); 4179 4180 err = igb_request_irq(adapter); 4181 if (err) 4182 goto err_req_irq; 4183 4184 /* Notify the stack of the actual queue counts. */ 4185 err = netif_set_real_num_tx_queues(adapter->netdev, 4186 adapter->num_tx_queues); 4187 if (err) 4188 goto err_set_queues; 4189 4190 err = netif_set_real_num_rx_queues(adapter->netdev, 4191 adapter->num_rx_queues); 4192 if (err) 4193 goto err_set_queues; 4194 4195 /* From here on the code is the same as igb_up() */ 4196 clear_bit(__IGB_DOWN, &adapter->state); 4197 4198 for (i = 0; i < adapter->num_q_vectors; i++) 4199 napi_enable(&(adapter->q_vector[i]->napi)); 4200 4201 /* Clear any pending interrupts. */ 4202 rd32(E1000_TSICR); 4203 rd32(E1000_ICR); 4204 4205 igb_irq_enable(adapter); 4206 4207 /* notify VFs that reset has been completed */ 4208 if (adapter->vfs_allocated_count) { 4209 u32 reg_data = rd32(E1000_CTRL_EXT); 4210 4211 reg_data |= E1000_CTRL_EXT_PFRSTD; 4212 wr32(E1000_CTRL_EXT, reg_data); 4213 } 4214 4215 netif_tx_start_all_queues(netdev); 4216 4217 if (!resuming) 4218 pm_runtime_put(&pdev->dev); 4219 4220 /* start the watchdog. */ 4221 hw->mac.get_link_status = 1; 4222 schedule_work(&adapter->watchdog_task); 4223 4224 return 0; 4225 4226 err_set_queues: 4227 igb_free_irq(adapter); 4228 err_req_irq: 4229 igb_release_hw_control(adapter); 4230 igb_power_down_link(adapter); 4231 igb_free_all_rx_resources(adapter); 4232 err_setup_rx: 4233 igb_free_all_tx_resources(adapter); 4234 err_setup_tx: 4235 igb_reset(adapter); 4236 if (!resuming) 4237 pm_runtime_put(&pdev->dev); 4238 4239 return err; 4240 } 4241 4242 int igb_open(struct net_device *netdev) 4243 { 4244 return __igb_open(netdev, false); 4245 } 4246 4247 /** 4248 * __igb_close - Disables a network interface 4249 * @netdev: network interface device structure 4250 * @suspending: indicates we are in a suspend call 4251 * 4252 * Returns 0, this is not allowed to fail 4253 * 4254 * The close entry point is called when an interface is de-activated 4255 * by the OS. The hardware is still under the driver's control, but 4256 * needs to be disabled. A global MAC reset is issued to stop the 4257 * hardware, and all transmit and receive resources are freed. 4258 **/ 4259 static int __igb_close(struct net_device *netdev, bool suspending) 4260 { 4261 struct igb_adapter *adapter = netdev_priv(netdev); 4262 struct pci_dev *pdev = adapter->pdev; 4263 4264 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state)); 4265 4266 if (!suspending) 4267 pm_runtime_get_sync(&pdev->dev); 4268 4269 igb_down(adapter); 4270 igb_free_irq(adapter); 4271 4272 igb_free_all_tx_resources(adapter); 4273 igb_free_all_rx_resources(adapter); 4274 4275 if (!suspending) 4276 pm_runtime_put_sync(&pdev->dev); 4277 return 0; 4278 } 4279 4280 int igb_close(struct net_device *netdev) 4281 { 4282 if (netif_device_present(netdev) || netdev->dismantle) 4283 return __igb_close(netdev, false); 4284 return 0; 4285 } 4286 4287 /** 4288 * igb_setup_tx_resources - allocate Tx resources (Descriptors) 4289 * @tx_ring: tx descriptor ring (for a specific queue) to setup 4290 * 4291 * Return 0 on success, negative on failure 4292 **/ 4293 int igb_setup_tx_resources(struct igb_ring *tx_ring) 4294 { 4295 struct device *dev = tx_ring->dev; 4296 int size; 4297 4298 size = sizeof(struct igb_tx_buffer) * tx_ring->count; 4299 4300 tx_ring->tx_buffer_info = vmalloc(size); 4301 if (!tx_ring->tx_buffer_info) 4302 goto err; 4303 4304 /* round up to nearest 4K */ 4305 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc); 4306 tx_ring->size = ALIGN(tx_ring->size, 4096); 4307 4308 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 4309 &tx_ring->dma, GFP_KERNEL); 4310 if (!tx_ring->desc) 4311 goto err; 4312 4313 tx_ring->next_to_use = 0; 4314 tx_ring->next_to_clean = 0; 4315 4316 return 0; 4317 4318 err: 4319 vfree(tx_ring->tx_buffer_info); 4320 tx_ring->tx_buffer_info = NULL; 4321 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n"); 4322 return -ENOMEM; 4323 } 4324 4325 /** 4326 * igb_setup_all_tx_resources - wrapper to allocate Tx resources 4327 * (Descriptors) for all queues 4328 * @adapter: board private structure 4329 * 4330 * Return 0 on success, negative on failure 4331 **/ 4332 static int igb_setup_all_tx_resources(struct igb_adapter *adapter) 4333 { 4334 struct pci_dev *pdev = adapter->pdev; 4335 int i, err = 0; 4336 4337 for (i = 0; i < adapter->num_tx_queues; i++) { 4338 err = igb_setup_tx_resources(adapter->tx_ring[i]); 4339 if (err) { 4340 dev_err(&pdev->dev, 4341 "Allocation for Tx Queue %u failed\n", i); 4342 for (i--; i >= 0; i--) 4343 igb_free_tx_resources(adapter->tx_ring[i]); 4344 break; 4345 } 4346 } 4347 4348 return err; 4349 } 4350 4351 /** 4352 * igb_setup_tctl - configure the transmit control registers 4353 * @adapter: Board private structure 4354 **/ 4355 void igb_setup_tctl(struct igb_adapter *adapter) 4356 { 4357 struct e1000_hw *hw = &adapter->hw; 4358 u32 tctl; 4359 4360 /* disable queue 0 which is enabled by default on 82575 and 82576 */ 4361 wr32(E1000_TXDCTL(0), 0); 4362 4363 /* Program the Transmit Control Register */ 4364 tctl = rd32(E1000_TCTL); 4365 tctl &= ~E1000_TCTL_CT; 4366 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | 4367 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 4368 4369 igb_config_collision_dist(hw); 4370 4371 /* Enable transmits */ 4372 tctl |= E1000_TCTL_EN; 4373 4374 wr32(E1000_TCTL, tctl); 4375 } 4376 4377 /** 4378 * igb_configure_tx_ring - Configure transmit ring after Reset 4379 * @adapter: board private structure 4380 * @ring: tx ring to configure 4381 * 4382 * Configure a transmit ring after a reset. 4383 **/ 4384 void igb_configure_tx_ring(struct igb_adapter *adapter, 4385 struct igb_ring *ring) 4386 { 4387 struct e1000_hw *hw = &adapter->hw; 4388 u32 txdctl = 0; 4389 u64 tdba = ring->dma; 4390 int reg_idx = ring->reg_idx; 4391 4392 wr32(E1000_TDLEN(reg_idx), 4393 ring->count * sizeof(union e1000_adv_tx_desc)); 4394 wr32(E1000_TDBAL(reg_idx), 4395 tdba & 0x00000000ffffffffULL); 4396 wr32(E1000_TDBAH(reg_idx), tdba >> 32); 4397 4398 ring->tail = adapter->io_addr + E1000_TDT(reg_idx); 4399 wr32(E1000_TDH(reg_idx), 0); 4400 writel(0, ring->tail); 4401 4402 txdctl |= IGB_TX_PTHRESH; 4403 txdctl |= IGB_TX_HTHRESH << 8; 4404 txdctl |= IGB_TX_WTHRESH << 16; 4405 4406 /* reinitialize tx_buffer_info */ 4407 memset(ring->tx_buffer_info, 0, 4408 sizeof(struct igb_tx_buffer) * ring->count); 4409 4410 txdctl |= E1000_TXDCTL_QUEUE_ENABLE; 4411 wr32(E1000_TXDCTL(reg_idx), txdctl); 4412 } 4413 4414 /** 4415 * igb_configure_tx - Configure transmit Unit after Reset 4416 * @adapter: board private structure 4417 * 4418 * Configure the Tx unit of the MAC after a reset. 4419 **/ 4420 static void igb_configure_tx(struct igb_adapter *adapter) 4421 { 4422 struct e1000_hw *hw = &adapter->hw; 4423 int i; 4424 4425 /* disable the queues */ 4426 for (i = 0; i < adapter->num_tx_queues; i++) 4427 wr32(E1000_TXDCTL(adapter->tx_ring[i]->reg_idx), 0); 4428 4429 wrfl(); 4430 usleep_range(10000, 20000); 4431 4432 for (i = 0; i < adapter->num_tx_queues; i++) 4433 igb_configure_tx_ring(adapter, adapter->tx_ring[i]); 4434 } 4435 4436 /** 4437 * igb_setup_rx_resources - allocate Rx resources (Descriptors) 4438 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 4439 * 4440 * Returns 0 on success, negative on failure 4441 **/ 4442 int igb_setup_rx_resources(struct igb_ring *rx_ring) 4443 { 4444 struct igb_adapter *adapter = netdev_priv(rx_ring->netdev); 4445 struct device *dev = rx_ring->dev; 4446 int size, res; 4447 4448 /* XDP RX-queue info */ 4449 if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) 4450 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 4451 res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, 4452 rx_ring->queue_index, 0); 4453 if (res < 0) { 4454 dev_err(dev, "Failed to register xdp_rxq index %u\n", 4455 rx_ring->queue_index); 4456 return res; 4457 } 4458 4459 size = sizeof(struct igb_rx_buffer) * rx_ring->count; 4460 4461 rx_ring->rx_buffer_info = vmalloc(size); 4462 if (!rx_ring->rx_buffer_info) 4463 goto err; 4464 4465 /* Round up to nearest 4K */ 4466 rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc); 4467 rx_ring->size = ALIGN(rx_ring->size, 4096); 4468 4469 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 4470 &rx_ring->dma, GFP_KERNEL); 4471 if (!rx_ring->desc) 4472 goto err; 4473 4474 rx_ring->next_to_alloc = 0; 4475 rx_ring->next_to_clean = 0; 4476 rx_ring->next_to_use = 0; 4477 4478 rx_ring->xdp_prog = adapter->xdp_prog; 4479 4480 return 0; 4481 4482 err: 4483 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 4484 vfree(rx_ring->rx_buffer_info); 4485 rx_ring->rx_buffer_info = NULL; 4486 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); 4487 return -ENOMEM; 4488 } 4489 4490 /** 4491 * igb_setup_all_rx_resources - wrapper to allocate Rx resources 4492 * (Descriptors) for all queues 4493 * @adapter: board private structure 4494 * 4495 * Return 0 on success, negative on failure 4496 **/ 4497 static int igb_setup_all_rx_resources(struct igb_adapter *adapter) 4498 { 4499 struct pci_dev *pdev = adapter->pdev; 4500 int i, err = 0; 4501 4502 for (i = 0; i < adapter->num_rx_queues; i++) { 4503 err = igb_setup_rx_resources(adapter->rx_ring[i]); 4504 if (err) { 4505 dev_err(&pdev->dev, 4506 "Allocation for Rx Queue %u failed\n", i); 4507 for (i--; i >= 0; i--) 4508 igb_free_rx_resources(adapter->rx_ring[i]); 4509 break; 4510 } 4511 } 4512 4513 return err; 4514 } 4515 4516 /** 4517 * igb_setup_mrqc - configure the multiple receive queue control registers 4518 * @adapter: Board private structure 4519 **/ 4520 static void igb_setup_mrqc(struct igb_adapter *adapter) 4521 { 4522 struct e1000_hw *hw = &adapter->hw; 4523 u32 mrqc, rxcsum; 4524 u32 j, num_rx_queues; 4525 u32 rss_key[10]; 4526 4527 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 4528 for (j = 0; j < 10; j++) 4529 wr32(E1000_RSSRK(j), rss_key[j]); 4530 4531 num_rx_queues = adapter->rss_queues; 4532 4533 switch (hw->mac.type) { 4534 case e1000_82576: 4535 /* 82576 supports 2 RSS queues for SR-IOV */ 4536 if (adapter->vfs_allocated_count) 4537 num_rx_queues = 2; 4538 break; 4539 default: 4540 break; 4541 } 4542 4543 if (adapter->rss_indir_tbl_init != num_rx_queues) { 4544 for (j = 0; j < IGB_RETA_SIZE; j++) 4545 adapter->rss_indir_tbl[j] = 4546 (j * num_rx_queues) / IGB_RETA_SIZE; 4547 adapter->rss_indir_tbl_init = num_rx_queues; 4548 } 4549 igb_write_rss_indir_tbl(adapter); 4550 4551 /* Disable raw packet checksumming so that RSS hash is placed in 4552 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 4553 * offloads as they are enabled by default 4554 */ 4555 rxcsum = rd32(E1000_RXCSUM); 4556 rxcsum |= E1000_RXCSUM_PCSD; 4557 4558 if (adapter->hw.mac.type >= e1000_82576) 4559 /* Enable Receive Checksum Offload for SCTP */ 4560 rxcsum |= E1000_RXCSUM_CRCOFL; 4561 4562 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 4563 wr32(E1000_RXCSUM, rxcsum); 4564 4565 /* Generate RSS hash based on packet types, TCP/UDP 4566 * port numbers and/or IPv4/v6 src and dst addresses 4567 */ 4568 mrqc = E1000_MRQC_RSS_FIELD_IPV4 | 4569 E1000_MRQC_RSS_FIELD_IPV4_TCP | 4570 E1000_MRQC_RSS_FIELD_IPV6 | 4571 E1000_MRQC_RSS_FIELD_IPV6_TCP | 4572 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX; 4573 4574 if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP) 4575 mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP; 4576 if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP) 4577 mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP; 4578 4579 /* If VMDq is enabled then we set the appropriate mode for that, else 4580 * we default to RSS so that an RSS hash is calculated per packet even 4581 * if we are only using one queue 4582 */ 4583 if (adapter->vfs_allocated_count) { 4584 if (hw->mac.type > e1000_82575) { 4585 /* Set the default pool for the PF's first queue */ 4586 u32 vtctl = rd32(E1000_VT_CTL); 4587 4588 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK | 4589 E1000_VT_CTL_DISABLE_DEF_POOL); 4590 vtctl |= adapter->vfs_allocated_count << 4591 E1000_VT_CTL_DEFAULT_POOL_SHIFT; 4592 wr32(E1000_VT_CTL, vtctl); 4593 } 4594 if (adapter->rss_queues > 1) 4595 mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_MQ; 4596 else 4597 mrqc |= E1000_MRQC_ENABLE_VMDQ; 4598 } else { 4599 mrqc |= E1000_MRQC_ENABLE_RSS_MQ; 4600 } 4601 igb_vmm_control(adapter); 4602 4603 wr32(E1000_MRQC, mrqc); 4604 } 4605 4606 /** 4607 * igb_setup_rctl - configure the receive control registers 4608 * @adapter: Board private structure 4609 **/ 4610 void igb_setup_rctl(struct igb_adapter *adapter) 4611 { 4612 struct e1000_hw *hw = &adapter->hw; 4613 u32 rctl; 4614 4615 rctl = rd32(E1000_RCTL); 4616 4617 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4618 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC); 4619 4620 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF | 4621 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4622 4623 /* enable stripping of CRC. It's unlikely this will break BMC 4624 * redirection as it did with e1000. Newer features require 4625 * that the HW strips the CRC. 4626 */ 4627 rctl |= E1000_RCTL_SECRC; 4628 4629 /* disable store bad packets and clear size bits. */ 4630 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256); 4631 4632 /* enable LPE to allow for reception of jumbo frames */ 4633 rctl |= E1000_RCTL_LPE; 4634 4635 /* disable queue 0 to prevent tail write w/o re-config */ 4636 wr32(E1000_RXDCTL(0), 0); 4637 4638 /* Attention!!! For SR-IOV PF driver operations you must enable 4639 * queue drop for all VF and PF queues to prevent head of line blocking 4640 * if an un-trusted VF does not provide descriptors to hardware. 4641 */ 4642 if (adapter->vfs_allocated_count) { 4643 /* set all queue drop enable bits */ 4644 wr32(E1000_QDE, ALL_QUEUES); 4645 } 4646 4647 /* This is useful for sniffing bad packets. */ 4648 if (adapter->netdev->features & NETIF_F_RXALL) { 4649 /* UPE and MPE will be handled by normal PROMISC logic 4650 * in e1000e_set_rx_mode 4651 */ 4652 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */ 4653 E1000_RCTL_BAM | /* RX All Bcast Pkts */ 4654 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 4655 4656 rctl &= ~(E1000_RCTL_DPF | /* Allow filtered pause */ 4657 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */ 4658 /* Do not mess with E1000_CTRL_VME, it affects transmit as well, 4659 * and that breaks VLANs. 4660 */ 4661 } 4662 4663 wr32(E1000_RCTL, rctl); 4664 } 4665 4666 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size, 4667 int vfn) 4668 { 4669 struct e1000_hw *hw = &adapter->hw; 4670 u32 vmolr; 4671 4672 if (size > MAX_JUMBO_FRAME_SIZE) 4673 size = MAX_JUMBO_FRAME_SIZE; 4674 4675 vmolr = rd32(E1000_VMOLR(vfn)); 4676 vmolr &= ~E1000_VMOLR_RLPML_MASK; 4677 vmolr |= size | E1000_VMOLR_LPE; 4678 wr32(E1000_VMOLR(vfn), vmolr); 4679 4680 return 0; 4681 } 4682 4683 static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter, 4684 int vfn, bool enable) 4685 { 4686 struct e1000_hw *hw = &adapter->hw; 4687 u32 val, reg; 4688 4689 if (hw->mac.type < e1000_82576) 4690 return; 4691 4692 if (hw->mac.type == e1000_i350) 4693 reg = E1000_DVMOLR(vfn); 4694 else 4695 reg = E1000_VMOLR(vfn); 4696 4697 val = rd32(reg); 4698 if (enable) 4699 val |= E1000_VMOLR_STRVLAN; 4700 else 4701 val &= ~(E1000_VMOLR_STRVLAN); 4702 wr32(reg, val); 4703 } 4704 4705 static inline void igb_set_vmolr(struct igb_adapter *adapter, 4706 int vfn, bool aupe) 4707 { 4708 struct e1000_hw *hw = &adapter->hw; 4709 u32 vmolr; 4710 4711 /* This register exists only on 82576 and newer so if we are older then 4712 * we should exit and do nothing 4713 */ 4714 if (hw->mac.type < e1000_82576) 4715 return; 4716 4717 vmolr = rd32(E1000_VMOLR(vfn)); 4718 if (aupe) 4719 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */ 4720 else 4721 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */ 4722 4723 /* clear all bits that might not be set */ 4724 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE); 4725 4726 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count) 4727 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */ 4728 /* for VMDq only allow the VFs and pool 0 to accept broadcast and 4729 * multicast packets 4730 */ 4731 if (vfn <= adapter->vfs_allocated_count) 4732 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */ 4733 4734 wr32(E1000_VMOLR(vfn), vmolr); 4735 } 4736 4737 /** 4738 * igb_setup_srrctl - configure the split and replication receive control 4739 * registers 4740 * @adapter: Board private structure 4741 * @ring: receive ring to be configured 4742 **/ 4743 void igb_setup_srrctl(struct igb_adapter *adapter, struct igb_ring *ring) 4744 { 4745 struct e1000_hw *hw = &adapter->hw; 4746 int reg_idx = ring->reg_idx; 4747 u32 srrctl = 0; 4748 4749 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; 4750 if (ring_uses_large_buffer(ring)) 4751 srrctl |= IGB_RXBUFFER_3072 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 4752 else 4753 srrctl |= IGB_RXBUFFER_2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 4754 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; 4755 if (hw->mac.type >= e1000_82580) 4756 srrctl |= E1000_SRRCTL_TIMESTAMP; 4757 /* Only set Drop Enable if VFs allocated, or we are supporting multiple 4758 * queues and rx flow control is disabled 4759 */ 4760 if (adapter->vfs_allocated_count || 4761 (!(hw->fc.current_mode & e1000_fc_rx_pause) && 4762 adapter->num_rx_queues > 1)) 4763 srrctl |= E1000_SRRCTL_DROP_EN; 4764 4765 wr32(E1000_SRRCTL(reg_idx), srrctl); 4766 } 4767 4768 /** 4769 * igb_configure_rx_ring - Configure a receive ring after Reset 4770 * @adapter: board private structure 4771 * @ring: receive ring to be configured 4772 * 4773 * Configure the Rx unit of the MAC after a reset. 4774 **/ 4775 void igb_configure_rx_ring(struct igb_adapter *adapter, 4776 struct igb_ring *ring) 4777 { 4778 struct e1000_hw *hw = &adapter->hw; 4779 union e1000_adv_rx_desc *rx_desc; 4780 u64 rdba = ring->dma; 4781 int reg_idx = ring->reg_idx; 4782 u32 rxdctl = 0; 4783 4784 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); 4785 WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 4786 MEM_TYPE_PAGE_SHARED, NULL)); 4787 4788 /* disable the queue */ 4789 wr32(E1000_RXDCTL(reg_idx), 0); 4790 4791 /* Set DMA base address registers */ 4792 wr32(E1000_RDBAL(reg_idx), 4793 rdba & 0x00000000ffffffffULL); 4794 wr32(E1000_RDBAH(reg_idx), rdba >> 32); 4795 wr32(E1000_RDLEN(reg_idx), 4796 ring->count * sizeof(union e1000_adv_rx_desc)); 4797 4798 /* initialize head and tail */ 4799 ring->tail = adapter->io_addr + E1000_RDT(reg_idx); 4800 wr32(E1000_RDH(reg_idx), 0); 4801 writel(0, ring->tail); 4802 4803 /* set descriptor configuration */ 4804 igb_setup_srrctl(adapter, ring); 4805 4806 /* set filtering for VMDQ pools */ 4807 igb_set_vmolr(adapter, reg_idx & 0x7, true); 4808 4809 rxdctl |= IGB_RX_PTHRESH; 4810 rxdctl |= IGB_RX_HTHRESH << 8; 4811 rxdctl |= IGB_RX_WTHRESH << 16; 4812 4813 /* initialize rx_buffer_info */ 4814 memset(ring->rx_buffer_info, 0, 4815 sizeof(struct igb_rx_buffer) * ring->count); 4816 4817 /* initialize Rx descriptor 0 */ 4818 rx_desc = IGB_RX_DESC(ring, 0); 4819 rx_desc->wb.upper.length = 0; 4820 4821 /* enable receive descriptor fetching */ 4822 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 4823 wr32(E1000_RXDCTL(reg_idx), rxdctl); 4824 } 4825 4826 static void igb_set_rx_buffer_len(struct igb_adapter *adapter, 4827 struct igb_ring *rx_ring) 4828 { 4829 #if (PAGE_SIZE < 8192) 4830 struct e1000_hw *hw = &adapter->hw; 4831 #endif 4832 4833 /* set build_skb and buffer size flags */ 4834 clear_ring_build_skb_enabled(rx_ring); 4835 clear_ring_uses_large_buffer(rx_ring); 4836 4837 if (adapter->flags & IGB_FLAG_RX_LEGACY) 4838 return; 4839 4840 set_ring_build_skb_enabled(rx_ring); 4841 4842 #if (PAGE_SIZE < 8192) 4843 if (adapter->max_frame_size > IGB_MAX_FRAME_BUILD_SKB || 4844 IGB_2K_TOO_SMALL_WITH_PADDING || 4845 rd32(E1000_RCTL) & E1000_RCTL_SBP) 4846 set_ring_uses_large_buffer(rx_ring); 4847 #endif 4848 } 4849 4850 /** 4851 * igb_configure_rx - Configure receive Unit after Reset 4852 * @adapter: board private structure 4853 * 4854 * Configure the Rx unit of the MAC after a reset. 4855 **/ 4856 static void igb_configure_rx(struct igb_adapter *adapter) 4857 { 4858 int i; 4859 4860 /* set the correct pool for the PF default MAC address in entry 0 */ 4861 igb_set_default_mac_filter(adapter); 4862 4863 /* Setup the HW Rx Head and Tail Descriptor Pointers and 4864 * the Base and Length of the Rx Descriptor Ring 4865 */ 4866 for (i = 0; i < adapter->num_rx_queues; i++) { 4867 struct igb_ring *rx_ring = adapter->rx_ring[i]; 4868 4869 igb_set_rx_buffer_len(adapter, rx_ring); 4870 igb_configure_rx_ring(adapter, rx_ring); 4871 } 4872 } 4873 4874 /** 4875 * igb_free_tx_resources - Free Tx Resources per Queue 4876 * @tx_ring: Tx descriptor ring for a specific queue 4877 * 4878 * Free all transmit software resources 4879 **/ 4880 void igb_free_tx_resources(struct igb_ring *tx_ring) 4881 { 4882 igb_clean_tx_ring(tx_ring); 4883 4884 vfree(tx_ring->tx_buffer_info); 4885 tx_ring->tx_buffer_info = NULL; 4886 4887 /* if not set, then don't free */ 4888 if (!tx_ring->desc) 4889 return; 4890 4891 dma_free_coherent(tx_ring->dev, tx_ring->size, 4892 tx_ring->desc, tx_ring->dma); 4893 4894 tx_ring->desc = NULL; 4895 } 4896 4897 /** 4898 * igb_free_all_tx_resources - Free Tx Resources for All Queues 4899 * @adapter: board private structure 4900 * 4901 * Free all transmit software resources 4902 **/ 4903 static void igb_free_all_tx_resources(struct igb_adapter *adapter) 4904 { 4905 int i; 4906 4907 for (i = 0; i < adapter->num_tx_queues; i++) 4908 if (adapter->tx_ring[i]) 4909 igb_free_tx_resources(adapter->tx_ring[i]); 4910 } 4911 4912 /** 4913 * igb_clean_tx_ring - Free Tx Buffers 4914 * @tx_ring: ring to be cleaned 4915 **/ 4916 static void igb_clean_tx_ring(struct igb_ring *tx_ring) 4917 { 4918 u16 i = tx_ring->next_to_clean; 4919 struct igb_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 4920 4921 while (i != tx_ring->next_to_use) { 4922 union e1000_adv_tx_desc *eop_desc, *tx_desc; 4923 4924 /* Free all the Tx ring sk_buffs or xdp frames */ 4925 if (tx_buffer->type == IGB_TYPE_SKB) 4926 dev_kfree_skb_any(tx_buffer->skb); 4927 else 4928 xdp_return_frame(tx_buffer->xdpf); 4929 4930 /* unmap skb header data */ 4931 dma_unmap_single(tx_ring->dev, 4932 dma_unmap_addr(tx_buffer, dma), 4933 dma_unmap_len(tx_buffer, len), 4934 DMA_TO_DEVICE); 4935 4936 /* check for eop_desc to determine the end of the packet */ 4937 eop_desc = tx_buffer->next_to_watch; 4938 tx_desc = IGB_TX_DESC(tx_ring, i); 4939 4940 /* unmap remaining buffers */ 4941 while (tx_desc != eop_desc) { 4942 tx_buffer++; 4943 tx_desc++; 4944 i++; 4945 if (unlikely(i == tx_ring->count)) { 4946 i = 0; 4947 tx_buffer = tx_ring->tx_buffer_info; 4948 tx_desc = IGB_TX_DESC(tx_ring, 0); 4949 } 4950 4951 /* unmap any remaining paged data */ 4952 if (dma_unmap_len(tx_buffer, len)) 4953 dma_unmap_page(tx_ring->dev, 4954 dma_unmap_addr(tx_buffer, dma), 4955 dma_unmap_len(tx_buffer, len), 4956 DMA_TO_DEVICE); 4957 } 4958 4959 tx_buffer->next_to_watch = NULL; 4960 4961 /* move us one more past the eop_desc for start of next pkt */ 4962 tx_buffer++; 4963 i++; 4964 if (unlikely(i == tx_ring->count)) { 4965 i = 0; 4966 tx_buffer = tx_ring->tx_buffer_info; 4967 } 4968 } 4969 4970 /* reset BQL for queue */ 4971 netdev_tx_reset_queue(txring_txq(tx_ring)); 4972 4973 /* reset next_to_use and next_to_clean */ 4974 tx_ring->next_to_use = 0; 4975 tx_ring->next_to_clean = 0; 4976 } 4977 4978 /** 4979 * igb_clean_all_tx_rings - Free Tx Buffers for all queues 4980 * @adapter: board private structure 4981 **/ 4982 static void igb_clean_all_tx_rings(struct igb_adapter *adapter) 4983 { 4984 int i; 4985 4986 for (i = 0; i < adapter->num_tx_queues; i++) 4987 if (adapter->tx_ring[i]) 4988 igb_clean_tx_ring(adapter->tx_ring[i]); 4989 } 4990 4991 /** 4992 * igb_free_rx_resources - Free Rx Resources 4993 * @rx_ring: ring to clean the resources from 4994 * 4995 * Free all receive software resources 4996 **/ 4997 void igb_free_rx_resources(struct igb_ring *rx_ring) 4998 { 4999 igb_clean_rx_ring(rx_ring); 5000 5001 rx_ring->xdp_prog = NULL; 5002 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 5003 vfree(rx_ring->rx_buffer_info); 5004 rx_ring->rx_buffer_info = NULL; 5005 5006 /* if not set, then don't free */ 5007 if (!rx_ring->desc) 5008 return; 5009 5010 dma_free_coherent(rx_ring->dev, rx_ring->size, 5011 rx_ring->desc, rx_ring->dma); 5012 5013 rx_ring->desc = NULL; 5014 } 5015 5016 /** 5017 * igb_free_all_rx_resources - Free Rx Resources for All Queues 5018 * @adapter: board private structure 5019 * 5020 * Free all receive software resources 5021 **/ 5022 static void igb_free_all_rx_resources(struct igb_adapter *adapter) 5023 { 5024 int i; 5025 5026 for (i = 0; i < adapter->num_rx_queues; i++) 5027 if (adapter->rx_ring[i]) 5028 igb_free_rx_resources(adapter->rx_ring[i]); 5029 } 5030 5031 /** 5032 * igb_clean_rx_ring - Free Rx Buffers per Queue 5033 * @rx_ring: ring to free buffers from 5034 **/ 5035 static void igb_clean_rx_ring(struct igb_ring *rx_ring) 5036 { 5037 u16 i = rx_ring->next_to_clean; 5038 5039 dev_kfree_skb(rx_ring->skb); 5040 rx_ring->skb = NULL; 5041 5042 /* Free all the Rx ring sk_buffs */ 5043 while (i != rx_ring->next_to_alloc) { 5044 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 5045 5046 /* Invalidate cache lines that may have been written to by 5047 * device so that we avoid corrupting memory. 5048 */ 5049 dma_sync_single_range_for_cpu(rx_ring->dev, 5050 buffer_info->dma, 5051 buffer_info->page_offset, 5052 igb_rx_bufsz(rx_ring), 5053 DMA_FROM_DEVICE); 5054 5055 /* free resources associated with mapping */ 5056 dma_unmap_page_attrs(rx_ring->dev, 5057 buffer_info->dma, 5058 igb_rx_pg_size(rx_ring), 5059 DMA_FROM_DEVICE, 5060 IGB_RX_DMA_ATTR); 5061 __page_frag_cache_drain(buffer_info->page, 5062 buffer_info->pagecnt_bias); 5063 5064 i++; 5065 if (i == rx_ring->count) 5066 i = 0; 5067 } 5068 5069 rx_ring->next_to_alloc = 0; 5070 rx_ring->next_to_clean = 0; 5071 rx_ring->next_to_use = 0; 5072 } 5073 5074 /** 5075 * igb_clean_all_rx_rings - Free Rx Buffers for all queues 5076 * @adapter: board private structure 5077 **/ 5078 static void igb_clean_all_rx_rings(struct igb_adapter *adapter) 5079 { 5080 int i; 5081 5082 for (i = 0; i < adapter->num_rx_queues; i++) 5083 if (adapter->rx_ring[i]) 5084 igb_clean_rx_ring(adapter->rx_ring[i]); 5085 } 5086 5087 /** 5088 * igb_set_mac - Change the Ethernet Address of the NIC 5089 * @netdev: network interface device structure 5090 * @p: pointer to an address structure 5091 * 5092 * Returns 0 on success, negative on failure 5093 **/ 5094 static int igb_set_mac(struct net_device *netdev, void *p) 5095 { 5096 struct igb_adapter *adapter = netdev_priv(netdev); 5097 struct e1000_hw *hw = &adapter->hw; 5098 struct sockaddr *addr = p; 5099 5100 if (!is_valid_ether_addr(addr->sa_data)) 5101 return -EADDRNOTAVAIL; 5102 5103 eth_hw_addr_set(netdev, addr->sa_data); 5104 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 5105 5106 /* set the correct pool for the new PF MAC address in entry 0 */ 5107 igb_set_default_mac_filter(adapter); 5108 5109 return 0; 5110 } 5111 5112 /** 5113 * igb_write_mc_addr_list - write multicast addresses to MTA 5114 * @netdev: network interface device structure 5115 * 5116 * Writes multicast address list to the MTA hash table. 5117 * Returns: -ENOMEM on failure 5118 * 0 on no addresses written 5119 * X on writing X addresses to MTA 5120 **/ 5121 static int igb_write_mc_addr_list(struct net_device *netdev) 5122 { 5123 struct igb_adapter *adapter = netdev_priv(netdev); 5124 struct e1000_hw *hw = &adapter->hw; 5125 struct netdev_hw_addr *ha; 5126 u8 *mta_list; 5127 int i; 5128 5129 if (netdev_mc_empty(netdev)) { 5130 /* nothing to program, so clear mc list */ 5131 igb_update_mc_addr_list(hw, NULL, 0); 5132 igb_restore_vf_multicasts(adapter); 5133 return 0; 5134 } 5135 5136 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 5137 if (!mta_list) 5138 return -ENOMEM; 5139 5140 /* The shared function expects a packed array of only addresses. */ 5141 i = 0; 5142 netdev_for_each_mc_addr(ha, netdev) 5143 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 5144 5145 igb_update_mc_addr_list(hw, mta_list, i); 5146 kfree(mta_list); 5147 5148 return netdev_mc_count(netdev); 5149 } 5150 5151 static int igb_vlan_promisc_enable(struct igb_adapter *adapter) 5152 { 5153 struct e1000_hw *hw = &adapter->hw; 5154 u32 i, pf_id; 5155 5156 switch (hw->mac.type) { 5157 case e1000_i210: 5158 case e1000_i211: 5159 case e1000_i350: 5160 /* VLAN filtering needed for VLAN prio filter */ 5161 if (adapter->netdev->features & NETIF_F_NTUPLE) 5162 break; 5163 fallthrough; 5164 case e1000_82576: 5165 case e1000_82580: 5166 case e1000_i354: 5167 /* VLAN filtering needed for pool filtering */ 5168 if (adapter->vfs_allocated_count) 5169 break; 5170 fallthrough; 5171 default: 5172 return 1; 5173 } 5174 5175 /* We are already in VLAN promisc, nothing to do */ 5176 if (adapter->flags & IGB_FLAG_VLAN_PROMISC) 5177 return 0; 5178 5179 if (!adapter->vfs_allocated_count) 5180 goto set_vfta; 5181 5182 /* Add PF to all active pools */ 5183 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT; 5184 5185 for (i = E1000_VLVF_ARRAY_SIZE; --i;) { 5186 u32 vlvf = rd32(E1000_VLVF(i)); 5187 5188 vlvf |= BIT(pf_id); 5189 wr32(E1000_VLVF(i), vlvf); 5190 } 5191 5192 set_vfta: 5193 /* Set all bits in the VLAN filter table array */ 5194 for (i = E1000_VLAN_FILTER_TBL_SIZE; i--;) 5195 hw->mac.ops.write_vfta(hw, i, ~0U); 5196 5197 /* Set flag so we don't redo unnecessary work */ 5198 adapter->flags |= IGB_FLAG_VLAN_PROMISC; 5199 5200 return 0; 5201 } 5202 5203 #define VFTA_BLOCK_SIZE 8 5204 static void igb_scrub_vfta(struct igb_adapter *adapter, u32 vfta_offset) 5205 { 5206 struct e1000_hw *hw = &adapter->hw; 5207 u32 vfta[VFTA_BLOCK_SIZE] = { 0 }; 5208 u32 vid_start = vfta_offset * 32; 5209 u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32); 5210 u32 i, vid, word, bits, pf_id; 5211 5212 /* guarantee that we don't scrub out management VLAN */ 5213 vid = adapter->mng_vlan_id; 5214 if (vid >= vid_start && vid < vid_end) 5215 vfta[(vid - vid_start) / 32] |= BIT(vid % 32); 5216 5217 if (!adapter->vfs_allocated_count) 5218 goto set_vfta; 5219 5220 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT; 5221 5222 for (i = E1000_VLVF_ARRAY_SIZE; --i;) { 5223 u32 vlvf = rd32(E1000_VLVF(i)); 5224 5225 /* pull VLAN ID from VLVF */ 5226 vid = vlvf & VLAN_VID_MASK; 5227 5228 /* only concern ourselves with a certain range */ 5229 if (vid < vid_start || vid >= vid_end) 5230 continue; 5231 5232 if (vlvf & E1000_VLVF_VLANID_ENABLE) { 5233 /* record VLAN ID in VFTA */ 5234 vfta[(vid - vid_start) / 32] |= BIT(vid % 32); 5235 5236 /* if PF is part of this then continue */ 5237 if (test_bit(vid, adapter->active_vlans)) 5238 continue; 5239 } 5240 5241 /* remove PF from the pool */ 5242 bits = ~BIT(pf_id); 5243 bits &= rd32(E1000_VLVF(i)); 5244 wr32(E1000_VLVF(i), bits); 5245 } 5246 5247 set_vfta: 5248 /* extract values from active_vlans and write back to VFTA */ 5249 for (i = VFTA_BLOCK_SIZE; i--;) { 5250 vid = (vfta_offset + i) * 32; 5251 word = vid / BITS_PER_LONG; 5252 bits = vid % BITS_PER_LONG; 5253 5254 vfta[i] |= adapter->active_vlans[word] >> bits; 5255 5256 hw->mac.ops.write_vfta(hw, vfta_offset + i, vfta[i]); 5257 } 5258 } 5259 5260 static void igb_vlan_promisc_disable(struct igb_adapter *adapter) 5261 { 5262 u32 i; 5263 5264 /* We are not in VLAN promisc, nothing to do */ 5265 if (!(adapter->flags & IGB_FLAG_VLAN_PROMISC)) 5266 return; 5267 5268 /* Set flag so we don't redo unnecessary work */ 5269 adapter->flags &= ~IGB_FLAG_VLAN_PROMISC; 5270 5271 for (i = 0; i < E1000_VLAN_FILTER_TBL_SIZE; i += VFTA_BLOCK_SIZE) 5272 igb_scrub_vfta(adapter, i); 5273 } 5274 5275 /** 5276 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 5277 * @netdev: network interface device structure 5278 * 5279 * The set_rx_mode entry point is called whenever the unicast or multicast 5280 * address lists or the network interface flags are updated. This routine is 5281 * responsible for configuring the hardware for proper unicast, multicast, 5282 * promiscuous mode, and all-multi behavior. 5283 **/ 5284 static void igb_set_rx_mode(struct net_device *netdev) 5285 { 5286 struct igb_adapter *adapter = netdev_priv(netdev); 5287 struct e1000_hw *hw = &adapter->hw; 5288 unsigned int vfn = adapter->vfs_allocated_count; 5289 u32 rctl = 0, vmolr = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 5290 int count; 5291 5292 /* Check for Promiscuous and All Multicast modes */ 5293 if (netdev->flags & IFF_PROMISC) { 5294 rctl |= E1000_RCTL_UPE | E1000_RCTL_MPE; 5295 vmolr |= E1000_VMOLR_MPME; 5296 5297 /* enable use of UTA filter to force packets to default pool */ 5298 if (hw->mac.type == e1000_82576) 5299 vmolr |= E1000_VMOLR_ROPE; 5300 } else { 5301 if (netdev->flags & IFF_ALLMULTI) { 5302 rctl |= E1000_RCTL_MPE; 5303 vmolr |= E1000_VMOLR_MPME; 5304 } else { 5305 /* Write addresses to the MTA, if the attempt fails 5306 * then we should just turn on promiscuous mode so 5307 * that we can at least receive multicast traffic 5308 */ 5309 count = igb_write_mc_addr_list(netdev); 5310 if (count < 0) { 5311 rctl |= E1000_RCTL_MPE; 5312 vmolr |= E1000_VMOLR_MPME; 5313 } else if (count) { 5314 vmolr |= E1000_VMOLR_ROMPE; 5315 } 5316 } 5317 } 5318 5319 /* Write addresses to available RAR registers, if there is not 5320 * sufficient space to store all the addresses then enable 5321 * unicast promiscuous mode 5322 */ 5323 if (__dev_uc_sync(netdev, igb_uc_sync, igb_uc_unsync)) { 5324 rctl |= E1000_RCTL_UPE; 5325 vmolr |= E1000_VMOLR_ROPE; 5326 } 5327 5328 /* enable VLAN filtering by default */ 5329 rctl |= E1000_RCTL_VFE; 5330 5331 /* disable VLAN filtering for modes that require it */ 5332 if ((netdev->flags & IFF_PROMISC) || 5333 (netdev->features & NETIF_F_RXALL)) { 5334 /* if we fail to set all rules then just clear VFE */ 5335 if (igb_vlan_promisc_enable(adapter)) 5336 rctl &= ~E1000_RCTL_VFE; 5337 } else { 5338 igb_vlan_promisc_disable(adapter); 5339 } 5340 5341 /* update state of unicast, multicast, and VLAN filtering modes */ 5342 rctl |= rd32(E1000_RCTL) & ~(E1000_RCTL_UPE | E1000_RCTL_MPE | 5343 E1000_RCTL_VFE); 5344 wr32(E1000_RCTL, rctl); 5345 5346 #if (PAGE_SIZE < 8192) 5347 if (!adapter->vfs_allocated_count) { 5348 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB) 5349 rlpml = IGB_MAX_FRAME_BUILD_SKB; 5350 } 5351 #endif 5352 wr32(E1000_RLPML, rlpml); 5353 5354 /* In order to support SR-IOV and eventually VMDq it is necessary to set 5355 * the VMOLR to enable the appropriate modes. Without this workaround 5356 * we will have issues with VLAN tag stripping not being done for frames 5357 * that are only arriving because we are the default pool 5358 */ 5359 if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350)) 5360 return; 5361 5362 /* set UTA to appropriate mode */ 5363 igb_set_uta(adapter, !!(vmolr & E1000_VMOLR_ROPE)); 5364 5365 vmolr |= rd32(E1000_VMOLR(vfn)) & 5366 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE); 5367 5368 /* enable Rx jumbo frames, restrict as needed to support build_skb */ 5369 vmolr &= ~E1000_VMOLR_RLPML_MASK; 5370 #if (PAGE_SIZE < 8192) 5371 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB) 5372 vmolr |= IGB_MAX_FRAME_BUILD_SKB; 5373 else 5374 #endif 5375 vmolr |= MAX_JUMBO_FRAME_SIZE; 5376 vmolr |= E1000_VMOLR_LPE; 5377 5378 wr32(E1000_VMOLR(vfn), vmolr); 5379 5380 igb_restore_vf_multicasts(adapter); 5381 } 5382 5383 static void igb_check_wvbr(struct igb_adapter *adapter) 5384 { 5385 struct e1000_hw *hw = &adapter->hw; 5386 u32 wvbr = 0; 5387 5388 switch (hw->mac.type) { 5389 case e1000_82576: 5390 case e1000_i350: 5391 wvbr = rd32(E1000_WVBR); 5392 if (!wvbr) 5393 return; 5394 break; 5395 default: 5396 break; 5397 } 5398 5399 adapter->wvbr |= wvbr; 5400 } 5401 5402 #define IGB_STAGGERED_QUEUE_OFFSET 8 5403 5404 static void igb_spoof_check(struct igb_adapter *adapter) 5405 { 5406 int j; 5407 5408 if (!adapter->wvbr) 5409 return; 5410 5411 for (j = 0; j < adapter->vfs_allocated_count; j++) { 5412 if (adapter->wvbr & BIT(j) || 5413 adapter->wvbr & BIT(j + IGB_STAGGERED_QUEUE_OFFSET)) { 5414 dev_warn(&adapter->pdev->dev, 5415 "Spoof event(s) detected on VF %d\n", j); 5416 adapter->wvbr &= 5417 ~(BIT(j) | 5418 BIT(j + IGB_STAGGERED_QUEUE_OFFSET)); 5419 } 5420 } 5421 } 5422 5423 /* Need to wait a few seconds after link up to get diagnostic information from 5424 * the phy 5425 */ 5426 static void igb_update_phy_info(struct timer_list *t) 5427 { 5428 struct igb_adapter *adapter = from_timer(adapter, t, phy_info_timer); 5429 igb_get_phy_info(&adapter->hw); 5430 } 5431 5432 /** 5433 * igb_has_link - check shared code for link and determine up/down 5434 * @adapter: pointer to driver private info 5435 **/ 5436 bool igb_has_link(struct igb_adapter *adapter) 5437 { 5438 struct e1000_hw *hw = &adapter->hw; 5439 bool link_active = false; 5440 5441 /* get_link_status is set on LSC (link status) interrupt or 5442 * rx sequence error interrupt. get_link_status will stay 5443 * false until the e1000_check_for_link establishes link 5444 * for copper adapters ONLY 5445 */ 5446 switch (hw->phy.media_type) { 5447 case e1000_media_type_copper: 5448 if (!hw->mac.get_link_status) 5449 return true; 5450 fallthrough; 5451 case e1000_media_type_internal_serdes: 5452 hw->mac.ops.check_for_link(hw); 5453 link_active = !hw->mac.get_link_status; 5454 break; 5455 default: 5456 case e1000_media_type_unknown: 5457 break; 5458 } 5459 5460 if (((hw->mac.type == e1000_i210) || 5461 (hw->mac.type == e1000_i211)) && 5462 (hw->phy.id == I210_I_PHY_ID)) { 5463 if (!netif_carrier_ok(adapter->netdev)) { 5464 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE; 5465 } else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) { 5466 adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE; 5467 adapter->link_check_timeout = jiffies; 5468 } 5469 } 5470 5471 return link_active; 5472 } 5473 5474 static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event) 5475 { 5476 bool ret = false; 5477 u32 ctrl_ext, thstat; 5478 5479 /* check for thermal sensor event on i350 copper only */ 5480 if (hw->mac.type == e1000_i350) { 5481 thstat = rd32(E1000_THSTAT); 5482 ctrl_ext = rd32(E1000_CTRL_EXT); 5483 5484 if ((hw->phy.media_type == e1000_media_type_copper) && 5485 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) 5486 ret = !!(thstat & event); 5487 } 5488 5489 return ret; 5490 } 5491 5492 /** 5493 * igb_check_lvmmc - check for malformed packets received 5494 * and indicated in LVMMC register 5495 * @adapter: pointer to adapter 5496 **/ 5497 static void igb_check_lvmmc(struct igb_adapter *adapter) 5498 { 5499 struct e1000_hw *hw = &adapter->hw; 5500 u32 lvmmc; 5501 5502 lvmmc = rd32(E1000_LVMMC); 5503 if (lvmmc) { 5504 if (unlikely(net_ratelimit())) { 5505 netdev_warn(adapter->netdev, 5506 "malformed Tx packet detected and dropped, LVMMC:0x%08x\n", 5507 lvmmc); 5508 } 5509 } 5510 } 5511 5512 /** 5513 * igb_watchdog - Timer Call-back 5514 * @t: pointer to timer_list containing our private info pointer 5515 **/ 5516 static void igb_watchdog(struct timer_list *t) 5517 { 5518 struct igb_adapter *adapter = from_timer(adapter, t, watchdog_timer); 5519 /* Do the rest outside of interrupt context */ 5520 schedule_work(&adapter->watchdog_task); 5521 } 5522 5523 static void igb_watchdog_task(struct work_struct *work) 5524 { 5525 struct igb_adapter *adapter = container_of(work, 5526 struct igb_adapter, 5527 watchdog_task); 5528 struct e1000_hw *hw = &adapter->hw; 5529 struct e1000_phy_info *phy = &hw->phy; 5530 struct net_device *netdev = adapter->netdev; 5531 u32 link; 5532 int i; 5533 u32 connsw; 5534 u16 phy_data, retry_count = 20; 5535 5536 link = igb_has_link(adapter); 5537 5538 if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) { 5539 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 5540 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE; 5541 else 5542 link = false; 5543 } 5544 5545 /* Force link down if we have fiber to swap to */ 5546 if (adapter->flags & IGB_FLAG_MAS_ENABLE) { 5547 if (hw->phy.media_type == e1000_media_type_copper) { 5548 connsw = rd32(E1000_CONNSW); 5549 if (!(connsw & E1000_CONNSW_AUTOSENSE_EN)) 5550 link = 0; 5551 } 5552 } 5553 if (link) { 5554 /* Perform a reset if the media type changed. */ 5555 if (hw->dev_spec._82575.media_changed) { 5556 hw->dev_spec._82575.media_changed = false; 5557 adapter->flags |= IGB_FLAG_MEDIA_RESET; 5558 igb_reset(adapter); 5559 } 5560 /* Cancel scheduled suspend requests. */ 5561 pm_runtime_resume(netdev->dev.parent); 5562 5563 if (!netif_carrier_ok(netdev)) { 5564 u32 ctrl; 5565 5566 hw->mac.ops.get_speed_and_duplex(hw, 5567 &adapter->link_speed, 5568 &adapter->link_duplex); 5569 5570 ctrl = rd32(E1000_CTRL); 5571 /* Links status message must follow this format */ 5572 netdev_info(netdev, 5573 "igb: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 5574 netdev->name, 5575 adapter->link_speed, 5576 adapter->link_duplex == FULL_DUPLEX ? 5577 "Full" : "Half", 5578 (ctrl & E1000_CTRL_TFCE) && 5579 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" : 5580 (ctrl & E1000_CTRL_RFCE) ? "RX" : 5581 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None"); 5582 5583 /* disable EEE if enabled */ 5584 if ((adapter->flags & IGB_FLAG_EEE) && 5585 (adapter->link_duplex == HALF_DUPLEX)) { 5586 dev_info(&adapter->pdev->dev, 5587 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex.\n"); 5588 adapter->hw.dev_spec._82575.eee_disable = true; 5589 adapter->flags &= ~IGB_FLAG_EEE; 5590 } 5591 5592 /* check if SmartSpeed worked */ 5593 igb_check_downshift(hw); 5594 if (phy->speed_downgraded) 5595 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 5596 5597 /* check for thermal sensor event */ 5598 if (igb_thermal_sensor_event(hw, 5599 E1000_THSTAT_LINK_THROTTLE)) 5600 netdev_info(netdev, "The network adapter link speed was downshifted because it overheated\n"); 5601 5602 /* adjust timeout factor according to speed/duplex */ 5603 adapter->tx_timeout_factor = 1; 5604 switch (adapter->link_speed) { 5605 case SPEED_10: 5606 adapter->tx_timeout_factor = 14; 5607 break; 5608 case SPEED_100: 5609 /* maybe add some timeout factor ? */ 5610 break; 5611 } 5612 5613 if (adapter->link_speed != SPEED_1000 || 5614 !hw->phy.ops.read_reg) 5615 goto no_wait; 5616 5617 /* wait for Remote receiver status OK */ 5618 retry_read_status: 5619 if (!igb_read_phy_reg(hw, PHY_1000T_STATUS, 5620 &phy_data)) { 5621 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 5622 retry_count) { 5623 msleep(100); 5624 retry_count--; 5625 goto retry_read_status; 5626 } else if (!retry_count) { 5627 dev_err(&adapter->pdev->dev, "exceed max 2 second\n"); 5628 } 5629 } else { 5630 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n"); 5631 } 5632 no_wait: 5633 netif_carrier_on(netdev); 5634 5635 igb_ping_all_vfs(adapter); 5636 igb_check_vf_rate_limit(adapter); 5637 5638 /* link state has changed, schedule phy info update */ 5639 if (!test_bit(__IGB_DOWN, &adapter->state)) 5640 mod_timer(&adapter->phy_info_timer, 5641 round_jiffies(jiffies + 2 * HZ)); 5642 } 5643 } else { 5644 if (netif_carrier_ok(netdev)) { 5645 adapter->link_speed = 0; 5646 adapter->link_duplex = 0; 5647 5648 /* check for thermal sensor event */ 5649 if (igb_thermal_sensor_event(hw, 5650 E1000_THSTAT_PWR_DOWN)) { 5651 netdev_err(netdev, "The network adapter was stopped because it overheated\n"); 5652 } 5653 5654 /* Links status message must follow this format */ 5655 netdev_info(netdev, "igb: %s NIC Link is Down\n", 5656 netdev->name); 5657 netif_carrier_off(netdev); 5658 5659 igb_ping_all_vfs(adapter); 5660 5661 /* link state has changed, schedule phy info update */ 5662 if (!test_bit(__IGB_DOWN, &adapter->state)) 5663 mod_timer(&adapter->phy_info_timer, 5664 round_jiffies(jiffies + 2 * HZ)); 5665 5666 /* link is down, time to check for alternate media */ 5667 if (adapter->flags & IGB_FLAG_MAS_ENABLE) { 5668 igb_check_swap_media(adapter); 5669 if (adapter->flags & IGB_FLAG_MEDIA_RESET) { 5670 schedule_work(&adapter->reset_task); 5671 /* return immediately */ 5672 return; 5673 } 5674 } 5675 pm_schedule_suspend(netdev->dev.parent, 5676 MSEC_PER_SEC * 5); 5677 5678 /* also check for alternate media here */ 5679 } else if (!netif_carrier_ok(netdev) && 5680 (adapter->flags & IGB_FLAG_MAS_ENABLE)) { 5681 igb_check_swap_media(adapter); 5682 if (adapter->flags & IGB_FLAG_MEDIA_RESET) { 5683 schedule_work(&adapter->reset_task); 5684 /* return immediately */ 5685 return; 5686 } 5687 } 5688 } 5689 5690 spin_lock(&adapter->stats64_lock); 5691 igb_update_stats(adapter); 5692 spin_unlock(&adapter->stats64_lock); 5693 5694 for (i = 0; i < adapter->num_tx_queues; i++) { 5695 struct igb_ring *tx_ring = adapter->tx_ring[i]; 5696 if (!netif_carrier_ok(netdev)) { 5697 /* We've lost link, so the controller stops DMA, 5698 * but we've got queued Tx work that's never going 5699 * to get done, so reset controller to flush Tx. 5700 * (Do the reset outside of interrupt context). 5701 */ 5702 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) { 5703 adapter->tx_timeout_count++; 5704 schedule_work(&adapter->reset_task); 5705 /* return immediately since reset is imminent */ 5706 return; 5707 } 5708 } 5709 5710 /* Force detection of hung controller every watchdog period */ 5711 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 5712 } 5713 5714 /* Cause software interrupt to ensure Rx ring is cleaned */ 5715 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 5716 u32 eics = 0; 5717 5718 for (i = 0; i < adapter->num_q_vectors; i++) 5719 eics |= adapter->q_vector[i]->eims_value; 5720 wr32(E1000_EICS, eics); 5721 } else { 5722 wr32(E1000_ICS, E1000_ICS_RXDMT0); 5723 } 5724 5725 igb_spoof_check(adapter); 5726 igb_ptp_rx_hang(adapter); 5727 igb_ptp_tx_hang(adapter); 5728 5729 /* Check LVMMC register on i350/i354 only */ 5730 if ((adapter->hw.mac.type == e1000_i350) || 5731 (adapter->hw.mac.type == e1000_i354)) 5732 igb_check_lvmmc(adapter); 5733 5734 /* Reset the timer */ 5735 if (!test_bit(__IGB_DOWN, &adapter->state)) { 5736 if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) 5737 mod_timer(&adapter->watchdog_timer, 5738 round_jiffies(jiffies + HZ)); 5739 else 5740 mod_timer(&adapter->watchdog_timer, 5741 round_jiffies(jiffies + 2 * HZ)); 5742 } 5743 } 5744 5745 enum latency_range { 5746 lowest_latency = 0, 5747 low_latency = 1, 5748 bulk_latency = 2, 5749 latency_invalid = 255 5750 }; 5751 5752 /** 5753 * igb_update_ring_itr - update the dynamic ITR value based on packet size 5754 * @q_vector: pointer to q_vector 5755 * 5756 * Stores a new ITR value based on strictly on packet size. This 5757 * algorithm is less sophisticated than that used in igb_update_itr, 5758 * due to the difficulty of synchronizing statistics across multiple 5759 * receive rings. The divisors and thresholds used by this function 5760 * were determined based on theoretical maximum wire speed and testing 5761 * data, in order to minimize response time while increasing bulk 5762 * throughput. 5763 * This functionality is controlled by ethtool's coalescing settings. 5764 * NOTE: This function is called only when operating in a multiqueue 5765 * receive environment. 5766 **/ 5767 static void igb_update_ring_itr(struct igb_q_vector *q_vector) 5768 { 5769 int new_val = q_vector->itr_val; 5770 int avg_wire_size = 0; 5771 struct igb_adapter *adapter = q_vector->adapter; 5772 unsigned int packets; 5773 5774 /* For non-gigabit speeds, just fix the interrupt rate at 4000 5775 * ints/sec - ITR timer value of 120 ticks. 5776 */ 5777 if (adapter->link_speed != SPEED_1000) { 5778 new_val = IGB_4K_ITR; 5779 goto set_itr_val; 5780 } 5781 5782 packets = q_vector->rx.total_packets; 5783 if (packets) 5784 avg_wire_size = q_vector->rx.total_bytes / packets; 5785 5786 packets = q_vector->tx.total_packets; 5787 if (packets) 5788 avg_wire_size = max_t(u32, avg_wire_size, 5789 q_vector->tx.total_bytes / packets); 5790 5791 /* if avg_wire_size isn't set no work was done */ 5792 if (!avg_wire_size) 5793 goto clear_counts; 5794 5795 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 5796 avg_wire_size += 24; 5797 5798 /* Don't starve jumbo frames */ 5799 avg_wire_size = min(avg_wire_size, 3000); 5800 5801 /* Give a little boost to mid-size frames */ 5802 if ((avg_wire_size > 300) && (avg_wire_size < 1200)) 5803 new_val = avg_wire_size / 3; 5804 else 5805 new_val = avg_wire_size / 2; 5806 5807 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 5808 if (new_val < IGB_20K_ITR && 5809 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 5810 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 5811 new_val = IGB_20K_ITR; 5812 5813 set_itr_val: 5814 if (new_val != q_vector->itr_val) { 5815 q_vector->itr_val = new_val; 5816 q_vector->set_itr = 1; 5817 } 5818 clear_counts: 5819 q_vector->rx.total_bytes = 0; 5820 q_vector->rx.total_packets = 0; 5821 q_vector->tx.total_bytes = 0; 5822 q_vector->tx.total_packets = 0; 5823 } 5824 5825 /** 5826 * igb_update_itr - update the dynamic ITR value based on statistics 5827 * @q_vector: pointer to q_vector 5828 * @ring_container: ring info to update the itr for 5829 * 5830 * Stores a new ITR value based on packets and byte 5831 * counts during the last interrupt. The advantage of per interrupt 5832 * computation is faster updates and more accurate ITR for the current 5833 * traffic pattern. Constants in this function were computed 5834 * based on theoretical maximum wire speed and thresholds were set based 5835 * on testing data as well as attempting to minimize response time 5836 * while increasing bulk throughput. 5837 * This functionality is controlled by ethtool's coalescing settings. 5838 * NOTE: These calculations are only valid when operating in a single- 5839 * queue environment. 5840 **/ 5841 static void igb_update_itr(struct igb_q_vector *q_vector, 5842 struct igb_ring_container *ring_container) 5843 { 5844 unsigned int packets = ring_container->total_packets; 5845 unsigned int bytes = ring_container->total_bytes; 5846 u8 itrval = ring_container->itr; 5847 5848 /* no packets, exit with status unchanged */ 5849 if (packets == 0) 5850 return; 5851 5852 switch (itrval) { 5853 case lowest_latency: 5854 /* handle TSO and jumbo frames */ 5855 if (bytes/packets > 8000) 5856 itrval = bulk_latency; 5857 else if ((packets < 5) && (bytes > 512)) 5858 itrval = low_latency; 5859 break; 5860 case low_latency: /* 50 usec aka 20000 ints/s */ 5861 if (bytes > 10000) { 5862 /* this if handles the TSO accounting */ 5863 if (bytes/packets > 8000) 5864 itrval = bulk_latency; 5865 else if ((packets < 10) || ((bytes/packets) > 1200)) 5866 itrval = bulk_latency; 5867 else if ((packets > 35)) 5868 itrval = lowest_latency; 5869 } else if (bytes/packets > 2000) { 5870 itrval = bulk_latency; 5871 } else if (packets <= 2 && bytes < 512) { 5872 itrval = lowest_latency; 5873 } 5874 break; 5875 case bulk_latency: /* 250 usec aka 4000 ints/s */ 5876 if (bytes > 25000) { 5877 if (packets > 35) 5878 itrval = low_latency; 5879 } else if (bytes < 1500) { 5880 itrval = low_latency; 5881 } 5882 break; 5883 } 5884 5885 /* clear work counters since we have the values we need */ 5886 ring_container->total_bytes = 0; 5887 ring_container->total_packets = 0; 5888 5889 /* write updated itr to ring container */ 5890 ring_container->itr = itrval; 5891 } 5892 5893 static void igb_set_itr(struct igb_q_vector *q_vector) 5894 { 5895 struct igb_adapter *adapter = q_vector->adapter; 5896 u32 new_itr = q_vector->itr_val; 5897 u8 current_itr = 0; 5898 5899 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 5900 if (adapter->link_speed != SPEED_1000) { 5901 current_itr = 0; 5902 new_itr = IGB_4K_ITR; 5903 goto set_itr_now; 5904 } 5905 5906 igb_update_itr(q_vector, &q_vector->tx); 5907 igb_update_itr(q_vector, &q_vector->rx); 5908 5909 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 5910 5911 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 5912 if (current_itr == lowest_latency && 5913 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 5914 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 5915 current_itr = low_latency; 5916 5917 switch (current_itr) { 5918 /* counts and packets in update_itr are dependent on these numbers */ 5919 case lowest_latency: 5920 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */ 5921 break; 5922 case low_latency: 5923 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */ 5924 break; 5925 case bulk_latency: 5926 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */ 5927 break; 5928 default: 5929 break; 5930 } 5931 5932 set_itr_now: 5933 if (new_itr != q_vector->itr_val) { 5934 /* this attempts to bias the interrupt rate towards Bulk 5935 * by adding intermediate steps when interrupt rate is 5936 * increasing 5937 */ 5938 new_itr = new_itr > q_vector->itr_val ? 5939 max((new_itr * q_vector->itr_val) / 5940 (new_itr + (q_vector->itr_val >> 2)), 5941 new_itr) : new_itr; 5942 /* Don't write the value here; it resets the adapter's 5943 * internal timer, and causes us to delay far longer than 5944 * we should between interrupts. Instead, we write the ITR 5945 * value at the beginning of the next interrupt so the timing 5946 * ends up being correct. 5947 */ 5948 q_vector->itr_val = new_itr; 5949 q_vector->set_itr = 1; 5950 } 5951 } 5952 5953 static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, 5954 struct igb_tx_buffer *first, 5955 u32 vlan_macip_lens, u32 type_tucmd, 5956 u32 mss_l4len_idx) 5957 { 5958 struct e1000_adv_tx_context_desc *context_desc; 5959 u16 i = tx_ring->next_to_use; 5960 struct timespec64 ts; 5961 5962 context_desc = IGB_TX_CTXTDESC(tx_ring, i); 5963 5964 i++; 5965 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 5966 5967 /* set bits to identify this as an advanced context descriptor */ 5968 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 5969 5970 /* For 82575, context index must be unique per ring. */ 5971 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 5972 mss_l4len_idx |= tx_ring->reg_idx << 4; 5973 5974 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 5975 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 5976 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 5977 5978 /* We assume there is always a valid tx time available. Invalid times 5979 * should have been handled by the upper layers. 5980 */ 5981 if (tx_ring->launchtime_enable) { 5982 ts = ktime_to_timespec64(first->skb->tstamp); 5983 skb_txtime_consumed(first->skb); 5984 context_desc->seqnum_seed = cpu_to_le32(ts.tv_nsec / 32); 5985 } else { 5986 context_desc->seqnum_seed = 0; 5987 } 5988 } 5989 5990 static int igb_tso(struct igb_ring *tx_ring, 5991 struct igb_tx_buffer *first, 5992 u8 *hdr_len) 5993 { 5994 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 5995 struct sk_buff *skb = first->skb; 5996 union { 5997 struct iphdr *v4; 5998 struct ipv6hdr *v6; 5999 unsigned char *hdr; 6000 } ip; 6001 union { 6002 struct tcphdr *tcp; 6003 struct udphdr *udp; 6004 unsigned char *hdr; 6005 } l4; 6006 u32 paylen, l4_offset; 6007 int err; 6008 6009 if (skb->ip_summed != CHECKSUM_PARTIAL) 6010 return 0; 6011 6012 if (!skb_is_gso(skb)) 6013 return 0; 6014 6015 err = skb_cow_head(skb, 0); 6016 if (err < 0) 6017 return err; 6018 6019 ip.hdr = skb_network_header(skb); 6020 l4.hdr = skb_checksum_start(skb); 6021 6022 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 6023 type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ? 6024 E1000_ADVTXD_TUCMD_L4T_UDP : E1000_ADVTXD_TUCMD_L4T_TCP; 6025 6026 /* initialize outer IP header fields */ 6027 if (ip.v4->version == 4) { 6028 unsigned char *csum_start = skb_checksum_start(skb); 6029 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 6030 6031 /* IP header will have to cancel out any data that 6032 * is not a part of the outer IP header 6033 */ 6034 ip.v4->check = csum_fold(csum_partial(trans_start, 6035 csum_start - trans_start, 6036 0)); 6037 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4; 6038 6039 ip.v4->tot_len = 0; 6040 first->tx_flags |= IGB_TX_FLAGS_TSO | 6041 IGB_TX_FLAGS_CSUM | 6042 IGB_TX_FLAGS_IPV4; 6043 } else { 6044 ip.v6->payload_len = 0; 6045 first->tx_flags |= IGB_TX_FLAGS_TSO | 6046 IGB_TX_FLAGS_CSUM; 6047 } 6048 6049 /* determine offset of inner transport header */ 6050 l4_offset = l4.hdr - skb->data; 6051 6052 /* remove payload length from inner checksum */ 6053 paylen = skb->len - l4_offset; 6054 if (type_tucmd & E1000_ADVTXD_TUCMD_L4T_TCP) { 6055 /* compute length of segmentation header */ 6056 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 6057 csum_replace_by_diff(&l4.tcp->check, 6058 (__force __wsum)htonl(paylen)); 6059 } else { 6060 /* compute length of segmentation header */ 6061 *hdr_len = sizeof(*l4.udp) + l4_offset; 6062 csum_replace_by_diff(&l4.udp->check, 6063 (__force __wsum)htonl(paylen)); 6064 } 6065 6066 /* update gso size and bytecount with header size */ 6067 first->gso_segs = skb_shinfo(skb)->gso_segs; 6068 first->bytecount += (first->gso_segs - 1) * *hdr_len; 6069 6070 /* MSS L4LEN IDX */ 6071 mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT; 6072 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT; 6073 6074 /* VLAN MACLEN IPLEN */ 6075 vlan_macip_lens = l4.hdr - ip.hdr; 6076 vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT; 6077 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK; 6078 6079 igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, 6080 type_tucmd, mss_l4len_idx); 6081 6082 return 1; 6083 } 6084 6085 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first) 6086 { 6087 struct sk_buff *skb = first->skb; 6088 u32 vlan_macip_lens = 0; 6089 u32 type_tucmd = 0; 6090 6091 if (skb->ip_summed != CHECKSUM_PARTIAL) { 6092 csum_failed: 6093 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN) && 6094 !tx_ring->launchtime_enable) 6095 return; 6096 goto no_csum; 6097 } 6098 6099 switch (skb->csum_offset) { 6100 case offsetof(struct tcphdr, check): 6101 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP; 6102 fallthrough; 6103 case offsetof(struct udphdr, check): 6104 break; 6105 case offsetof(struct sctphdr, checksum): 6106 /* validate that this is actually an SCTP request */ 6107 if (skb_csum_is_sctp(skb)) { 6108 type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP; 6109 break; 6110 } 6111 fallthrough; 6112 default: 6113 skb_checksum_help(skb); 6114 goto csum_failed; 6115 } 6116 6117 /* update TX checksum flag */ 6118 first->tx_flags |= IGB_TX_FLAGS_CSUM; 6119 vlan_macip_lens = skb_checksum_start_offset(skb) - 6120 skb_network_offset(skb); 6121 no_csum: 6122 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT; 6123 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK; 6124 6125 igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0); 6126 } 6127 6128 #define IGB_SET_FLAG(_input, _flag, _result) \ 6129 ((_flag <= _result) ? \ 6130 ((u32)(_input & _flag) * (_result / _flag)) : \ 6131 ((u32)(_input & _flag) / (_flag / _result))) 6132 6133 static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 6134 { 6135 /* set type for advanced descriptor with frame checksum insertion */ 6136 u32 cmd_type = E1000_ADVTXD_DTYP_DATA | 6137 E1000_ADVTXD_DCMD_DEXT | 6138 E1000_ADVTXD_DCMD_IFCS; 6139 6140 /* set HW vlan bit if vlan is present */ 6141 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN, 6142 (E1000_ADVTXD_DCMD_VLE)); 6143 6144 /* set segmentation bits for TSO */ 6145 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO, 6146 (E1000_ADVTXD_DCMD_TSE)); 6147 6148 /* set timestamp bit if present */ 6149 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP, 6150 (E1000_ADVTXD_MAC_TSTAMP)); 6151 6152 /* insert frame checksum */ 6153 cmd_type ^= IGB_SET_FLAG(skb->no_fcs, 1, E1000_ADVTXD_DCMD_IFCS); 6154 6155 return cmd_type; 6156 } 6157 6158 static void igb_tx_olinfo_status(struct igb_ring *tx_ring, 6159 union e1000_adv_tx_desc *tx_desc, 6160 u32 tx_flags, unsigned int paylen) 6161 { 6162 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT; 6163 6164 /* 82575 requires a unique index per ring */ 6165 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 6166 olinfo_status |= tx_ring->reg_idx << 4; 6167 6168 /* insert L4 checksum */ 6169 olinfo_status |= IGB_SET_FLAG(tx_flags, 6170 IGB_TX_FLAGS_CSUM, 6171 (E1000_TXD_POPTS_TXSM << 8)); 6172 6173 /* insert IPv4 checksum */ 6174 olinfo_status |= IGB_SET_FLAG(tx_flags, 6175 IGB_TX_FLAGS_IPV4, 6176 (E1000_TXD_POPTS_IXSM << 8)); 6177 6178 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 6179 } 6180 6181 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size) 6182 { 6183 struct net_device *netdev = tx_ring->netdev; 6184 6185 netif_stop_subqueue(netdev, tx_ring->queue_index); 6186 6187 /* Herbert's original patch had: 6188 * smp_mb__after_netif_stop_queue(); 6189 * but since that doesn't exist yet, just open code it. 6190 */ 6191 smp_mb(); 6192 6193 /* We need to check again in a case another CPU has just 6194 * made room available. 6195 */ 6196 if (igb_desc_unused(tx_ring) < size) 6197 return -EBUSY; 6198 6199 /* A reprieve! */ 6200 netif_wake_subqueue(netdev, tx_ring->queue_index); 6201 6202 u64_stats_update_begin(&tx_ring->tx_syncp2); 6203 tx_ring->tx_stats.restart_queue2++; 6204 u64_stats_update_end(&tx_ring->tx_syncp2); 6205 6206 return 0; 6207 } 6208 6209 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size) 6210 { 6211 if (igb_desc_unused(tx_ring) >= size) 6212 return 0; 6213 return __igb_maybe_stop_tx(tx_ring, size); 6214 } 6215 6216 static int igb_tx_map(struct igb_ring *tx_ring, 6217 struct igb_tx_buffer *first, 6218 const u8 hdr_len) 6219 { 6220 struct sk_buff *skb = first->skb; 6221 struct igb_tx_buffer *tx_buffer; 6222 union e1000_adv_tx_desc *tx_desc; 6223 skb_frag_t *frag; 6224 dma_addr_t dma; 6225 unsigned int data_len, size; 6226 u32 tx_flags = first->tx_flags; 6227 u32 cmd_type = igb_tx_cmd_type(skb, tx_flags); 6228 u16 i = tx_ring->next_to_use; 6229 6230 tx_desc = IGB_TX_DESC(tx_ring, i); 6231 6232 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 6233 6234 size = skb_headlen(skb); 6235 data_len = skb->data_len; 6236 6237 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 6238 6239 tx_buffer = first; 6240 6241 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 6242 if (dma_mapping_error(tx_ring->dev, dma)) 6243 goto dma_error; 6244 6245 /* record length, and DMA address */ 6246 dma_unmap_len_set(tx_buffer, len, size); 6247 dma_unmap_addr_set(tx_buffer, dma, dma); 6248 6249 tx_desc->read.buffer_addr = cpu_to_le64(dma); 6250 6251 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) { 6252 tx_desc->read.cmd_type_len = 6253 cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD); 6254 6255 i++; 6256 tx_desc++; 6257 if (i == tx_ring->count) { 6258 tx_desc = IGB_TX_DESC(tx_ring, 0); 6259 i = 0; 6260 } 6261 tx_desc->read.olinfo_status = 0; 6262 6263 dma += IGB_MAX_DATA_PER_TXD; 6264 size -= IGB_MAX_DATA_PER_TXD; 6265 6266 tx_desc->read.buffer_addr = cpu_to_le64(dma); 6267 } 6268 6269 if (likely(!data_len)) 6270 break; 6271 6272 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 6273 6274 i++; 6275 tx_desc++; 6276 if (i == tx_ring->count) { 6277 tx_desc = IGB_TX_DESC(tx_ring, 0); 6278 i = 0; 6279 } 6280 tx_desc->read.olinfo_status = 0; 6281 6282 size = skb_frag_size(frag); 6283 data_len -= size; 6284 6285 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 6286 size, DMA_TO_DEVICE); 6287 6288 tx_buffer = &tx_ring->tx_buffer_info[i]; 6289 } 6290 6291 /* write last descriptor with RS and EOP bits */ 6292 cmd_type |= size | IGB_TXD_DCMD; 6293 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 6294 6295 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 6296 6297 /* set the timestamp */ 6298 first->time_stamp = jiffies; 6299 6300 skb_tx_timestamp(skb); 6301 6302 /* Force memory writes to complete before letting h/w know there 6303 * are new descriptors to fetch. (Only applicable for weak-ordered 6304 * memory model archs, such as IA-64). 6305 * 6306 * We also need this memory barrier to make certain all of the 6307 * status bits have been updated before next_to_watch is written. 6308 */ 6309 dma_wmb(); 6310 6311 /* set next_to_watch value indicating a packet is present */ 6312 first->next_to_watch = tx_desc; 6313 6314 i++; 6315 if (i == tx_ring->count) 6316 i = 0; 6317 6318 tx_ring->next_to_use = i; 6319 6320 /* Make sure there is space in the ring for the next send. */ 6321 igb_maybe_stop_tx(tx_ring, DESC_NEEDED); 6322 6323 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 6324 writel(i, tx_ring->tail); 6325 } 6326 return 0; 6327 6328 dma_error: 6329 dev_err(tx_ring->dev, "TX DMA map failed\n"); 6330 tx_buffer = &tx_ring->tx_buffer_info[i]; 6331 6332 /* clear dma mappings for failed tx_buffer_info map */ 6333 while (tx_buffer != first) { 6334 if (dma_unmap_len(tx_buffer, len)) 6335 dma_unmap_page(tx_ring->dev, 6336 dma_unmap_addr(tx_buffer, dma), 6337 dma_unmap_len(tx_buffer, len), 6338 DMA_TO_DEVICE); 6339 dma_unmap_len_set(tx_buffer, len, 0); 6340 6341 if (i-- == 0) 6342 i += tx_ring->count; 6343 tx_buffer = &tx_ring->tx_buffer_info[i]; 6344 } 6345 6346 if (dma_unmap_len(tx_buffer, len)) 6347 dma_unmap_single(tx_ring->dev, 6348 dma_unmap_addr(tx_buffer, dma), 6349 dma_unmap_len(tx_buffer, len), 6350 DMA_TO_DEVICE); 6351 dma_unmap_len_set(tx_buffer, len, 0); 6352 6353 dev_kfree_skb_any(tx_buffer->skb); 6354 tx_buffer->skb = NULL; 6355 6356 tx_ring->next_to_use = i; 6357 6358 return -1; 6359 } 6360 6361 int igb_xmit_xdp_ring(struct igb_adapter *adapter, 6362 struct igb_ring *tx_ring, 6363 struct xdp_frame *xdpf) 6364 { 6365 struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); 6366 u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; 6367 u16 count, i, index = tx_ring->next_to_use; 6368 struct igb_tx_buffer *tx_head = &tx_ring->tx_buffer_info[index]; 6369 struct igb_tx_buffer *tx_buffer = tx_head; 6370 union e1000_adv_tx_desc *tx_desc = IGB_TX_DESC(tx_ring, index); 6371 u32 len = xdpf->len, cmd_type, olinfo_status; 6372 void *data = xdpf->data; 6373 6374 count = TXD_USE_COUNT(len); 6375 for (i = 0; i < nr_frags; i++) 6376 count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i])); 6377 6378 if (igb_maybe_stop_tx(tx_ring, count + 3)) 6379 return IGB_XDP_CONSUMED; 6380 6381 i = 0; 6382 /* record the location of the first descriptor for this packet */ 6383 tx_head->bytecount = xdp_get_frame_len(xdpf); 6384 tx_head->type = IGB_TYPE_XDP; 6385 tx_head->gso_segs = 1; 6386 tx_head->xdpf = xdpf; 6387 6388 olinfo_status = tx_head->bytecount << E1000_ADVTXD_PAYLEN_SHIFT; 6389 /* 82575 requires a unique index per ring */ 6390 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 6391 olinfo_status |= tx_ring->reg_idx << 4; 6392 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 6393 6394 for (;;) { 6395 dma_addr_t dma; 6396 6397 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 6398 if (dma_mapping_error(tx_ring->dev, dma)) 6399 goto unmap; 6400 6401 /* record length, and DMA address */ 6402 dma_unmap_len_set(tx_buffer, len, len); 6403 dma_unmap_addr_set(tx_buffer, dma, dma); 6404 6405 /* put descriptor type bits */ 6406 cmd_type = E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_DEXT | 6407 E1000_ADVTXD_DCMD_IFCS | len; 6408 6409 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 6410 tx_desc->read.buffer_addr = cpu_to_le64(dma); 6411 6412 tx_buffer->protocol = 0; 6413 6414 if (++index == tx_ring->count) 6415 index = 0; 6416 6417 if (i == nr_frags) 6418 break; 6419 6420 tx_buffer = &tx_ring->tx_buffer_info[index]; 6421 tx_desc = IGB_TX_DESC(tx_ring, index); 6422 tx_desc->read.olinfo_status = 0; 6423 6424 data = skb_frag_address(&sinfo->frags[i]); 6425 len = skb_frag_size(&sinfo->frags[i]); 6426 i++; 6427 } 6428 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_TXD_DCMD); 6429 6430 netdev_tx_sent_queue(txring_txq(tx_ring), tx_head->bytecount); 6431 /* set the timestamp */ 6432 tx_head->time_stamp = jiffies; 6433 6434 /* Avoid any potential race with xdp_xmit and cleanup */ 6435 smp_wmb(); 6436 6437 /* set next_to_watch value indicating a packet is present */ 6438 tx_head->next_to_watch = tx_desc; 6439 tx_ring->next_to_use = index; 6440 6441 /* Make sure there is space in the ring for the next send. */ 6442 igb_maybe_stop_tx(tx_ring, DESC_NEEDED); 6443 6444 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) 6445 writel(index, tx_ring->tail); 6446 6447 return IGB_XDP_TX; 6448 6449 unmap: 6450 for (;;) { 6451 tx_buffer = &tx_ring->tx_buffer_info[index]; 6452 if (dma_unmap_len(tx_buffer, len)) 6453 dma_unmap_page(tx_ring->dev, 6454 dma_unmap_addr(tx_buffer, dma), 6455 dma_unmap_len(tx_buffer, len), 6456 DMA_TO_DEVICE); 6457 dma_unmap_len_set(tx_buffer, len, 0); 6458 if (tx_buffer == tx_head) 6459 break; 6460 6461 if (!index) 6462 index += tx_ring->count; 6463 index--; 6464 } 6465 6466 return IGB_XDP_CONSUMED; 6467 } 6468 6469 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb, 6470 struct igb_ring *tx_ring) 6471 { 6472 struct igb_tx_buffer *first; 6473 int tso; 6474 u32 tx_flags = 0; 6475 unsigned short f; 6476 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 6477 __be16 protocol = vlan_get_protocol(skb); 6478 u8 hdr_len = 0; 6479 6480 /* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD, 6481 * + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD, 6482 * + 2 desc gap to keep tail from touching head, 6483 * + 1 desc for context descriptor, 6484 * otherwise try next time 6485 */ 6486 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 6487 count += TXD_USE_COUNT(skb_frag_size( 6488 &skb_shinfo(skb)->frags[f])); 6489 6490 if (igb_maybe_stop_tx(tx_ring, count + 3)) { 6491 /* this is a hard error */ 6492 return NETDEV_TX_BUSY; 6493 } 6494 6495 /* record the location of the first descriptor for this packet */ 6496 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 6497 first->type = IGB_TYPE_SKB; 6498 first->skb = skb; 6499 first->bytecount = skb->len; 6500 first->gso_segs = 1; 6501 6502 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 6503 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev); 6504 6505 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && 6506 !test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS, 6507 &adapter->state)) { 6508 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 6509 tx_flags |= IGB_TX_FLAGS_TSTAMP; 6510 6511 adapter->ptp_tx_skb = skb_get(skb); 6512 adapter->ptp_tx_start = jiffies; 6513 if (adapter->hw.mac.type == e1000_82576) 6514 schedule_work(&adapter->ptp_tx_work); 6515 } else { 6516 adapter->tx_hwtstamp_skipped++; 6517 } 6518 } 6519 6520 if (skb_vlan_tag_present(skb)) { 6521 tx_flags |= IGB_TX_FLAGS_VLAN; 6522 tx_flags |= (skb_vlan_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT); 6523 } 6524 6525 /* record initial flags and protocol */ 6526 first->tx_flags = tx_flags; 6527 first->protocol = protocol; 6528 6529 tso = igb_tso(tx_ring, first, &hdr_len); 6530 if (tso < 0) 6531 goto out_drop; 6532 else if (!tso) 6533 igb_tx_csum(tx_ring, first); 6534 6535 if (igb_tx_map(tx_ring, first, hdr_len)) 6536 goto cleanup_tx_tstamp; 6537 6538 return NETDEV_TX_OK; 6539 6540 out_drop: 6541 dev_kfree_skb_any(first->skb); 6542 first->skb = NULL; 6543 cleanup_tx_tstamp: 6544 if (unlikely(tx_flags & IGB_TX_FLAGS_TSTAMP)) { 6545 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev); 6546 6547 dev_kfree_skb_any(adapter->ptp_tx_skb); 6548 adapter->ptp_tx_skb = NULL; 6549 if (adapter->hw.mac.type == e1000_82576) 6550 cancel_work_sync(&adapter->ptp_tx_work); 6551 clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state); 6552 } 6553 6554 return NETDEV_TX_OK; 6555 } 6556 6557 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter, 6558 struct sk_buff *skb) 6559 { 6560 unsigned int r_idx = skb->queue_mapping; 6561 6562 if (r_idx >= adapter->num_tx_queues) 6563 r_idx = r_idx % adapter->num_tx_queues; 6564 6565 return adapter->tx_ring[r_idx]; 6566 } 6567 6568 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, 6569 struct net_device *netdev) 6570 { 6571 struct igb_adapter *adapter = netdev_priv(netdev); 6572 6573 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 6574 * in order to meet this minimum size requirement. 6575 */ 6576 if (skb_put_padto(skb, 17)) 6577 return NETDEV_TX_OK; 6578 6579 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb)); 6580 } 6581 6582 /** 6583 * igb_tx_timeout - Respond to a Tx Hang 6584 * @netdev: network interface device structure 6585 * @txqueue: number of the Tx queue that hung (unused) 6586 **/ 6587 static void igb_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue) 6588 { 6589 struct igb_adapter *adapter = netdev_priv(netdev); 6590 struct e1000_hw *hw = &adapter->hw; 6591 6592 /* Do the reset outside of interrupt context */ 6593 adapter->tx_timeout_count++; 6594 6595 if (hw->mac.type >= e1000_82580) 6596 hw->dev_spec._82575.global_device_reset = true; 6597 6598 schedule_work(&adapter->reset_task); 6599 wr32(E1000_EICS, 6600 (adapter->eims_enable_mask & ~adapter->eims_other)); 6601 } 6602 6603 static void igb_reset_task(struct work_struct *work) 6604 { 6605 struct igb_adapter *adapter; 6606 adapter = container_of(work, struct igb_adapter, reset_task); 6607 6608 rtnl_lock(); 6609 /* If we're already down or resetting, just bail */ 6610 if (test_bit(__IGB_DOWN, &adapter->state) || 6611 test_bit(__IGB_RESETTING, &adapter->state)) { 6612 rtnl_unlock(); 6613 return; 6614 } 6615 6616 igb_dump(adapter); 6617 netdev_err(adapter->netdev, "Reset adapter\n"); 6618 igb_reinit_locked(adapter); 6619 rtnl_unlock(); 6620 } 6621 6622 /** 6623 * igb_get_stats64 - Get System Network Statistics 6624 * @netdev: network interface device structure 6625 * @stats: rtnl_link_stats64 pointer 6626 **/ 6627 static void igb_get_stats64(struct net_device *netdev, 6628 struct rtnl_link_stats64 *stats) 6629 { 6630 struct igb_adapter *adapter = netdev_priv(netdev); 6631 6632 spin_lock(&adapter->stats64_lock); 6633 igb_update_stats(adapter); 6634 memcpy(stats, &adapter->stats64, sizeof(*stats)); 6635 spin_unlock(&adapter->stats64_lock); 6636 } 6637 6638 /** 6639 * igb_change_mtu - Change the Maximum Transfer Unit 6640 * @netdev: network interface device structure 6641 * @new_mtu: new value for maximum frame size 6642 * 6643 * Returns 0 on success, negative on failure 6644 **/ 6645 static int igb_change_mtu(struct net_device *netdev, int new_mtu) 6646 { 6647 struct igb_adapter *adapter = netdev_priv(netdev); 6648 int max_frame = new_mtu + IGB_ETH_PKT_HDR_PAD; 6649 6650 if (adapter->xdp_prog) { 6651 int i; 6652 6653 for (i = 0; i < adapter->num_rx_queues; i++) { 6654 struct igb_ring *ring = adapter->rx_ring[i]; 6655 6656 if (max_frame > igb_rx_bufsz(ring)) { 6657 netdev_warn(adapter->netdev, 6658 "Requested MTU size is not supported with XDP. Max frame size is %d\n", 6659 max_frame); 6660 return -EINVAL; 6661 } 6662 } 6663 } 6664 6665 /* adjust max frame to be at least the size of a standard frame */ 6666 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 6667 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 6668 6669 while (test_and_set_bit(__IGB_RESETTING, &adapter->state)) 6670 usleep_range(1000, 2000); 6671 6672 /* igb_down has a dependency on max_frame_size */ 6673 adapter->max_frame_size = max_frame; 6674 6675 if (netif_running(netdev)) 6676 igb_down(adapter); 6677 6678 netdev_dbg(netdev, "changing MTU from %d to %d\n", 6679 netdev->mtu, new_mtu); 6680 netdev->mtu = new_mtu; 6681 6682 if (netif_running(netdev)) 6683 igb_up(adapter); 6684 else 6685 igb_reset(adapter); 6686 6687 clear_bit(__IGB_RESETTING, &adapter->state); 6688 6689 return 0; 6690 } 6691 6692 /** 6693 * igb_update_stats - Update the board statistics counters 6694 * @adapter: board private structure 6695 **/ 6696 void igb_update_stats(struct igb_adapter *adapter) 6697 { 6698 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 6699 struct e1000_hw *hw = &adapter->hw; 6700 struct pci_dev *pdev = adapter->pdev; 6701 u32 reg, mpc; 6702 int i; 6703 u64 bytes, packets; 6704 unsigned int start; 6705 u64 _bytes, _packets; 6706 6707 /* Prevent stats update while adapter is being reset, or if the pci 6708 * connection is down. 6709 */ 6710 if (adapter->link_speed == 0) 6711 return; 6712 if (pci_channel_offline(pdev)) 6713 return; 6714 6715 bytes = 0; 6716 packets = 0; 6717 6718 rcu_read_lock(); 6719 for (i = 0; i < adapter->num_rx_queues; i++) { 6720 struct igb_ring *ring = adapter->rx_ring[i]; 6721 u32 rqdpc = rd32(E1000_RQDPC(i)); 6722 if (hw->mac.type >= e1000_i210) 6723 wr32(E1000_RQDPC(i), 0); 6724 6725 if (rqdpc) { 6726 ring->rx_stats.drops += rqdpc; 6727 net_stats->rx_fifo_errors += rqdpc; 6728 } 6729 6730 do { 6731 start = u64_stats_fetch_begin(&ring->rx_syncp); 6732 _bytes = ring->rx_stats.bytes; 6733 _packets = ring->rx_stats.packets; 6734 } while (u64_stats_fetch_retry(&ring->rx_syncp, start)); 6735 bytes += _bytes; 6736 packets += _packets; 6737 } 6738 6739 net_stats->rx_bytes = bytes; 6740 net_stats->rx_packets = packets; 6741 6742 bytes = 0; 6743 packets = 0; 6744 for (i = 0; i < adapter->num_tx_queues; i++) { 6745 struct igb_ring *ring = adapter->tx_ring[i]; 6746 do { 6747 start = u64_stats_fetch_begin(&ring->tx_syncp); 6748 _bytes = ring->tx_stats.bytes; 6749 _packets = ring->tx_stats.packets; 6750 } while (u64_stats_fetch_retry(&ring->tx_syncp, start)); 6751 bytes += _bytes; 6752 packets += _packets; 6753 } 6754 net_stats->tx_bytes = bytes; 6755 net_stats->tx_packets = packets; 6756 rcu_read_unlock(); 6757 6758 /* read stats registers */ 6759 adapter->stats.crcerrs += rd32(E1000_CRCERRS); 6760 adapter->stats.gprc += rd32(E1000_GPRC); 6761 adapter->stats.gorc += rd32(E1000_GORCL); 6762 rd32(E1000_GORCH); /* clear GORCL */ 6763 adapter->stats.bprc += rd32(E1000_BPRC); 6764 adapter->stats.mprc += rd32(E1000_MPRC); 6765 adapter->stats.roc += rd32(E1000_ROC); 6766 6767 adapter->stats.prc64 += rd32(E1000_PRC64); 6768 adapter->stats.prc127 += rd32(E1000_PRC127); 6769 adapter->stats.prc255 += rd32(E1000_PRC255); 6770 adapter->stats.prc511 += rd32(E1000_PRC511); 6771 adapter->stats.prc1023 += rd32(E1000_PRC1023); 6772 adapter->stats.prc1522 += rd32(E1000_PRC1522); 6773 adapter->stats.symerrs += rd32(E1000_SYMERRS); 6774 adapter->stats.sec += rd32(E1000_SEC); 6775 6776 mpc = rd32(E1000_MPC); 6777 adapter->stats.mpc += mpc; 6778 net_stats->rx_fifo_errors += mpc; 6779 adapter->stats.scc += rd32(E1000_SCC); 6780 adapter->stats.ecol += rd32(E1000_ECOL); 6781 adapter->stats.mcc += rd32(E1000_MCC); 6782 adapter->stats.latecol += rd32(E1000_LATECOL); 6783 adapter->stats.dc += rd32(E1000_DC); 6784 adapter->stats.rlec += rd32(E1000_RLEC); 6785 adapter->stats.xonrxc += rd32(E1000_XONRXC); 6786 adapter->stats.xontxc += rd32(E1000_XONTXC); 6787 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC); 6788 adapter->stats.xofftxc += rd32(E1000_XOFFTXC); 6789 adapter->stats.fcruc += rd32(E1000_FCRUC); 6790 adapter->stats.gptc += rd32(E1000_GPTC); 6791 adapter->stats.gotc += rd32(E1000_GOTCL); 6792 rd32(E1000_GOTCH); /* clear GOTCL */ 6793 adapter->stats.rnbc += rd32(E1000_RNBC); 6794 adapter->stats.ruc += rd32(E1000_RUC); 6795 adapter->stats.rfc += rd32(E1000_RFC); 6796 adapter->stats.rjc += rd32(E1000_RJC); 6797 adapter->stats.tor += rd32(E1000_TORH); 6798 adapter->stats.tot += rd32(E1000_TOTH); 6799 adapter->stats.tpr += rd32(E1000_TPR); 6800 6801 adapter->stats.ptc64 += rd32(E1000_PTC64); 6802 adapter->stats.ptc127 += rd32(E1000_PTC127); 6803 adapter->stats.ptc255 += rd32(E1000_PTC255); 6804 adapter->stats.ptc511 += rd32(E1000_PTC511); 6805 adapter->stats.ptc1023 += rd32(E1000_PTC1023); 6806 adapter->stats.ptc1522 += rd32(E1000_PTC1522); 6807 6808 adapter->stats.mptc += rd32(E1000_MPTC); 6809 adapter->stats.bptc += rd32(E1000_BPTC); 6810 6811 adapter->stats.tpt += rd32(E1000_TPT); 6812 adapter->stats.colc += rd32(E1000_COLC); 6813 6814 adapter->stats.algnerrc += rd32(E1000_ALGNERRC); 6815 /* read internal phy specific stats */ 6816 reg = rd32(E1000_CTRL_EXT); 6817 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) { 6818 adapter->stats.rxerrc += rd32(E1000_RXERRC); 6819 6820 /* this stat has invalid values on i210/i211 */ 6821 if ((hw->mac.type != e1000_i210) && 6822 (hw->mac.type != e1000_i211)) 6823 adapter->stats.tncrs += rd32(E1000_TNCRS); 6824 } 6825 6826 adapter->stats.tsctc += rd32(E1000_TSCTC); 6827 adapter->stats.tsctfc += rd32(E1000_TSCTFC); 6828 6829 adapter->stats.iac += rd32(E1000_IAC); 6830 adapter->stats.icrxoc += rd32(E1000_ICRXOC); 6831 adapter->stats.icrxptc += rd32(E1000_ICRXPTC); 6832 adapter->stats.icrxatc += rd32(E1000_ICRXATC); 6833 adapter->stats.ictxptc += rd32(E1000_ICTXPTC); 6834 adapter->stats.ictxatc += rd32(E1000_ICTXATC); 6835 adapter->stats.ictxqec += rd32(E1000_ICTXQEC); 6836 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC); 6837 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC); 6838 6839 /* Fill out the OS statistics structure */ 6840 net_stats->multicast = adapter->stats.mprc; 6841 net_stats->collisions = adapter->stats.colc; 6842 6843 /* Rx Errors */ 6844 6845 /* RLEC on some newer hardware can be incorrect so build 6846 * our own version based on RUC and ROC 6847 */ 6848 net_stats->rx_errors = adapter->stats.rxerrc + 6849 adapter->stats.crcerrs + adapter->stats.algnerrc + 6850 adapter->stats.ruc + adapter->stats.roc + 6851 adapter->stats.cexterr; 6852 net_stats->rx_length_errors = adapter->stats.ruc + 6853 adapter->stats.roc; 6854 net_stats->rx_crc_errors = adapter->stats.crcerrs; 6855 net_stats->rx_frame_errors = adapter->stats.algnerrc; 6856 net_stats->rx_missed_errors = adapter->stats.mpc; 6857 6858 /* Tx Errors */ 6859 net_stats->tx_errors = adapter->stats.ecol + 6860 adapter->stats.latecol; 6861 net_stats->tx_aborted_errors = adapter->stats.ecol; 6862 net_stats->tx_window_errors = adapter->stats.latecol; 6863 net_stats->tx_carrier_errors = adapter->stats.tncrs; 6864 6865 /* Tx Dropped needs to be maintained elsewhere */ 6866 6867 /* Management Stats */ 6868 adapter->stats.mgptc += rd32(E1000_MGTPTC); 6869 adapter->stats.mgprc += rd32(E1000_MGTPRC); 6870 adapter->stats.mgpdc += rd32(E1000_MGTPDC); 6871 6872 /* OS2BMC Stats */ 6873 reg = rd32(E1000_MANC); 6874 if (reg & E1000_MANC_EN_BMC2OS) { 6875 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC); 6876 adapter->stats.o2bspc += rd32(E1000_O2BSPC); 6877 adapter->stats.b2ospc += rd32(E1000_B2OSPC); 6878 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC); 6879 } 6880 } 6881 6882 static void igb_perout(struct igb_adapter *adapter, int tsintr_tt) 6883 { 6884 int pin = ptp_find_pin(adapter->ptp_clock, PTP_PF_PEROUT, tsintr_tt); 6885 struct e1000_hw *hw = &adapter->hw; 6886 struct timespec64 ts; 6887 u32 tsauxc; 6888 6889 if (pin < 0 || pin >= IGB_N_SDP) 6890 return; 6891 6892 spin_lock(&adapter->tmreg_lock); 6893 6894 if (hw->mac.type == e1000_82580 || 6895 hw->mac.type == e1000_i354 || 6896 hw->mac.type == e1000_i350) { 6897 s64 ns = timespec64_to_ns(&adapter->perout[tsintr_tt].period); 6898 u32 systiml, systimh, level_mask, level, rem; 6899 u64 systim, now; 6900 6901 /* read systim registers in sequence */ 6902 rd32(E1000_SYSTIMR); 6903 systiml = rd32(E1000_SYSTIML); 6904 systimh = rd32(E1000_SYSTIMH); 6905 systim = (((u64)(systimh & 0xFF)) << 32) | ((u64)systiml); 6906 now = timecounter_cyc2time(&adapter->tc, systim); 6907 6908 if (pin < 2) { 6909 level_mask = (tsintr_tt == 1) ? 0x80000 : 0x40000; 6910 level = (rd32(E1000_CTRL) & level_mask) ? 1 : 0; 6911 } else { 6912 level_mask = (tsintr_tt == 1) ? 0x80 : 0x40; 6913 level = (rd32(E1000_CTRL_EXT) & level_mask) ? 1 : 0; 6914 } 6915 6916 div_u64_rem(now, ns, &rem); 6917 systim = systim + (ns - rem); 6918 6919 /* synchronize pin level with rising/falling edges */ 6920 div_u64_rem(now, ns << 1, &rem); 6921 if (rem < ns) { 6922 /* first half of period */ 6923 if (level == 0) { 6924 /* output is already low, skip this period */ 6925 systim += ns; 6926 pr_notice("igb: periodic output on %s missed falling edge\n", 6927 adapter->sdp_config[pin].name); 6928 } 6929 } else { 6930 /* second half of period */ 6931 if (level == 1) { 6932 /* output is already high, skip this period */ 6933 systim += ns; 6934 pr_notice("igb: periodic output on %s missed rising edge\n", 6935 adapter->sdp_config[pin].name); 6936 } 6937 } 6938 6939 /* for this chip family tv_sec is the upper part of the binary value, 6940 * so not seconds 6941 */ 6942 ts.tv_nsec = (u32)systim; 6943 ts.tv_sec = ((u32)(systim >> 32)) & 0xFF; 6944 } else { 6945 ts = timespec64_add(adapter->perout[tsintr_tt].start, 6946 adapter->perout[tsintr_tt].period); 6947 } 6948 6949 /* u32 conversion of tv_sec is safe until y2106 */ 6950 wr32((tsintr_tt == 1) ? E1000_TRGTTIML1 : E1000_TRGTTIML0, ts.tv_nsec); 6951 wr32((tsintr_tt == 1) ? E1000_TRGTTIMH1 : E1000_TRGTTIMH0, (u32)ts.tv_sec); 6952 tsauxc = rd32(E1000_TSAUXC); 6953 tsauxc |= TSAUXC_EN_TT0; 6954 wr32(E1000_TSAUXC, tsauxc); 6955 adapter->perout[tsintr_tt].start = ts; 6956 6957 spin_unlock(&adapter->tmreg_lock); 6958 } 6959 6960 static void igb_extts(struct igb_adapter *adapter, int tsintr_tt) 6961 { 6962 int pin = ptp_find_pin(adapter->ptp_clock, PTP_PF_EXTTS, tsintr_tt); 6963 int auxstmpl = (tsintr_tt == 1) ? E1000_AUXSTMPL1 : E1000_AUXSTMPL0; 6964 int auxstmph = (tsintr_tt == 1) ? E1000_AUXSTMPH1 : E1000_AUXSTMPH0; 6965 struct e1000_hw *hw = &adapter->hw; 6966 struct ptp_clock_event event; 6967 struct timespec64 ts; 6968 unsigned long flags; 6969 6970 if (pin < 0 || pin >= IGB_N_SDP) 6971 return; 6972 6973 if (hw->mac.type == e1000_82580 || 6974 hw->mac.type == e1000_i354 || 6975 hw->mac.type == e1000_i350) { 6976 u64 ns = rd32(auxstmpl); 6977 6978 ns += ((u64)(rd32(auxstmph) & 0xFF)) << 32; 6979 spin_lock_irqsave(&adapter->tmreg_lock, flags); 6980 ns = timecounter_cyc2time(&adapter->tc, ns); 6981 spin_unlock_irqrestore(&adapter->tmreg_lock, flags); 6982 ts = ns_to_timespec64(ns); 6983 } else { 6984 ts.tv_nsec = rd32(auxstmpl); 6985 ts.tv_sec = rd32(auxstmph); 6986 } 6987 6988 event.type = PTP_CLOCK_EXTTS; 6989 event.index = tsintr_tt; 6990 event.timestamp = ts.tv_sec * 1000000000ULL + ts.tv_nsec; 6991 ptp_clock_event(adapter->ptp_clock, &event); 6992 } 6993 6994 static void igb_tsync_interrupt(struct igb_adapter *adapter) 6995 { 6996 const u32 mask = (TSINTR_SYS_WRAP | E1000_TSICR_TXTS | 6997 TSINTR_TT0 | TSINTR_TT1 | 6998 TSINTR_AUTT0 | TSINTR_AUTT1); 6999 struct e1000_hw *hw = &adapter->hw; 7000 u32 tsicr = rd32(E1000_TSICR); 7001 struct ptp_clock_event event; 7002 7003 if (hw->mac.type == e1000_82580) { 7004 /* 82580 has a hardware bug that requires an explicit 7005 * write to clear the TimeSync interrupt cause. 7006 */ 7007 wr32(E1000_TSICR, tsicr & mask); 7008 } 7009 7010 if (tsicr & TSINTR_SYS_WRAP) { 7011 event.type = PTP_CLOCK_PPS; 7012 if (adapter->ptp_caps.pps) 7013 ptp_clock_event(adapter->ptp_clock, &event); 7014 } 7015 7016 if (tsicr & E1000_TSICR_TXTS) { 7017 /* retrieve hardware timestamp */ 7018 schedule_work(&adapter->ptp_tx_work); 7019 } 7020 7021 if (tsicr & TSINTR_TT0) 7022 igb_perout(adapter, 0); 7023 7024 if (tsicr & TSINTR_TT1) 7025 igb_perout(adapter, 1); 7026 7027 if (tsicr & TSINTR_AUTT0) 7028 igb_extts(adapter, 0); 7029 7030 if (tsicr & TSINTR_AUTT1) 7031 igb_extts(adapter, 1); 7032 } 7033 7034 static irqreturn_t igb_msix_other(int irq, void *data) 7035 { 7036 struct igb_adapter *adapter = data; 7037 struct e1000_hw *hw = &adapter->hw; 7038 u32 icr = rd32(E1000_ICR); 7039 /* reading ICR causes bit 31 of EICR to be cleared */ 7040 7041 if (icr & E1000_ICR_DRSTA) 7042 schedule_work(&adapter->reset_task); 7043 7044 if (icr & E1000_ICR_DOUTSYNC) { 7045 /* HW is reporting DMA is out of sync */ 7046 adapter->stats.doosync++; 7047 /* The DMA Out of Sync is also indication of a spoof event 7048 * in IOV mode. Check the Wrong VM Behavior register to 7049 * see if it is really a spoof event. 7050 */ 7051 igb_check_wvbr(adapter); 7052 } 7053 7054 /* Check for a mailbox event */ 7055 if (icr & E1000_ICR_VMMB) 7056 igb_msg_task(adapter); 7057 7058 if (icr & E1000_ICR_LSC) { 7059 hw->mac.get_link_status = 1; 7060 /* guard against interrupt when we're going down */ 7061 if (!test_bit(__IGB_DOWN, &adapter->state)) 7062 mod_timer(&adapter->watchdog_timer, jiffies + 1); 7063 } 7064 7065 if (icr & E1000_ICR_TS) 7066 igb_tsync_interrupt(adapter); 7067 7068 wr32(E1000_EIMS, adapter->eims_other); 7069 7070 return IRQ_HANDLED; 7071 } 7072 7073 static void igb_write_itr(struct igb_q_vector *q_vector) 7074 { 7075 struct igb_adapter *adapter = q_vector->adapter; 7076 u32 itr_val = q_vector->itr_val & 0x7FFC; 7077 7078 if (!q_vector->set_itr) 7079 return; 7080 7081 if (!itr_val) 7082 itr_val = 0x4; 7083 7084 if (adapter->hw.mac.type == e1000_82575) 7085 itr_val |= itr_val << 16; 7086 else 7087 itr_val |= E1000_EITR_CNT_IGNR; 7088 7089 writel(itr_val, q_vector->itr_register); 7090 q_vector->set_itr = 0; 7091 } 7092 7093 static irqreturn_t igb_msix_ring(int irq, void *data) 7094 { 7095 struct igb_q_vector *q_vector = data; 7096 7097 /* Write the ITR value calculated from the previous interrupt. */ 7098 igb_write_itr(q_vector); 7099 7100 napi_schedule(&q_vector->napi); 7101 7102 return IRQ_HANDLED; 7103 } 7104 7105 #ifdef CONFIG_IGB_DCA 7106 static void igb_update_tx_dca(struct igb_adapter *adapter, 7107 struct igb_ring *tx_ring, 7108 int cpu) 7109 { 7110 struct e1000_hw *hw = &adapter->hw; 7111 u32 txctrl = dca3_get_tag(tx_ring->dev, cpu); 7112 7113 if (hw->mac.type != e1000_82575) 7114 txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT; 7115 7116 /* We can enable relaxed ordering for reads, but not writes when 7117 * DCA is enabled. This is due to a known issue in some chipsets 7118 * which will cause the DCA tag to be cleared. 7119 */ 7120 txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN | 7121 E1000_DCA_TXCTRL_DATA_RRO_EN | 7122 E1000_DCA_TXCTRL_DESC_DCA_EN; 7123 7124 wr32(E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl); 7125 } 7126 7127 static void igb_update_rx_dca(struct igb_adapter *adapter, 7128 struct igb_ring *rx_ring, 7129 int cpu) 7130 { 7131 struct e1000_hw *hw = &adapter->hw; 7132 u32 rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu); 7133 7134 if (hw->mac.type != e1000_82575) 7135 rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT; 7136 7137 /* We can enable relaxed ordering for reads, but not writes when 7138 * DCA is enabled. This is due to a known issue in some chipsets 7139 * which will cause the DCA tag to be cleared. 7140 */ 7141 rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN | 7142 E1000_DCA_RXCTRL_DESC_DCA_EN; 7143 7144 wr32(E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl); 7145 } 7146 7147 static void igb_update_dca(struct igb_q_vector *q_vector) 7148 { 7149 struct igb_adapter *adapter = q_vector->adapter; 7150 int cpu = get_cpu(); 7151 7152 if (q_vector->cpu == cpu) 7153 goto out_no_update; 7154 7155 if (q_vector->tx.ring) 7156 igb_update_tx_dca(adapter, q_vector->tx.ring, cpu); 7157 7158 if (q_vector->rx.ring) 7159 igb_update_rx_dca(adapter, q_vector->rx.ring, cpu); 7160 7161 q_vector->cpu = cpu; 7162 out_no_update: 7163 put_cpu(); 7164 } 7165 7166 static void igb_setup_dca(struct igb_adapter *adapter) 7167 { 7168 struct e1000_hw *hw = &adapter->hw; 7169 int i; 7170 7171 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED)) 7172 return; 7173 7174 /* Always use CB2 mode, difference is masked in the CB driver. */ 7175 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2); 7176 7177 for (i = 0; i < adapter->num_q_vectors; i++) { 7178 adapter->q_vector[i]->cpu = -1; 7179 igb_update_dca(adapter->q_vector[i]); 7180 } 7181 } 7182 7183 static int __igb_notify_dca(struct device *dev, void *data) 7184 { 7185 struct net_device *netdev = dev_get_drvdata(dev); 7186 struct igb_adapter *adapter = netdev_priv(netdev); 7187 struct pci_dev *pdev = adapter->pdev; 7188 struct e1000_hw *hw = &adapter->hw; 7189 unsigned long event = *(unsigned long *)data; 7190 7191 switch (event) { 7192 case DCA_PROVIDER_ADD: 7193 /* if already enabled, don't do it again */ 7194 if (adapter->flags & IGB_FLAG_DCA_ENABLED) 7195 break; 7196 if (dca_add_requester(dev) == 0) { 7197 adapter->flags |= IGB_FLAG_DCA_ENABLED; 7198 dev_info(&pdev->dev, "DCA enabled\n"); 7199 igb_setup_dca(adapter); 7200 break; 7201 } 7202 fallthrough; /* since DCA is disabled. */ 7203 case DCA_PROVIDER_REMOVE: 7204 if (adapter->flags & IGB_FLAG_DCA_ENABLED) { 7205 /* without this a class_device is left 7206 * hanging around in the sysfs model 7207 */ 7208 dca_remove_requester(dev); 7209 dev_info(&pdev->dev, "DCA disabled\n"); 7210 adapter->flags &= ~IGB_FLAG_DCA_ENABLED; 7211 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE); 7212 } 7213 break; 7214 } 7215 7216 return 0; 7217 } 7218 7219 static int igb_notify_dca(struct notifier_block *nb, unsigned long event, 7220 void *p) 7221 { 7222 int ret_val; 7223 7224 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event, 7225 __igb_notify_dca); 7226 7227 return ret_val ? NOTIFY_BAD : NOTIFY_DONE; 7228 } 7229 #endif /* CONFIG_IGB_DCA */ 7230 7231 #ifdef CONFIG_PCI_IOV 7232 static int igb_vf_configure(struct igb_adapter *adapter, int vf) 7233 { 7234 unsigned char mac_addr[ETH_ALEN]; 7235 7236 eth_zero_addr(mac_addr); 7237 igb_set_vf_mac(adapter, vf, mac_addr); 7238 7239 /* By default spoof check is enabled for all VFs */ 7240 adapter->vf_data[vf].spoofchk_enabled = true; 7241 7242 /* By default VFs are not trusted */ 7243 adapter->vf_data[vf].trusted = false; 7244 7245 return 0; 7246 } 7247 7248 #endif 7249 static void igb_ping_all_vfs(struct igb_adapter *adapter) 7250 { 7251 struct e1000_hw *hw = &adapter->hw; 7252 u32 ping; 7253 int i; 7254 7255 for (i = 0 ; i < adapter->vfs_allocated_count; i++) { 7256 ping = E1000_PF_CONTROL_MSG; 7257 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS) 7258 ping |= E1000_VT_MSGTYPE_CTS; 7259 igb_write_mbx(hw, &ping, 1, i); 7260 } 7261 } 7262 7263 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf) 7264 { 7265 struct e1000_hw *hw = &adapter->hw; 7266 u32 vmolr = rd32(E1000_VMOLR(vf)); 7267 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 7268 7269 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC | 7270 IGB_VF_FLAG_MULTI_PROMISC); 7271 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME); 7272 7273 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) { 7274 vmolr |= E1000_VMOLR_MPME; 7275 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC; 7276 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST; 7277 } else { 7278 /* if we have hashes and we are clearing a multicast promisc 7279 * flag we need to write the hashes to the MTA as this step 7280 * was previously skipped 7281 */ 7282 if (vf_data->num_vf_mc_hashes > 30) { 7283 vmolr |= E1000_VMOLR_MPME; 7284 } else if (vf_data->num_vf_mc_hashes) { 7285 int j; 7286 7287 vmolr |= E1000_VMOLR_ROMPE; 7288 for (j = 0; j < vf_data->num_vf_mc_hashes; j++) 7289 igb_mta_set(hw, vf_data->vf_mc_hashes[j]); 7290 } 7291 } 7292 7293 wr32(E1000_VMOLR(vf), vmolr); 7294 7295 /* there are flags left unprocessed, likely not supported */ 7296 if (*msgbuf & E1000_VT_MSGINFO_MASK) 7297 return -EINVAL; 7298 7299 return 0; 7300 } 7301 7302 static int igb_set_vf_multicasts(struct igb_adapter *adapter, 7303 u32 *msgbuf, u32 vf) 7304 { 7305 int n = FIELD_GET(E1000_VT_MSGINFO_MASK, msgbuf[0]); 7306 u16 *hash_list = (u16 *)&msgbuf[1]; 7307 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 7308 int i; 7309 7310 /* salt away the number of multicast addresses assigned 7311 * to this VF for later use to restore when the PF multi cast 7312 * list changes 7313 */ 7314 vf_data->num_vf_mc_hashes = n; 7315 7316 /* only up to 30 hash values supported */ 7317 if (n > 30) 7318 n = 30; 7319 7320 /* store the hashes for later use */ 7321 for (i = 0; i < n; i++) 7322 vf_data->vf_mc_hashes[i] = hash_list[i]; 7323 7324 /* Flush and reset the mta with the new values */ 7325 igb_set_rx_mode(adapter->netdev); 7326 7327 return 0; 7328 } 7329 7330 static void igb_restore_vf_multicasts(struct igb_adapter *adapter) 7331 { 7332 struct e1000_hw *hw = &adapter->hw; 7333 struct vf_data_storage *vf_data; 7334 int i, j; 7335 7336 for (i = 0; i < adapter->vfs_allocated_count; i++) { 7337 u32 vmolr = rd32(E1000_VMOLR(i)); 7338 7339 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME); 7340 7341 vf_data = &adapter->vf_data[i]; 7342 7343 if ((vf_data->num_vf_mc_hashes > 30) || 7344 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) { 7345 vmolr |= E1000_VMOLR_MPME; 7346 } else if (vf_data->num_vf_mc_hashes) { 7347 vmolr |= E1000_VMOLR_ROMPE; 7348 for (j = 0; j < vf_data->num_vf_mc_hashes; j++) 7349 igb_mta_set(hw, vf_data->vf_mc_hashes[j]); 7350 } 7351 wr32(E1000_VMOLR(i), vmolr); 7352 } 7353 } 7354 7355 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf) 7356 { 7357 struct e1000_hw *hw = &adapter->hw; 7358 u32 pool_mask, vlvf_mask, i; 7359 7360 /* create mask for VF and other pools */ 7361 pool_mask = E1000_VLVF_POOLSEL_MASK; 7362 vlvf_mask = BIT(E1000_VLVF_POOLSEL_SHIFT + vf); 7363 7364 /* drop PF from pool bits */ 7365 pool_mask &= ~BIT(E1000_VLVF_POOLSEL_SHIFT + 7366 adapter->vfs_allocated_count); 7367 7368 /* Find the vlan filter for this id */ 7369 for (i = E1000_VLVF_ARRAY_SIZE; i--;) { 7370 u32 vlvf = rd32(E1000_VLVF(i)); 7371 u32 vfta_mask, vid, vfta; 7372 7373 /* remove the vf from the pool */ 7374 if (!(vlvf & vlvf_mask)) 7375 continue; 7376 7377 /* clear out bit from VLVF */ 7378 vlvf ^= vlvf_mask; 7379 7380 /* if other pools are present, just remove ourselves */ 7381 if (vlvf & pool_mask) 7382 goto update_vlvfb; 7383 7384 /* if PF is present, leave VFTA */ 7385 if (vlvf & E1000_VLVF_POOLSEL_MASK) 7386 goto update_vlvf; 7387 7388 vid = vlvf & E1000_VLVF_VLANID_MASK; 7389 vfta_mask = BIT(vid % 32); 7390 7391 /* clear bit from VFTA */ 7392 vfta = adapter->shadow_vfta[vid / 32]; 7393 if (vfta & vfta_mask) 7394 hw->mac.ops.write_vfta(hw, vid / 32, vfta ^ vfta_mask); 7395 update_vlvf: 7396 /* clear pool selection enable */ 7397 if (adapter->flags & IGB_FLAG_VLAN_PROMISC) 7398 vlvf &= E1000_VLVF_POOLSEL_MASK; 7399 else 7400 vlvf = 0; 7401 update_vlvfb: 7402 /* clear pool bits */ 7403 wr32(E1000_VLVF(i), vlvf); 7404 } 7405 } 7406 7407 static int igb_find_vlvf_entry(struct e1000_hw *hw, u32 vlan) 7408 { 7409 u32 vlvf; 7410 int idx; 7411 7412 /* short cut the special case */ 7413 if (vlan == 0) 7414 return 0; 7415 7416 /* Search for the VLAN id in the VLVF entries */ 7417 for (idx = E1000_VLVF_ARRAY_SIZE; --idx;) { 7418 vlvf = rd32(E1000_VLVF(idx)); 7419 if ((vlvf & VLAN_VID_MASK) == vlan) 7420 break; 7421 } 7422 7423 return idx; 7424 } 7425 7426 static void igb_update_pf_vlvf(struct igb_adapter *adapter, u32 vid) 7427 { 7428 struct e1000_hw *hw = &adapter->hw; 7429 u32 bits, pf_id; 7430 int idx; 7431 7432 idx = igb_find_vlvf_entry(hw, vid); 7433 if (!idx) 7434 return; 7435 7436 /* See if any other pools are set for this VLAN filter 7437 * entry other than the PF. 7438 */ 7439 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT; 7440 bits = ~BIT(pf_id) & E1000_VLVF_POOLSEL_MASK; 7441 bits &= rd32(E1000_VLVF(idx)); 7442 7443 /* Disable the filter so this falls into the default pool. */ 7444 if (!bits) { 7445 if (adapter->flags & IGB_FLAG_VLAN_PROMISC) 7446 wr32(E1000_VLVF(idx), BIT(pf_id)); 7447 else 7448 wr32(E1000_VLVF(idx), 0); 7449 } 7450 } 7451 7452 static s32 igb_set_vf_vlan(struct igb_adapter *adapter, u32 vid, 7453 bool add, u32 vf) 7454 { 7455 int pf_id = adapter->vfs_allocated_count; 7456 struct e1000_hw *hw = &adapter->hw; 7457 int err; 7458 7459 /* If VLAN overlaps with one the PF is currently monitoring make 7460 * sure that we are able to allocate a VLVF entry. This may be 7461 * redundant but it guarantees PF will maintain visibility to 7462 * the VLAN. 7463 */ 7464 if (add && test_bit(vid, adapter->active_vlans)) { 7465 err = igb_vfta_set(hw, vid, pf_id, true, false); 7466 if (err) 7467 return err; 7468 } 7469 7470 err = igb_vfta_set(hw, vid, vf, add, false); 7471 7472 if (add && !err) 7473 return err; 7474 7475 /* If we failed to add the VF VLAN or we are removing the VF VLAN 7476 * we may need to drop the PF pool bit in order to allow us to free 7477 * up the VLVF resources. 7478 */ 7479 if (test_bit(vid, adapter->active_vlans) || 7480 (adapter->flags & IGB_FLAG_VLAN_PROMISC)) 7481 igb_update_pf_vlvf(adapter, vid); 7482 7483 return err; 7484 } 7485 7486 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf) 7487 { 7488 struct e1000_hw *hw = &adapter->hw; 7489 7490 if (vid) 7491 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT)); 7492 else 7493 wr32(E1000_VMVIR(vf), 0); 7494 } 7495 7496 static int igb_enable_port_vlan(struct igb_adapter *adapter, int vf, 7497 u16 vlan, u8 qos) 7498 { 7499 int err; 7500 7501 err = igb_set_vf_vlan(adapter, vlan, true, vf); 7502 if (err) 7503 return err; 7504 7505 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf); 7506 igb_set_vmolr(adapter, vf, !vlan); 7507 7508 /* revoke access to previous VLAN */ 7509 if (vlan != adapter->vf_data[vf].pf_vlan) 7510 igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan, 7511 false, vf); 7512 7513 adapter->vf_data[vf].pf_vlan = vlan; 7514 adapter->vf_data[vf].pf_qos = qos; 7515 igb_set_vf_vlan_strip(adapter, vf, true); 7516 dev_info(&adapter->pdev->dev, 7517 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf); 7518 if (test_bit(__IGB_DOWN, &adapter->state)) { 7519 dev_warn(&adapter->pdev->dev, 7520 "The VF VLAN has been set, but the PF device is not up.\n"); 7521 dev_warn(&adapter->pdev->dev, 7522 "Bring the PF device up before attempting to use the VF device.\n"); 7523 } 7524 7525 return err; 7526 } 7527 7528 static int igb_disable_port_vlan(struct igb_adapter *adapter, int vf) 7529 { 7530 /* Restore tagless access via VLAN 0 */ 7531 igb_set_vf_vlan(adapter, 0, true, vf); 7532 7533 igb_set_vmvir(adapter, 0, vf); 7534 igb_set_vmolr(adapter, vf, true); 7535 7536 /* Remove any PF assigned VLAN */ 7537 if (adapter->vf_data[vf].pf_vlan) 7538 igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan, 7539 false, vf); 7540 7541 adapter->vf_data[vf].pf_vlan = 0; 7542 adapter->vf_data[vf].pf_qos = 0; 7543 igb_set_vf_vlan_strip(adapter, vf, false); 7544 7545 return 0; 7546 } 7547 7548 static int igb_ndo_set_vf_vlan(struct net_device *netdev, int vf, 7549 u16 vlan, u8 qos, __be16 vlan_proto) 7550 { 7551 struct igb_adapter *adapter = netdev_priv(netdev); 7552 7553 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7)) 7554 return -EINVAL; 7555 7556 if (vlan_proto != htons(ETH_P_8021Q)) 7557 return -EPROTONOSUPPORT; 7558 7559 return (vlan || qos) ? igb_enable_port_vlan(adapter, vf, vlan, qos) : 7560 igb_disable_port_vlan(adapter, vf); 7561 } 7562 7563 static int igb_set_vf_vlan_msg(struct igb_adapter *adapter, u32 *msgbuf, u32 vf) 7564 { 7565 int add = FIELD_GET(E1000_VT_MSGINFO_MASK, msgbuf[0]); 7566 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK); 7567 int ret; 7568 7569 if (adapter->vf_data[vf].pf_vlan) 7570 return -1; 7571 7572 /* VLAN 0 is a special case, don't allow it to be removed */ 7573 if (!vid && !add) 7574 return 0; 7575 7576 ret = igb_set_vf_vlan(adapter, vid, !!add, vf); 7577 if (!ret) 7578 igb_set_vf_vlan_strip(adapter, vf, !!vid); 7579 return ret; 7580 } 7581 7582 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf) 7583 { 7584 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 7585 7586 /* clear flags - except flag that indicates PF has set the MAC */ 7587 vf_data->flags &= IGB_VF_FLAG_PF_SET_MAC; 7588 vf_data->last_nack = jiffies; 7589 7590 /* reset vlans for device */ 7591 igb_clear_vf_vfta(adapter, vf); 7592 igb_set_vf_vlan(adapter, vf_data->pf_vlan, true, vf); 7593 igb_set_vmvir(adapter, vf_data->pf_vlan | 7594 (vf_data->pf_qos << VLAN_PRIO_SHIFT), vf); 7595 igb_set_vmolr(adapter, vf, !vf_data->pf_vlan); 7596 igb_set_vf_vlan_strip(adapter, vf, !!(vf_data->pf_vlan)); 7597 7598 /* reset multicast table array for vf */ 7599 adapter->vf_data[vf].num_vf_mc_hashes = 0; 7600 7601 /* Flush and reset the mta with the new values */ 7602 igb_set_rx_mode(adapter->netdev); 7603 } 7604 7605 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf) 7606 { 7607 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses; 7608 7609 /* clear mac address as we were hotplug removed/added */ 7610 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC)) 7611 eth_zero_addr(vf_mac); 7612 7613 /* process remaining reset events */ 7614 igb_vf_reset(adapter, vf); 7615 } 7616 7617 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf) 7618 { 7619 struct e1000_hw *hw = &adapter->hw; 7620 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses; 7621 u32 reg, msgbuf[3] = {}; 7622 u8 *addr = (u8 *)(&msgbuf[1]); 7623 7624 /* process all the same items cleared in a function level reset */ 7625 igb_vf_reset(adapter, vf); 7626 7627 /* set vf mac address */ 7628 igb_set_vf_mac(adapter, vf, vf_mac); 7629 7630 /* enable transmit and receive for vf */ 7631 reg = rd32(E1000_VFTE); 7632 wr32(E1000_VFTE, reg | BIT(vf)); 7633 reg = rd32(E1000_VFRE); 7634 wr32(E1000_VFRE, reg | BIT(vf)); 7635 7636 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS; 7637 7638 /* reply to reset with ack and vf mac address */ 7639 if (!is_zero_ether_addr(vf_mac)) { 7640 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK; 7641 memcpy(addr, vf_mac, ETH_ALEN); 7642 } else { 7643 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK; 7644 } 7645 igb_write_mbx(hw, msgbuf, 3, vf); 7646 } 7647 7648 static void igb_flush_mac_table(struct igb_adapter *adapter) 7649 { 7650 struct e1000_hw *hw = &adapter->hw; 7651 int i; 7652 7653 for (i = 0; i < hw->mac.rar_entry_count; i++) { 7654 adapter->mac_table[i].state &= ~IGB_MAC_STATE_IN_USE; 7655 eth_zero_addr(adapter->mac_table[i].addr); 7656 adapter->mac_table[i].queue = 0; 7657 igb_rar_set_index(adapter, i); 7658 } 7659 } 7660 7661 static int igb_available_rars(struct igb_adapter *adapter, u8 queue) 7662 { 7663 struct e1000_hw *hw = &adapter->hw; 7664 /* do not count rar entries reserved for VFs MAC addresses */ 7665 int rar_entries = hw->mac.rar_entry_count - 7666 adapter->vfs_allocated_count; 7667 int i, count = 0; 7668 7669 for (i = 0; i < rar_entries; i++) { 7670 /* do not count default entries */ 7671 if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) 7672 continue; 7673 7674 /* do not count "in use" entries for different queues */ 7675 if ((adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE) && 7676 (adapter->mac_table[i].queue != queue)) 7677 continue; 7678 7679 count++; 7680 } 7681 7682 return count; 7683 } 7684 7685 /* Set default MAC address for the PF in the first RAR entry */ 7686 static void igb_set_default_mac_filter(struct igb_adapter *adapter) 7687 { 7688 struct igb_mac_addr *mac_table = &adapter->mac_table[0]; 7689 7690 ether_addr_copy(mac_table->addr, adapter->hw.mac.addr); 7691 mac_table->queue = adapter->vfs_allocated_count; 7692 mac_table->state = IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE; 7693 7694 igb_rar_set_index(adapter, 0); 7695 } 7696 7697 /* If the filter to be added and an already existing filter express 7698 * the same address and address type, it should be possible to only 7699 * override the other configurations, for example the queue to steer 7700 * traffic. 7701 */ 7702 static bool igb_mac_entry_can_be_used(const struct igb_mac_addr *entry, 7703 const u8 *addr, const u8 flags) 7704 { 7705 if (!(entry->state & IGB_MAC_STATE_IN_USE)) 7706 return true; 7707 7708 if ((entry->state & IGB_MAC_STATE_SRC_ADDR) != 7709 (flags & IGB_MAC_STATE_SRC_ADDR)) 7710 return false; 7711 7712 if (!ether_addr_equal(addr, entry->addr)) 7713 return false; 7714 7715 return true; 7716 } 7717 7718 /* Add a MAC filter for 'addr' directing matching traffic to 'queue', 7719 * 'flags' is used to indicate what kind of match is made, match is by 7720 * default for the destination address, if matching by source address 7721 * is desired the flag IGB_MAC_STATE_SRC_ADDR can be used. 7722 */ 7723 static int igb_add_mac_filter_flags(struct igb_adapter *adapter, 7724 const u8 *addr, const u8 queue, 7725 const u8 flags) 7726 { 7727 struct e1000_hw *hw = &adapter->hw; 7728 int rar_entries = hw->mac.rar_entry_count - 7729 adapter->vfs_allocated_count; 7730 int i; 7731 7732 if (is_zero_ether_addr(addr)) 7733 return -EINVAL; 7734 7735 /* Search for the first empty entry in the MAC table. 7736 * Do not touch entries at the end of the table reserved for the VF MAC 7737 * addresses. 7738 */ 7739 for (i = 0; i < rar_entries; i++) { 7740 if (!igb_mac_entry_can_be_used(&adapter->mac_table[i], 7741 addr, flags)) 7742 continue; 7743 7744 ether_addr_copy(adapter->mac_table[i].addr, addr); 7745 adapter->mac_table[i].queue = queue; 7746 adapter->mac_table[i].state |= IGB_MAC_STATE_IN_USE | flags; 7747 7748 igb_rar_set_index(adapter, i); 7749 return i; 7750 } 7751 7752 return -ENOSPC; 7753 } 7754 7755 static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr, 7756 const u8 queue) 7757 { 7758 return igb_add_mac_filter_flags(adapter, addr, queue, 0); 7759 } 7760 7761 /* Remove a MAC filter for 'addr' directing matching traffic to 7762 * 'queue', 'flags' is used to indicate what kind of match need to be 7763 * removed, match is by default for the destination address, if 7764 * matching by source address is to be removed the flag 7765 * IGB_MAC_STATE_SRC_ADDR can be used. 7766 */ 7767 static int igb_del_mac_filter_flags(struct igb_adapter *adapter, 7768 const u8 *addr, const u8 queue, 7769 const u8 flags) 7770 { 7771 struct e1000_hw *hw = &adapter->hw; 7772 int rar_entries = hw->mac.rar_entry_count - 7773 adapter->vfs_allocated_count; 7774 int i; 7775 7776 if (is_zero_ether_addr(addr)) 7777 return -EINVAL; 7778 7779 /* Search for matching entry in the MAC table based on given address 7780 * and queue. Do not touch entries at the end of the table reserved 7781 * for the VF MAC addresses. 7782 */ 7783 for (i = 0; i < rar_entries; i++) { 7784 if (!(adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE)) 7785 continue; 7786 if ((adapter->mac_table[i].state & flags) != flags) 7787 continue; 7788 if (adapter->mac_table[i].queue != queue) 7789 continue; 7790 if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) 7791 continue; 7792 7793 /* When a filter for the default address is "deleted", 7794 * we return it to its initial configuration 7795 */ 7796 if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) { 7797 adapter->mac_table[i].state = 7798 IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE; 7799 adapter->mac_table[i].queue = 7800 adapter->vfs_allocated_count; 7801 } else { 7802 adapter->mac_table[i].state = 0; 7803 adapter->mac_table[i].queue = 0; 7804 eth_zero_addr(adapter->mac_table[i].addr); 7805 } 7806 7807 igb_rar_set_index(adapter, i); 7808 return 0; 7809 } 7810 7811 return -ENOENT; 7812 } 7813 7814 static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr, 7815 const u8 queue) 7816 { 7817 return igb_del_mac_filter_flags(adapter, addr, queue, 0); 7818 } 7819 7820 int igb_add_mac_steering_filter(struct igb_adapter *adapter, 7821 const u8 *addr, u8 queue, u8 flags) 7822 { 7823 struct e1000_hw *hw = &adapter->hw; 7824 7825 /* In theory, this should be supported on 82575 as well, but 7826 * that part wasn't easily accessible during development. 7827 */ 7828 if (hw->mac.type != e1000_i210) 7829 return -EOPNOTSUPP; 7830 7831 return igb_add_mac_filter_flags(adapter, addr, queue, 7832 IGB_MAC_STATE_QUEUE_STEERING | flags); 7833 } 7834 7835 int igb_del_mac_steering_filter(struct igb_adapter *adapter, 7836 const u8 *addr, u8 queue, u8 flags) 7837 { 7838 return igb_del_mac_filter_flags(adapter, addr, queue, 7839 IGB_MAC_STATE_QUEUE_STEERING | flags); 7840 } 7841 7842 static int igb_uc_sync(struct net_device *netdev, const unsigned char *addr) 7843 { 7844 struct igb_adapter *adapter = netdev_priv(netdev); 7845 int ret; 7846 7847 ret = igb_add_mac_filter(adapter, addr, adapter->vfs_allocated_count); 7848 7849 return min_t(int, ret, 0); 7850 } 7851 7852 static int igb_uc_unsync(struct net_device *netdev, const unsigned char *addr) 7853 { 7854 struct igb_adapter *adapter = netdev_priv(netdev); 7855 7856 igb_del_mac_filter(adapter, addr, adapter->vfs_allocated_count); 7857 7858 return 0; 7859 } 7860 7861 static int igb_set_vf_mac_filter(struct igb_adapter *adapter, const int vf, 7862 const u32 info, const u8 *addr) 7863 { 7864 struct pci_dev *pdev = adapter->pdev; 7865 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 7866 struct list_head *pos; 7867 struct vf_mac_filter *entry = NULL; 7868 int ret = 0; 7869 7870 if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) && 7871 !vf_data->trusted) { 7872 dev_warn(&pdev->dev, 7873 "VF %d requested MAC filter but is administratively denied\n", 7874 vf); 7875 return -EINVAL; 7876 } 7877 if (!is_valid_ether_addr(addr)) { 7878 dev_warn(&pdev->dev, 7879 "VF %d attempted to set invalid MAC filter\n", 7880 vf); 7881 return -EINVAL; 7882 } 7883 7884 switch (info) { 7885 case E1000_VF_MAC_FILTER_CLR: 7886 /* remove all unicast MAC filters related to the current VF */ 7887 list_for_each(pos, &adapter->vf_macs.l) { 7888 entry = list_entry(pos, struct vf_mac_filter, l); 7889 if (entry->vf == vf) { 7890 entry->vf = -1; 7891 entry->free = true; 7892 igb_del_mac_filter(adapter, entry->vf_mac, vf); 7893 } 7894 } 7895 break; 7896 case E1000_VF_MAC_FILTER_ADD: 7897 /* try to find empty slot in the list */ 7898 list_for_each(pos, &adapter->vf_macs.l) { 7899 entry = list_entry(pos, struct vf_mac_filter, l); 7900 if (entry->free) 7901 break; 7902 } 7903 7904 if (entry && entry->free) { 7905 entry->free = false; 7906 entry->vf = vf; 7907 ether_addr_copy(entry->vf_mac, addr); 7908 7909 ret = igb_add_mac_filter(adapter, addr, vf); 7910 ret = min_t(int, ret, 0); 7911 } else { 7912 ret = -ENOSPC; 7913 } 7914 7915 if (ret == -ENOSPC) 7916 dev_warn(&pdev->dev, 7917 "VF %d has requested MAC filter but there is no space for it\n", 7918 vf); 7919 break; 7920 default: 7921 ret = -EINVAL; 7922 break; 7923 } 7924 7925 return ret; 7926 } 7927 7928 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf) 7929 { 7930 struct pci_dev *pdev = adapter->pdev; 7931 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 7932 u32 info = msg[0] & E1000_VT_MSGINFO_MASK; 7933 7934 /* The VF MAC Address is stored in a packed array of bytes 7935 * starting at the second 32 bit word of the msg array 7936 */ 7937 unsigned char *addr = (unsigned char *)&msg[1]; 7938 int ret = 0; 7939 7940 if (!info) { 7941 if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) && 7942 !vf_data->trusted) { 7943 dev_warn(&pdev->dev, 7944 "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n", 7945 vf); 7946 return -EINVAL; 7947 } 7948 7949 if (!is_valid_ether_addr(addr)) { 7950 dev_warn(&pdev->dev, 7951 "VF %d attempted to set invalid MAC\n", 7952 vf); 7953 return -EINVAL; 7954 } 7955 7956 ret = igb_set_vf_mac(adapter, vf, addr); 7957 } else { 7958 ret = igb_set_vf_mac_filter(adapter, vf, info, addr); 7959 } 7960 7961 return ret; 7962 } 7963 7964 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf) 7965 { 7966 struct e1000_hw *hw = &adapter->hw; 7967 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 7968 u32 msg = E1000_VT_MSGTYPE_NACK; 7969 7970 /* if device isn't clear to send it shouldn't be reading either */ 7971 if (!(vf_data->flags & IGB_VF_FLAG_CTS) && 7972 time_after(jiffies, vf_data->last_nack + (2 * HZ))) { 7973 igb_write_mbx(hw, &msg, 1, vf); 7974 vf_data->last_nack = jiffies; 7975 } 7976 } 7977 7978 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf) 7979 { 7980 struct pci_dev *pdev = adapter->pdev; 7981 u32 msgbuf[E1000_VFMAILBOX_SIZE]; 7982 struct e1000_hw *hw = &adapter->hw; 7983 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 7984 s32 retval; 7985 7986 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf, false); 7987 7988 if (retval) { 7989 /* if receive failed revoke VF CTS stats and restart init */ 7990 dev_err(&pdev->dev, "Error receiving message from VF\n"); 7991 vf_data->flags &= ~IGB_VF_FLAG_CTS; 7992 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ))) 7993 goto unlock; 7994 goto out; 7995 } 7996 7997 /* this is a message we already processed, do nothing */ 7998 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK)) 7999 goto unlock; 8000 8001 /* until the vf completes a reset it should not be 8002 * allowed to start any configuration. 8003 */ 8004 if (msgbuf[0] == E1000_VF_RESET) { 8005 /* unlocks mailbox */ 8006 igb_vf_reset_msg(adapter, vf); 8007 return; 8008 } 8009 8010 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) { 8011 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ))) 8012 goto unlock; 8013 retval = -1; 8014 goto out; 8015 } 8016 8017 switch ((msgbuf[0] & 0xFFFF)) { 8018 case E1000_VF_SET_MAC_ADDR: 8019 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf); 8020 break; 8021 case E1000_VF_SET_PROMISC: 8022 retval = igb_set_vf_promisc(adapter, msgbuf, vf); 8023 break; 8024 case E1000_VF_SET_MULTICAST: 8025 retval = igb_set_vf_multicasts(adapter, msgbuf, vf); 8026 break; 8027 case E1000_VF_SET_LPE: 8028 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf); 8029 break; 8030 case E1000_VF_SET_VLAN: 8031 retval = -1; 8032 if (vf_data->pf_vlan) 8033 dev_warn(&pdev->dev, 8034 "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n", 8035 vf); 8036 else 8037 retval = igb_set_vf_vlan_msg(adapter, msgbuf, vf); 8038 break; 8039 default: 8040 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]); 8041 retval = -1; 8042 break; 8043 } 8044 8045 msgbuf[0] |= E1000_VT_MSGTYPE_CTS; 8046 out: 8047 /* notify the VF of the results of what it sent us */ 8048 if (retval) 8049 msgbuf[0] |= E1000_VT_MSGTYPE_NACK; 8050 else 8051 msgbuf[0] |= E1000_VT_MSGTYPE_ACK; 8052 8053 /* unlocks mailbox */ 8054 igb_write_mbx(hw, msgbuf, 1, vf); 8055 return; 8056 8057 unlock: 8058 igb_unlock_mbx(hw, vf); 8059 } 8060 8061 static void igb_msg_task(struct igb_adapter *adapter) 8062 { 8063 struct e1000_hw *hw = &adapter->hw; 8064 unsigned long flags; 8065 u32 vf; 8066 8067 spin_lock_irqsave(&adapter->vfs_lock, flags); 8068 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) { 8069 /* process any reset requests */ 8070 if (!igb_check_for_rst(hw, vf)) 8071 igb_vf_reset_event(adapter, vf); 8072 8073 /* process any messages pending */ 8074 if (!igb_check_for_msg(hw, vf)) 8075 igb_rcv_msg_from_vf(adapter, vf); 8076 8077 /* process any acks */ 8078 if (!igb_check_for_ack(hw, vf)) 8079 igb_rcv_ack_from_vf(adapter, vf); 8080 } 8081 spin_unlock_irqrestore(&adapter->vfs_lock, flags); 8082 } 8083 8084 /** 8085 * igb_set_uta - Set unicast filter table address 8086 * @adapter: board private structure 8087 * @set: boolean indicating if we are setting or clearing bits 8088 * 8089 * The unicast table address is a register array of 32-bit registers. 8090 * The table is meant to be used in a way similar to how the MTA is used 8091 * however due to certain limitations in the hardware it is necessary to 8092 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous 8093 * enable bit to allow vlan tag stripping when promiscuous mode is enabled 8094 **/ 8095 static void igb_set_uta(struct igb_adapter *adapter, bool set) 8096 { 8097 struct e1000_hw *hw = &adapter->hw; 8098 u32 uta = set ? ~0 : 0; 8099 int i; 8100 8101 /* we only need to do this if VMDq is enabled */ 8102 if (!adapter->vfs_allocated_count) 8103 return; 8104 8105 for (i = hw->mac.uta_reg_count; i--;) 8106 array_wr32(E1000_UTA, i, uta); 8107 } 8108 8109 /** 8110 * igb_intr_msi - Interrupt Handler 8111 * @irq: interrupt number 8112 * @data: pointer to a network interface device structure 8113 **/ 8114 static irqreturn_t igb_intr_msi(int irq, void *data) 8115 { 8116 struct igb_adapter *adapter = data; 8117 struct igb_q_vector *q_vector = adapter->q_vector[0]; 8118 struct e1000_hw *hw = &adapter->hw; 8119 /* read ICR disables interrupts using IAM */ 8120 u32 icr = rd32(E1000_ICR); 8121 8122 igb_write_itr(q_vector); 8123 8124 if (icr & E1000_ICR_DRSTA) 8125 schedule_work(&adapter->reset_task); 8126 8127 if (icr & E1000_ICR_DOUTSYNC) { 8128 /* HW is reporting DMA is out of sync */ 8129 adapter->stats.doosync++; 8130 } 8131 8132 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 8133 hw->mac.get_link_status = 1; 8134 if (!test_bit(__IGB_DOWN, &adapter->state)) 8135 mod_timer(&adapter->watchdog_timer, jiffies + 1); 8136 } 8137 8138 if (icr & E1000_ICR_TS) 8139 igb_tsync_interrupt(adapter); 8140 8141 napi_schedule(&q_vector->napi); 8142 8143 return IRQ_HANDLED; 8144 } 8145 8146 /** 8147 * igb_intr - Legacy Interrupt Handler 8148 * @irq: interrupt number 8149 * @data: pointer to a network interface device structure 8150 **/ 8151 static irqreturn_t igb_intr(int irq, void *data) 8152 { 8153 struct igb_adapter *adapter = data; 8154 struct igb_q_vector *q_vector = adapter->q_vector[0]; 8155 struct e1000_hw *hw = &adapter->hw; 8156 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 8157 * need for the IMC write 8158 */ 8159 u32 icr = rd32(E1000_ICR); 8160 8161 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 8162 * not set, then the adapter didn't send an interrupt 8163 */ 8164 if (!(icr & E1000_ICR_INT_ASSERTED)) 8165 return IRQ_NONE; 8166 8167 igb_write_itr(q_vector); 8168 8169 if (icr & E1000_ICR_DRSTA) 8170 schedule_work(&adapter->reset_task); 8171 8172 if (icr & E1000_ICR_DOUTSYNC) { 8173 /* HW is reporting DMA is out of sync */ 8174 adapter->stats.doosync++; 8175 } 8176 8177 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 8178 hw->mac.get_link_status = 1; 8179 /* guard against interrupt when we're going down */ 8180 if (!test_bit(__IGB_DOWN, &adapter->state)) 8181 mod_timer(&adapter->watchdog_timer, jiffies + 1); 8182 } 8183 8184 if (icr & E1000_ICR_TS) 8185 igb_tsync_interrupt(adapter); 8186 8187 napi_schedule(&q_vector->napi); 8188 8189 return IRQ_HANDLED; 8190 } 8191 8192 static void igb_ring_irq_enable(struct igb_q_vector *q_vector) 8193 { 8194 struct igb_adapter *adapter = q_vector->adapter; 8195 struct e1000_hw *hw = &adapter->hw; 8196 8197 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 8198 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 8199 if ((adapter->num_q_vectors == 1) && !adapter->vf_data) 8200 igb_set_itr(q_vector); 8201 else 8202 igb_update_ring_itr(q_vector); 8203 } 8204 8205 if (!test_bit(__IGB_DOWN, &adapter->state)) { 8206 if (adapter->flags & IGB_FLAG_HAS_MSIX) 8207 wr32(E1000_EIMS, q_vector->eims_value); 8208 else 8209 igb_irq_enable(adapter); 8210 } 8211 } 8212 8213 /** 8214 * igb_poll - NAPI Rx polling callback 8215 * @napi: napi polling structure 8216 * @budget: count of how many packets we should handle 8217 **/ 8218 static int igb_poll(struct napi_struct *napi, int budget) 8219 { 8220 struct igb_q_vector *q_vector = container_of(napi, 8221 struct igb_q_vector, 8222 napi); 8223 bool clean_complete = true; 8224 int work_done = 0; 8225 8226 #ifdef CONFIG_IGB_DCA 8227 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED) 8228 igb_update_dca(q_vector); 8229 #endif 8230 if (q_vector->tx.ring) 8231 clean_complete = igb_clean_tx_irq(q_vector, budget); 8232 8233 if (q_vector->rx.ring) { 8234 int cleaned = igb_clean_rx_irq(q_vector, budget); 8235 8236 work_done += cleaned; 8237 if (cleaned >= budget) 8238 clean_complete = false; 8239 } 8240 8241 /* If all work not completed, return budget and keep polling */ 8242 if (!clean_complete) 8243 return budget; 8244 8245 /* Exit the polling mode, but don't re-enable interrupts if stack might 8246 * poll us due to busy-polling 8247 */ 8248 if (likely(napi_complete_done(napi, work_done))) 8249 igb_ring_irq_enable(q_vector); 8250 8251 return work_done; 8252 } 8253 8254 /** 8255 * igb_clean_tx_irq - Reclaim resources after transmit completes 8256 * @q_vector: pointer to q_vector containing needed info 8257 * @napi_budget: Used to determine if we are in netpoll 8258 * 8259 * returns true if ring is completely cleaned 8260 **/ 8261 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector, int napi_budget) 8262 { 8263 struct igb_adapter *adapter = q_vector->adapter; 8264 struct igb_ring *tx_ring = q_vector->tx.ring; 8265 struct igb_tx_buffer *tx_buffer; 8266 union e1000_adv_tx_desc *tx_desc; 8267 unsigned int total_bytes = 0, total_packets = 0; 8268 unsigned int budget = q_vector->tx.work_limit; 8269 unsigned int i = tx_ring->next_to_clean; 8270 8271 if (test_bit(__IGB_DOWN, &adapter->state)) 8272 return true; 8273 8274 tx_buffer = &tx_ring->tx_buffer_info[i]; 8275 tx_desc = IGB_TX_DESC(tx_ring, i); 8276 i -= tx_ring->count; 8277 8278 do { 8279 union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 8280 8281 /* if next_to_watch is not set then there is no work pending */ 8282 if (!eop_desc) 8283 break; 8284 8285 /* prevent any other reads prior to eop_desc */ 8286 smp_rmb(); 8287 8288 /* if DD is not set pending work has not been completed */ 8289 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD))) 8290 break; 8291 8292 /* clear next_to_watch to prevent false hangs */ 8293 tx_buffer->next_to_watch = NULL; 8294 8295 /* update the statistics for this packet */ 8296 total_bytes += tx_buffer->bytecount; 8297 total_packets += tx_buffer->gso_segs; 8298 8299 /* free the skb */ 8300 if (tx_buffer->type == IGB_TYPE_SKB) 8301 napi_consume_skb(tx_buffer->skb, napi_budget); 8302 else 8303 xdp_return_frame(tx_buffer->xdpf); 8304 8305 /* unmap skb header data */ 8306 dma_unmap_single(tx_ring->dev, 8307 dma_unmap_addr(tx_buffer, dma), 8308 dma_unmap_len(tx_buffer, len), 8309 DMA_TO_DEVICE); 8310 8311 /* clear tx_buffer data */ 8312 dma_unmap_len_set(tx_buffer, len, 0); 8313 8314 /* clear last DMA location and unmap remaining buffers */ 8315 while (tx_desc != eop_desc) { 8316 tx_buffer++; 8317 tx_desc++; 8318 i++; 8319 if (unlikely(!i)) { 8320 i -= tx_ring->count; 8321 tx_buffer = tx_ring->tx_buffer_info; 8322 tx_desc = IGB_TX_DESC(tx_ring, 0); 8323 } 8324 8325 /* unmap any remaining paged data */ 8326 if (dma_unmap_len(tx_buffer, len)) { 8327 dma_unmap_page(tx_ring->dev, 8328 dma_unmap_addr(tx_buffer, dma), 8329 dma_unmap_len(tx_buffer, len), 8330 DMA_TO_DEVICE); 8331 dma_unmap_len_set(tx_buffer, len, 0); 8332 } 8333 } 8334 8335 /* move us one more past the eop_desc for start of next pkt */ 8336 tx_buffer++; 8337 tx_desc++; 8338 i++; 8339 if (unlikely(!i)) { 8340 i -= tx_ring->count; 8341 tx_buffer = tx_ring->tx_buffer_info; 8342 tx_desc = IGB_TX_DESC(tx_ring, 0); 8343 } 8344 8345 /* issue prefetch for next Tx descriptor */ 8346 prefetch(tx_desc); 8347 8348 /* update budget accounting */ 8349 budget--; 8350 } while (likely(budget)); 8351 8352 netdev_tx_completed_queue(txring_txq(tx_ring), 8353 total_packets, total_bytes); 8354 i += tx_ring->count; 8355 tx_ring->next_to_clean = i; 8356 u64_stats_update_begin(&tx_ring->tx_syncp); 8357 tx_ring->tx_stats.bytes += total_bytes; 8358 tx_ring->tx_stats.packets += total_packets; 8359 u64_stats_update_end(&tx_ring->tx_syncp); 8360 q_vector->tx.total_bytes += total_bytes; 8361 q_vector->tx.total_packets += total_packets; 8362 8363 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 8364 struct e1000_hw *hw = &adapter->hw; 8365 8366 /* Detect a transmit hang in hardware, this serializes the 8367 * check with the clearing of time_stamp and movement of i 8368 */ 8369 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 8370 if (tx_buffer->next_to_watch && 8371 time_after(jiffies, tx_buffer->time_stamp + 8372 (adapter->tx_timeout_factor * HZ)) && 8373 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) { 8374 8375 /* detected Tx unit hang */ 8376 dev_err(tx_ring->dev, 8377 "Detected Tx Unit Hang\n" 8378 " Tx Queue <%d>\n" 8379 " TDH <%x>\n" 8380 " TDT <%x>\n" 8381 " next_to_use <%x>\n" 8382 " next_to_clean <%x>\n" 8383 "buffer_info[next_to_clean]\n" 8384 " time_stamp <%lx>\n" 8385 " next_to_watch <%p>\n" 8386 " jiffies <%lx>\n" 8387 " desc.status <%x>\n", 8388 tx_ring->queue_index, 8389 rd32(E1000_TDH(tx_ring->reg_idx)), 8390 readl(tx_ring->tail), 8391 tx_ring->next_to_use, 8392 tx_ring->next_to_clean, 8393 tx_buffer->time_stamp, 8394 tx_buffer->next_to_watch, 8395 jiffies, 8396 tx_buffer->next_to_watch->wb.status); 8397 netif_stop_subqueue(tx_ring->netdev, 8398 tx_ring->queue_index); 8399 8400 /* we are about to reset, no point in enabling stuff */ 8401 return true; 8402 } 8403 } 8404 8405 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 8406 if (unlikely(total_packets && 8407 netif_carrier_ok(tx_ring->netdev) && 8408 igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 8409 /* Make sure that anybody stopping the queue after this 8410 * sees the new next_to_clean. 8411 */ 8412 smp_mb(); 8413 if (__netif_subqueue_stopped(tx_ring->netdev, 8414 tx_ring->queue_index) && 8415 !(test_bit(__IGB_DOWN, &adapter->state))) { 8416 netif_wake_subqueue(tx_ring->netdev, 8417 tx_ring->queue_index); 8418 8419 u64_stats_update_begin(&tx_ring->tx_syncp); 8420 tx_ring->tx_stats.restart_queue++; 8421 u64_stats_update_end(&tx_ring->tx_syncp); 8422 } 8423 } 8424 8425 return !!budget; 8426 } 8427 8428 /** 8429 * igb_reuse_rx_page - page flip buffer and store it back on the ring 8430 * @rx_ring: rx descriptor ring to store buffers on 8431 * @old_buff: donor buffer to have page reused 8432 * 8433 * Synchronizes page for reuse by the adapter 8434 **/ 8435 static void igb_reuse_rx_page(struct igb_ring *rx_ring, 8436 struct igb_rx_buffer *old_buff) 8437 { 8438 struct igb_rx_buffer *new_buff; 8439 u16 nta = rx_ring->next_to_alloc; 8440 8441 new_buff = &rx_ring->rx_buffer_info[nta]; 8442 8443 /* update, and store next to alloc */ 8444 nta++; 8445 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 8446 8447 /* Transfer page from old buffer to new buffer. 8448 * Move each member individually to avoid possible store 8449 * forwarding stalls. 8450 */ 8451 new_buff->dma = old_buff->dma; 8452 new_buff->page = old_buff->page; 8453 new_buff->page_offset = old_buff->page_offset; 8454 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 8455 } 8456 8457 static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer, 8458 int rx_buf_pgcnt) 8459 { 8460 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 8461 struct page *page = rx_buffer->page; 8462 8463 /* avoid re-using remote and pfmemalloc pages */ 8464 if (!dev_page_is_reusable(page)) 8465 return false; 8466 8467 #if (PAGE_SIZE < 8192) 8468 /* if we are only owner of page we can reuse it */ 8469 if (unlikely((rx_buf_pgcnt - pagecnt_bias) > 1)) 8470 return false; 8471 #else 8472 #define IGB_LAST_OFFSET \ 8473 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048) 8474 8475 if (rx_buffer->page_offset > IGB_LAST_OFFSET) 8476 return false; 8477 #endif 8478 8479 /* If we have drained the page fragment pool we need to update 8480 * the pagecnt_bias and page count so that we fully restock the 8481 * number of references the driver holds. 8482 */ 8483 if (unlikely(pagecnt_bias == 1)) { 8484 page_ref_add(page, USHRT_MAX - 1); 8485 rx_buffer->pagecnt_bias = USHRT_MAX; 8486 } 8487 8488 return true; 8489 } 8490 8491 /** 8492 * igb_add_rx_frag - Add contents of Rx buffer to sk_buff 8493 * @rx_ring: rx descriptor ring to transact packets on 8494 * @rx_buffer: buffer containing page to add 8495 * @skb: sk_buff to place the data into 8496 * @size: size of buffer to be added 8497 * 8498 * This function will add the data contained in rx_buffer->page to the skb. 8499 **/ 8500 static void igb_add_rx_frag(struct igb_ring *rx_ring, 8501 struct igb_rx_buffer *rx_buffer, 8502 struct sk_buff *skb, 8503 unsigned int size) 8504 { 8505 #if (PAGE_SIZE < 8192) 8506 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; 8507 #else 8508 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 8509 SKB_DATA_ALIGN(IGB_SKB_PAD + size) : 8510 SKB_DATA_ALIGN(size); 8511 #endif 8512 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 8513 rx_buffer->page_offset, size, truesize); 8514 #if (PAGE_SIZE < 8192) 8515 rx_buffer->page_offset ^= truesize; 8516 #else 8517 rx_buffer->page_offset += truesize; 8518 #endif 8519 } 8520 8521 static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring, 8522 struct igb_rx_buffer *rx_buffer, 8523 struct xdp_buff *xdp, 8524 ktime_t timestamp) 8525 { 8526 #if (PAGE_SIZE < 8192) 8527 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; 8528 #else 8529 unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end - 8530 xdp->data_hard_start); 8531 #endif 8532 unsigned int size = xdp->data_end - xdp->data; 8533 unsigned int headlen; 8534 struct sk_buff *skb; 8535 8536 /* prefetch first cache line of first page */ 8537 net_prefetch(xdp->data); 8538 8539 /* allocate a skb to store the frags */ 8540 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN); 8541 if (unlikely(!skb)) 8542 return NULL; 8543 8544 if (timestamp) 8545 skb_hwtstamps(skb)->hwtstamp = timestamp; 8546 8547 /* Determine available headroom for copy */ 8548 headlen = size; 8549 if (headlen > IGB_RX_HDR_LEN) 8550 headlen = eth_get_headlen(skb->dev, xdp->data, IGB_RX_HDR_LEN); 8551 8552 /* align pull length to size of long to optimize memcpy performance */ 8553 memcpy(__skb_put(skb, headlen), xdp->data, ALIGN(headlen, sizeof(long))); 8554 8555 /* update all of the pointers */ 8556 size -= headlen; 8557 if (size) { 8558 skb_add_rx_frag(skb, 0, rx_buffer->page, 8559 (xdp->data + headlen) - page_address(rx_buffer->page), 8560 size, truesize); 8561 #if (PAGE_SIZE < 8192) 8562 rx_buffer->page_offset ^= truesize; 8563 #else 8564 rx_buffer->page_offset += truesize; 8565 #endif 8566 } else { 8567 rx_buffer->pagecnt_bias++; 8568 } 8569 8570 return skb; 8571 } 8572 8573 static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring, 8574 struct igb_rx_buffer *rx_buffer, 8575 struct xdp_buff *xdp, 8576 ktime_t timestamp) 8577 { 8578 #if (PAGE_SIZE < 8192) 8579 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; 8580 #else 8581 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 8582 SKB_DATA_ALIGN(xdp->data_end - 8583 xdp->data_hard_start); 8584 #endif 8585 unsigned int metasize = xdp->data - xdp->data_meta; 8586 struct sk_buff *skb; 8587 8588 /* prefetch first cache line of first page */ 8589 net_prefetch(xdp->data_meta); 8590 8591 /* build an skb around the page buffer */ 8592 skb = napi_build_skb(xdp->data_hard_start, truesize); 8593 if (unlikely(!skb)) 8594 return NULL; 8595 8596 /* update pointers within the skb to store the data */ 8597 skb_reserve(skb, xdp->data - xdp->data_hard_start); 8598 __skb_put(skb, xdp->data_end - xdp->data); 8599 8600 if (metasize) 8601 skb_metadata_set(skb, metasize); 8602 8603 if (timestamp) 8604 skb_hwtstamps(skb)->hwtstamp = timestamp; 8605 8606 /* update buffer offset */ 8607 #if (PAGE_SIZE < 8192) 8608 rx_buffer->page_offset ^= truesize; 8609 #else 8610 rx_buffer->page_offset += truesize; 8611 #endif 8612 8613 return skb; 8614 } 8615 8616 static struct sk_buff *igb_run_xdp(struct igb_adapter *adapter, 8617 struct igb_ring *rx_ring, 8618 struct xdp_buff *xdp) 8619 { 8620 int err, result = IGB_XDP_PASS; 8621 struct bpf_prog *xdp_prog; 8622 u32 act; 8623 8624 xdp_prog = READ_ONCE(rx_ring->xdp_prog); 8625 8626 if (!xdp_prog) 8627 goto xdp_out; 8628 8629 prefetchw(xdp->data_hard_start); /* xdp_frame write */ 8630 8631 act = bpf_prog_run_xdp(xdp_prog, xdp); 8632 switch (act) { 8633 case XDP_PASS: 8634 break; 8635 case XDP_TX: 8636 result = igb_xdp_xmit_back(adapter, xdp); 8637 if (result == IGB_XDP_CONSUMED) 8638 goto out_failure; 8639 break; 8640 case XDP_REDIRECT: 8641 err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog); 8642 if (err) 8643 goto out_failure; 8644 result = IGB_XDP_REDIR; 8645 break; 8646 default: 8647 bpf_warn_invalid_xdp_action(adapter->netdev, xdp_prog, act); 8648 fallthrough; 8649 case XDP_ABORTED: 8650 out_failure: 8651 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); 8652 fallthrough; 8653 case XDP_DROP: 8654 result = IGB_XDP_CONSUMED; 8655 break; 8656 } 8657 xdp_out: 8658 return ERR_PTR(-result); 8659 } 8660 8661 static unsigned int igb_rx_frame_truesize(struct igb_ring *rx_ring, 8662 unsigned int size) 8663 { 8664 unsigned int truesize; 8665 8666 #if (PAGE_SIZE < 8192) 8667 truesize = igb_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ 8668 #else 8669 truesize = ring_uses_build_skb(rx_ring) ? 8670 SKB_DATA_ALIGN(IGB_SKB_PAD + size) + 8671 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : 8672 SKB_DATA_ALIGN(size); 8673 #endif 8674 return truesize; 8675 } 8676 8677 static void igb_rx_buffer_flip(struct igb_ring *rx_ring, 8678 struct igb_rx_buffer *rx_buffer, 8679 unsigned int size) 8680 { 8681 unsigned int truesize = igb_rx_frame_truesize(rx_ring, size); 8682 #if (PAGE_SIZE < 8192) 8683 rx_buffer->page_offset ^= truesize; 8684 #else 8685 rx_buffer->page_offset += truesize; 8686 #endif 8687 } 8688 8689 static inline void igb_rx_checksum(struct igb_ring *ring, 8690 union e1000_adv_rx_desc *rx_desc, 8691 struct sk_buff *skb) 8692 { 8693 skb_checksum_none_assert(skb); 8694 8695 /* Ignore Checksum bit is set */ 8696 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM)) 8697 return; 8698 8699 /* Rx checksum disabled via ethtool */ 8700 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 8701 return; 8702 8703 /* TCP/UDP checksum error bit is set */ 8704 if (igb_test_staterr(rx_desc, 8705 E1000_RXDEXT_STATERR_TCPE | 8706 E1000_RXDEXT_STATERR_IPE)) { 8707 /* work around errata with sctp packets where the TCPE aka 8708 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 8709 * packets, (aka let the stack check the crc32c) 8710 */ 8711 if (!((skb->len == 60) && 8712 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 8713 u64_stats_update_begin(&ring->rx_syncp); 8714 ring->rx_stats.csum_err++; 8715 u64_stats_update_end(&ring->rx_syncp); 8716 } 8717 /* let the stack verify checksum errors */ 8718 return; 8719 } 8720 /* It must be a TCP or UDP packet with a valid checksum */ 8721 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS | 8722 E1000_RXD_STAT_UDPCS)) 8723 skb->ip_summed = CHECKSUM_UNNECESSARY; 8724 8725 dev_dbg(ring->dev, "cksum success: bits %08X\n", 8726 le32_to_cpu(rx_desc->wb.upper.status_error)); 8727 } 8728 8729 static inline void igb_rx_hash(struct igb_ring *ring, 8730 union e1000_adv_rx_desc *rx_desc, 8731 struct sk_buff *skb) 8732 { 8733 if (ring->netdev->features & NETIF_F_RXHASH) 8734 skb_set_hash(skb, 8735 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 8736 PKT_HASH_TYPE_L3); 8737 } 8738 8739 /** 8740 * igb_is_non_eop - process handling of non-EOP buffers 8741 * @rx_ring: Rx ring being processed 8742 * @rx_desc: Rx descriptor for current buffer 8743 * 8744 * This function updates next to clean. If the buffer is an EOP buffer 8745 * this function exits returning false, otherwise it will place the 8746 * sk_buff in the next buffer to be chained and return true indicating 8747 * that this is in fact a non-EOP buffer. 8748 **/ 8749 static bool igb_is_non_eop(struct igb_ring *rx_ring, 8750 union e1000_adv_rx_desc *rx_desc) 8751 { 8752 u32 ntc = rx_ring->next_to_clean + 1; 8753 8754 /* fetch, update, and store next to clean */ 8755 ntc = (ntc < rx_ring->count) ? ntc : 0; 8756 rx_ring->next_to_clean = ntc; 8757 8758 prefetch(IGB_RX_DESC(rx_ring, ntc)); 8759 8760 if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP))) 8761 return false; 8762 8763 return true; 8764 } 8765 8766 /** 8767 * igb_cleanup_headers - Correct corrupted or empty headers 8768 * @rx_ring: rx descriptor ring packet is being transacted on 8769 * @rx_desc: pointer to the EOP Rx descriptor 8770 * @skb: pointer to current skb being fixed 8771 * 8772 * Address the case where we are pulling data in on pages only 8773 * and as such no data is present in the skb header. 8774 * 8775 * In addition if skb is not at least 60 bytes we need to pad it so that 8776 * it is large enough to qualify as a valid Ethernet frame. 8777 * 8778 * Returns true if an error was encountered and skb was freed. 8779 **/ 8780 static bool igb_cleanup_headers(struct igb_ring *rx_ring, 8781 union e1000_adv_rx_desc *rx_desc, 8782 struct sk_buff *skb) 8783 { 8784 /* XDP packets use error pointer so abort at this point */ 8785 if (IS_ERR(skb)) 8786 return true; 8787 8788 if (unlikely((igb_test_staterr(rx_desc, 8789 E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) { 8790 struct net_device *netdev = rx_ring->netdev; 8791 if (!(netdev->features & NETIF_F_RXALL)) { 8792 dev_kfree_skb_any(skb); 8793 return true; 8794 } 8795 } 8796 8797 /* if eth_skb_pad returns an error the skb was freed */ 8798 if (eth_skb_pad(skb)) 8799 return true; 8800 8801 return false; 8802 } 8803 8804 /** 8805 * igb_process_skb_fields - Populate skb header fields from Rx descriptor 8806 * @rx_ring: rx descriptor ring packet is being transacted on 8807 * @rx_desc: pointer to the EOP Rx descriptor 8808 * @skb: pointer to current skb being populated 8809 * 8810 * This function checks the ring, descriptor, and packet information in 8811 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 8812 * other fields within the skb. 8813 **/ 8814 static void igb_process_skb_fields(struct igb_ring *rx_ring, 8815 union e1000_adv_rx_desc *rx_desc, 8816 struct sk_buff *skb) 8817 { 8818 struct net_device *dev = rx_ring->netdev; 8819 8820 igb_rx_hash(rx_ring, rx_desc, skb); 8821 8822 igb_rx_checksum(rx_ring, rx_desc, skb); 8823 8824 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TS) && 8825 !igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) 8826 igb_ptp_rx_rgtstamp(rx_ring->q_vector, skb); 8827 8828 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && 8829 igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) { 8830 u16 vid; 8831 8832 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) && 8833 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags)) 8834 vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan); 8835 else 8836 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 8837 8838 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 8839 } 8840 8841 skb_record_rx_queue(skb, rx_ring->queue_index); 8842 8843 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 8844 } 8845 8846 static unsigned int igb_rx_offset(struct igb_ring *rx_ring) 8847 { 8848 return ring_uses_build_skb(rx_ring) ? IGB_SKB_PAD : 0; 8849 } 8850 8851 static struct igb_rx_buffer *igb_get_rx_buffer(struct igb_ring *rx_ring, 8852 const unsigned int size, int *rx_buf_pgcnt) 8853 { 8854 struct igb_rx_buffer *rx_buffer; 8855 8856 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 8857 *rx_buf_pgcnt = 8858 #if (PAGE_SIZE < 8192) 8859 page_count(rx_buffer->page); 8860 #else 8861 0; 8862 #endif 8863 prefetchw(rx_buffer->page); 8864 8865 /* we are reusing so sync this buffer for CPU use */ 8866 dma_sync_single_range_for_cpu(rx_ring->dev, 8867 rx_buffer->dma, 8868 rx_buffer->page_offset, 8869 size, 8870 DMA_FROM_DEVICE); 8871 8872 rx_buffer->pagecnt_bias--; 8873 8874 return rx_buffer; 8875 } 8876 8877 static void igb_put_rx_buffer(struct igb_ring *rx_ring, 8878 struct igb_rx_buffer *rx_buffer, int rx_buf_pgcnt) 8879 { 8880 if (igb_can_reuse_rx_page(rx_buffer, rx_buf_pgcnt)) { 8881 /* hand second half of page back to the ring */ 8882 igb_reuse_rx_page(rx_ring, rx_buffer); 8883 } else { 8884 /* We are not reusing the buffer so unmap it and free 8885 * any references we are holding to it 8886 */ 8887 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 8888 igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 8889 IGB_RX_DMA_ATTR); 8890 __page_frag_cache_drain(rx_buffer->page, 8891 rx_buffer->pagecnt_bias); 8892 } 8893 8894 /* clear contents of rx_buffer */ 8895 rx_buffer->page = NULL; 8896 } 8897 8898 static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget) 8899 { 8900 unsigned int total_bytes = 0, total_packets = 0; 8901 struct igb_adapter *adapter = q_vector->adapter; 8902 struct igb_ring *rx_ring = q_vector->rx.ring; 8903 u16 cleaned_count = igb_desc_unused(rx_ring); 8904 struct sk_buff *skb = rx_ring->skb; 8905 int cpu = smp_processor_id(); 8906 unsigned int xdp_xmit = 0; 8907 struct netdev_queue *nq; 8908 struct xdp_buff xdp; 8909 u32 frame_sz = 0; 8910 int rx_buf_pgcnt; 8911 8912 /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ 8913 #if (PAGE_SIZE < 8192) 8914 frame_sz = igb_rx_frame_truesize(rx_ring, 0); 8915 #endif 8916 xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); 8917 8918 while (likely(total_packets < budget)) { 8919 union e1000_adv_rx_desc *rx_desc; 8920 struct igb_rx_buffer *rx_buffer; 8921 ktime_t timestamp = 0; 8922 int pkt_offset = 0; 8923 unsigned int size; 8924 void *pktbuf; 8925 8926 /* return some buffers to hardware, one at a time is too slow */ 8927 if (cleaned_count >= IGB_RX_BUFFER_WRITE) { 8928 igb_alloc_rx_buffers(rx_ring, cleaned_count); 8929 cleaned_count = 0; 8930 } 8931 8932 rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean); 8933 size = le16_to_cpu(rx_desc->wb.upper.length); 8934 if (!size) 8935 break; 8936 8937 /* This memory barrier is needed to keep us from reading 8938 * any other fields out of the rx_desc until we know the 8939 * descriptor has been written back 8940 */ 8941 dma_rmb(); 8942 8943 rx_buffer = igb_get_rx_buffer(rx_ring, size, &rx_buf_pgcnt); 8944 pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset; 8945 8946 /* pull rx packet timestamp if available and valid */ 8947 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) { 8948 int ts_hdr_len; 8949 8950 ts_hdr_len = igb_ptp_rx_pktstamp(rx_ring->q_vector, 8951 pktbuf, ×tamp); 8952 8953 pkt_offset += ts_hdr_len; 8954 size -= ts_hdr_len; 8955 } 8956 8957 /* retrieve a buffer from the ring */ 8958 if (!skb) { 8959 unsigned char *hard_start = pktbuf - igb_rx_offset(rx_ring); 8960 unsigned int offset = pkt_offset + igb_rx_offset(rx_ring); 8961 8962 xdp_prepare_buff(&xdp, hard_start, offset, size, true); 8963 xdp_buff_clear_frags_flag(&xdp); 8964 #if (PAGE_SIZE > 4096) 8965 /* At larger PAGE_SIZE, frame_sz depend on len size */ 8966 xdp.frame_sz = igb_rx_frame_truesize(rx_ring, size); 8967 #endif 8968 skb = igb_run_xdp(adapter, rx_ring, &xdp); 8969 } 8970 8971 if (IS_ERR(skb)) { 8972 unsigned int xdp_res = -PTR_ERR(skb); 8973 8974 if (xdp_res & (IGB_XDP_TX | IGB_XDP_REDIR)) { 8975 xdp_xmit |= xdp_res; 8976 igb_rx_buffer_flip(rx_ring, rx_buffer, size); 8977 } else { 8978 rx_buffer->pagecnt_bias++; 8979 } 8980 total_packets++; 8981 total_bytes += size; 8982 } else if (skb) 8983 igb_add_rx_frag(rx_ring, rx_buffer, skb, size); 8984 else if (ring_uses_build_skb(rx_ring)) 8985 skb = igb_build_skb(rx_ring, rx_buffer, &xdp, 8986 timestamp); 8987 else 8988 skb = igb_construct_skb(rx_ring, rx_buffer, 8989 &xdp, timestamp); 8990 8991 /* exit if we failed to retrieve a buffer */ 8992 if (!skb) { 8993 rx_ring->rx_stats.alloc_failed++; 8994 rx_buffer->pagecnt_bias++; 8995 break; 8996 } 8997 8998 igb_put_rx_buffer(rx_ring, rx_buffer, rx_buf_pgcnt); 8999 cleaned_count++; 9000 9001 /* fetch next buffer in frame if non-eop */ 9002 if (igb_is_non_eop(rx_ring, rx_desc)) 9003 continue; 9004 9005 /* verify the packet layout is correct */ 9006 if (igb_cleanup_headers(rx_ring, rx_desc, skb)) { 9007 skb = NULL; 9008 continue; 9009 } 9010 9011 /* probably a little skewed due to removing CRC */ 9012 total_bytes += skb->len; 9013 9014 /* populate checksum, timestamp, VLAN, and protocol */ 9015 igb_process_skb_fields(rx_ring, rx_desc, skb); 9016 9017 napi_gro_receive(&q_vector->napi, skb); 9018 9019 /* reset skb pointer */ 9020 skb = NULL; 9021 9022 /* update budget accounting */ 9023 total_packets++; 9024 } 9025 9026 /* place incomplete frames back on ring for completion */ 9027 rx_ring->skb = skb; 9028 9029 if (xdp_xmit & IGB_XDP_REDIR) 9030 xdp_do_flush(); 9031 9032 if (xdp_xmit & IGB_XDP_TX) { 9033 struct igb_ring *tx_ring = igb_xdp_tx_queue_mapping(adapter); 9034 9035 nq = txring_txq(tx_ring); 9036 __netif_tx_lock(nq, cpu); 9037 igb_xdp_ring_update_tail(tx_ring); 9038 __netif_tx_unlock(nq); 9039 } 9040 9041 u64_stats_update_begin(&rx_ring->rx_syncp); 9042 rx_ring->rx_stats.packets += total_packets; 9043 rx_ring->rx_stats.bytes += total_bytes; 9044 u64_stats_update_end(&rx_ring->rx_syncp); 9045 q_vector->rx.total_packets += total_packets; 9046 q_vector->rx.total_bytes += total_bytes; 9047 9048 if (cleaned_count) 9049 igb_alloc_rx_buffers(rx_ring, cleaned_count); 9050 9051 return total_packets; 9052 } 9053 9054 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring, 9055 struct igb_rx_buffer *bi) 9056 { 9057 struct page *page = bi->page; 9058 dma_addr_t dma; 9059 9060 /* since we are recycling buffers we should seldom need to alloc */ 9061 if (likely(page)) 9062 return true; 9063 9064 /* alloc new page for storage */ 9065 page = dev_alloc_pages(igb_rx_pg_order(rx_ring)); 9066 if (unlikely(!page)) { 9067 rx_ring->rx_stats.alloc_failed++; 9068 return false; 9069 } 9070 9071 /* map page for use */ 9072 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 9073 igb_rx_pg_size(rx_ring), 9074 DMA_FROM_DEVICE, 9075 IGB_RX_DMA_ATTR); 9076 9077 /* if mapping failed free memory back to system since 9078 * there isn't much point in holding memory we can't use 9079 */ 9080 if (dma_mapping_error(rx_ring->dev, dma)) { 9081 __free_pages(page, igb_rx_pg_order(rx_ring)); 9082 9083 rx_ring->rx_stats.alloc_failed++; 9084 return false; 9085 } 9086 9087 bi->dma = dma; 9088 bi->page = page; 9089 bi->page_offset = igb_rx_offset(rx_ring); 9090 page_ref_add(page, USHRT_MAX - 1); 9091 bi->pagecnt_bias = USHRT_MAX; 9092 9093 return true; 9094 } 9095 9096 /** 9097 * igb_alloc_rx_buffers - Replace used receive buffers 9098 * @rx_ring: rx descriptor ring to allocate new receive buffers 9099 * @cleaned_count: count of buffers to allocate 9100 **/ 9101 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count) 9102 { 9103 union e1000_adv_rx_desc *rx_desc; 9104 struct igb_rx_buffer *bi; 9105 u16 i = rx_ring->next_to_use; 9106 u16 bufsz; 9107 9108 /* nothing to do */ 9109 if (!cleaned_count) 9110 return; 9111 9112 rx_desc = IGB_RX_DESC(rx_ring, i); 9113 bi = &rx_ring->rx_buffer_info[i]; 9114 i -= rx_ring->count; 9115 9116 bufsz = igb_rx_bufsz(rx_ring); 9117 9118 do { 9119 if (!igb_alloc_mapped_page(rx_ring, bi)) 9120 break; 9121 9122 /* sync the buffer for use by the device */ 9123 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 9124 bi->page_offset, bufsz, 9125 DMA_FROM_DEVICE); 9126 9127 /* Refresh the desc even if buffer_addrs didn't change 9128 * because each write-back erases this info. 9129 */ 9130 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 9131 9132 rx_desc++; 9133 bi++; 9134 i++; 9135 if (unlikely(!i)) { 9136 rx_desc = IGB_RX_DESC(rx_ring, 0); 9137 bi = rx_ring->rx_buffer_info; 9138 i -= rx_ring->count; 9139 } 9140 9141 /* clear the length for the next_to_use descriptor */ 9142 rx_desc->wb.upper.length = 0; 9143 9144 cleaned_count--; 9145 } while (cleaned_count); 9146 9147 i += rx_ring->count; 9148 9149 if (rx_ring->next_to_use != i) { 9150 /* record the next descriptor to use */ 9151 rx_ring->next_to_use = i; 9152 9153 /* update next to alloc since we have filled the ring */ 9154 rx_ring->next_to_alloc = i; 9155 9156 /* Force memory writes to complete before letting h/w 9157 * know there are new descriptors to fetch. (Only 9158 * applicable for weak-ordered memory model archs, 9159 * such as IA-64). 9160 */ 9161 dma_wmb(); 9162 writel(i, rx_ring->tail); 9163 } 9164 } 9165 9166 /** 9167 * igb_mii_ioctl - 9168 * @netdev: pointer to netdev struct 9169 * @ifr: interface structure 9170 * @cmd: ioctl command to execute 9171 **/ 9172 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 9173 { 9174 struct igb_adapter *adapter = netdev_priv(netdev); 9175 struct mii_ioctl_data *data = if_mii(ifr); 9176 9177 if (adapter->hw.phy.media_type != e1000_media_type_copper) 9178 return -EOPNOTSUPP; 9179 9180 switch (cmd) { 9181 case SIOCGMIIPHY: 9182 data->phy_id = adapter->hw.phy.addr; 9183 break; 9184 case SIOCGMIIREG: 9185 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F, 9186 &data->val_out)) 9187 return -EIO; 9188 break; 9189 case SIOCSMIIREG: 9190 default: 9191 return -EOPNOTSUPP; 9192 } 9193 return 0; 9194 } 9195 9196 /** 9197 * igb_ioctl - 9198 * @netdev: pointer to netdev struct 9199 * @ifr: interface structure 9200 * @cmd: ioctl command to execute 9201 **/ 9202 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 9203 { 9204 switch (cmd) { 9205 case SIOCGMIIPHY: 9206 case SIOCGMIIREG: 9207 case SIOCSMIIREG: 9208 return igb_mii_ioctl(netdev, ifr, cmd); 9209 case SIOCGHWTSTAMP: 9210 return igb_ptp_get_ts_config(netdev, ifr); 9211 case SIOCSHWTSTAMP: 9212 return igb_ptp_set_ts_config(netdev, ifr); 9213 default: 9214 return -EOPNOTSUPP; 9215 } 9216 } 9217 9218 void igb_read_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value) 9219 { 9220 struct igb_adapter *adapter = hw->back; 9221 9222 pci_read_config_word(adapter->pdev, reg, value); 9223 } 9224 9225 void igb_write_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value) 9226 { 9227 struct igb_adapter *adapter = hw->back; 9228 9229 pci_write_config_word(adapter->pdev, reg, *value); 9230 } 9231 9232 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value) 9233 { 9234 struct igb_adapter *adapter = hw->back; 9235 9236 if (pcie_capability_read_word(adapter->pdev, reg, value)) 9237 return -E1000_ERR_CONFIG; 9238 9239 return 0; 9240 } 9241 9242 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value) 9243 { 9244 struct igb_adapter *adapter = hw->back; 9245 9246 if (pcie_capability_write_word(adapter->pdev, reg, *value)) 9247 return -E1000_ERR_CONFIG; 9248 9249 return 0; 9250 } 9251 9252 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features) 9253 { 9254 struct igb_adapter *adapter = netdev_priv(netdev); 9255 struct e1000_hw *hw = &adapter->hw; 9256 u32 ctrl, rctl; 9257 bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX); 9258 9259 if (enable) { 9260 /* enable VLAN tag insert/strip */ 9261 ctrl = rd32(E1000_CTRL); 9262 ctrl |= E1000_CTRL_VME; 9263 wr32(E1000_CTRL, ctrl); 9264 9265 /* Disable CFI check */ 9266 rctl = rd32(E1000_RCTL); 9267 rctl &= ~E1000_RCTL_CFIEN; 9268 wr32(E1000_RCTL, rctl); 9269 } else { 9270 /* disable VLAN tag insert/strip */ 9271 ctrl = rd32(E1000_CTRL); 9272 ctrl &= ~E1000_CTRL_VME; 9273 wr32(E1000_CTRL, ctrl); 9274 } 9275 9276 igb_set_vf_vlan_strip(adapter, adapter->vfs_allocated_count, enable); 9277 } 9278 9279 static int igb_vlan_rx_add_vid(struct net_device *netdev, 9280 __be16 proto, u16 vid) 9281 { 9282 struct igb_adapter *adapter = netdev_priv(netdev); 9283 struct e1000_hw *hw = &adapter->hw; 9284 int pf_id = adapter->vfs_allocated_count; 9285 9286 /* add the filter since PF can receive vlans w/o entry in vlvf */ 9287 if (!vid || !(adapter->flags & IGB_FLAG_VLAN_PROMISC)) 9288 igb_vfta_set(hw, vid, pf_id, true, !!vid); 9289 9290 set_bit(vid, adapter->active_vlans); 9291 9292 return 0; 9293 } 9294 9295 static int igb_vlan_rx_kill_vid(struct net_device *netdev, 9296 __be16 proto, u16 vid) 9297 { 9298 struct igb_adapter *adapter = netdev_priv(netdev); 9299 int pf_id = adapter->vfs_allocated_count; 9300 struct e1000_hw *hw = &adapter->hw; 9301 9302 /* remove VID from filter table */ 9303 if (vid && !(adapter->flags & IGB_FLAG_VLAN_PROMISC)) 9304 igb_vfta_set(hw, vid, pf_id, false, true); 9305 9306 clear_bit(vid, adapter->active_vlans); 9307 9308 return 0; 9309 } 9310 9311 static void igb_restore_vlan(struct igb_adapter *adapter) 9312 { 9313 u16 vid = 1; 9314 9315 igb_vlan_mode(adapter->netdev, adapter->netdev->features); 9316 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0); 9317 9318 for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID) 9319 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid); 9320 } 9321 9322 int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx) 9323 { 9324 struct pci_dev *pdev = adapter->pdev; 9325 struct e1000_mac_info *mac = &adapter->hw.mac; 9326 9327 mac->autoneg = 0; 9328 9329 /* Make sure dplx is at most 1 bit and lsb of speed is not set 9330 * for the switch() below to work 9331 */ 9332 if ((spd & 1) || (dplx & ~1)) 9333 goto err_inval; 9334 9335 /* Fiber NIC's only allow 1000 gbps Full duplex 9336 * and 100Mbps Full duplex for 100baseFx sfp 9337 */ 9338 if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { 9339 switch (spd + dplx) { 9340 case SPEED_10 + DUPLEX_HALF: 9341 case SPEED_10 + DUPLEX_FULL: 9342 case SPEED_100 + DUPLEX_HALF: 9343 goto err_inval; 9344 default: 9345 break; 9346 } 9347 } 9348 9349 switch (spd + dplx) { 9350 case SPEED_10 + DUPLEX_HALF: 9351 mac->forced_speed_duplex = ADVERTISE_10_HALF; 9352 break; 9353 case SPEED_10 + DUPLEX_FULL: 9354 mac->forced_speed_duplex = ADVERTISE_10_FULL; 9355 break; 9356 case SPEED_100 + DUPLEX_HALF: 9357 mac->forced_speed_duplex = ADVERTISE_100_HALF; 9358 break; 9359 case SPEED_100 + DUPLEX_FULL: 9360 mac->forced_speed_duplex = ADVERTISE_100_FULL; 9361 break; 9362 case SPEED_1000 + DUPLEX_FULL: 9363 mac->autoneg = 1; 9364 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 9365 break; 9366 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 9367 default: 9368 goto err_inval; 9369 } 9370 9371 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 9372 adapter->hw.phy.mdix = AUTO_ALL_MODES; 9373 9374 return 0; 9375 9376 err_inval: 9377 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n"); 9378 return -EINVAL; 9379 } 9380 9381 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake, 9382 bool runtime) 9383 { 9384 struct net_device *netdev = pci_get_drvdata(pdev); 9385 struct igb_adapter *adapter = netdev_priv(netdev); 9386 struct e1000_hw *hw = &adapter->hw; 9387 u32 ctrl, rctl, status; 9388 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol; 9389 bool wake; 9390 9391 rtnl_lock(); 9392 netif_device_detach(netdev); 9393 9394 if (netif_running(netdev)) 9395 __igb_close(netdev, true); 9396 9397 igb_ptp_suspend(adapter); 9398 9399 igb_clear_interrupt_scheme(adapter); 9400 rtnl_unlock(); 9401 9402 status = rd32(E1000_STATUS); 9403 if (status & E1000_STATUS_LU) 9404 wufc &= ~E1000_WUFC_LNKC; 9405 9406 if (wufc) { 9407 igb_setup_rctl(adapter); 9408 igb_set_rx_mode(netdev); 9409 9410 /* turn on all-multi mode if wake on multicast is enabled */ 9411 if (wufc & E1000_WUFC_MC) { 9412 rctl = rd32(E1000_RCTL); 9413 rctl |= E1000_RCTL_MPE; 9414 wr32(E1000_RCTL, rctl); 9415 } 9416 9417 ctrl = rd32(E1000_CTRL); 9418 ctrl |= E1000_CTRL_ADVD3WUC; 9419 wr32(E1000_CTRL, ctrl); 9420 9421 /* Allow time for pending master requests to run */ 9422 igb_disable_pcie_master(hw); 9423 9424 wr32(E1000_WUC, E1000_WUC_PME_EN); 9425 wr32(E1000_WUFC, wufc); 9426 } else { 9427 wr32(E1000_WUC, 0); 9428 wr32(E1000_WUFC, 0); 9429 } 9430 9431 wake = wufc || adapter->en_mng_pt; 9432 if (!wake) 9433 igb_power_down_link(adapter); 9434 else 9435 igb_power_up_link(adapter); 9436 9437 if (enable_wake) 9438 *enable_wake = wake; 9439 9440 /* Release control of h/w to f/w. If f/w is AMT enabled, this 9441 * would have already happened in close and is redundant. 9442 */ 9443 igb_release_hw_control(adapter); 9444 9445 pci_disable_device(pdev); 9446 9447 return 0; 9448 } 9449 9450 static void igb_deliver_wake_packet(struct net_device *netdev) 9451 { 9452 struct igb_adapter *adapter = netdev_priv(netdev); 9453 struct e1000_hw *hw = &adapter->hw; 9454 struct sk_buff *skb; 9455 u32 wupl; 9456 9457 wupl = rd32(E1000_WUPL) & E1000_WUPL_MASK; 9458 9459 /* WUPM stores only the first 128 bytes of the wake packet. 9460 * Read the packet only if we have the whole thing. 9461 */ 9462 if ((wupl == 0) || (wupl > E1000_WUPM_BYTES)) 9463 return; 9464 9465 skb = netdev_alloc_skb_ip_align(netdev, E1000_WUPM_BYTES); 9466 if (!skb) 9467 return; 9468 9469 skb_put(skb, wupl); 9470 9471 /* Ensure reads are 32-bit aligned */ 9472 wupl = roundup(wupl, 4); 9473 9474 memcpy_fromio(skb->data, hw->hw_addr + E1000_WUPM_REG(0), wupl); 9475 9476 skb->protocol = eth_type_trans(skb, netdev); 9477 netif_rx(skb); 9478 } 9479 9480 static int __maybe_unused igb_suspend(struct device *dev) 9481 { 9482 return __igb_shutdown(to_pci_dev(dev), NULL, 0); 9483 } 9484 9485 static int __maybe_unused __igb_resume(struct device *dev, bool rpm) 9486 { 9487 struct pci_dev *pdev = to_pci_dev(dev); 9488 struct net_device *netdev = pci_get_drvdata(pdev); 9489 struct igb_adapter *adapter = netdev_priv(netdev); 9490 struct e1000_hw *hw = &adapter->hw; 9491 u32 err, val; 9492 9493 pci_set_power_state(pdev, PCI_D0); 9494 pci_restore_state(pdev); 9495 pci_save_state(pdev); 9496 9497 if (!pci_device_is_present(pdev)) 9498 return -ENODEV; 9499 err = pci_enable_device_mem(pdev); 9500 if (err) { 9501 dev_err(&pdev->dev, 9502 "igb: Cannot enable PCI device from suspend\n"); 9503 return err; 9504 } 9505 pci_set_master(pdev); 9506 9507 pci_enable_wake(pdev, PCI_D3hot, 0); 9508 pci_enable_wake(pdev, PCI_D3cold, 0); 9509 9510 if (igb_init_interrupt_scheme(adapter, true)) { 9511 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 9512 return -ENOMEM; 9513 } 9514 9515 igb_reset(adapter); 9516 9517 /* let the f/w know that the h/w is now under the control of the 9518 * driver. 9519 */ 9520 igb_get_hw_control(adapter); 9521 9522 val = rd32(E1000_WUS); 9523 if (val & WAKE_PKT_WUS) 9524 igb_deliver_wake_packet(netdev); 9525 9526 wr32(E1000_WUS, ~0); 9527 9528 if (!rpm) 9529 rtnl_lock(); 9530 if (!err && netif_running(netdev)) 9531 err = __igb_open(netdev, true); 9532 9533 if (!err) 9534 netif_device_attach(netdev); 9535 if (!rpm) 9536 rtnl_unlock(); 9537 9538 return err; 9539 } 9540 9541 static int __maybe_unused igb_resume(struct device *dev) 9542 { 9543 return __igb_resume(dev, false); 9544 } 9545 9546 static int __maybe_unused igb_runtime_idle(struct device *dev) 9547 { 9548 struct net_device *netdev = dev_get_drvdata(dev); 9549 struct igb_adapter *adapter = netdev_priv(netdev); 9550 9551 if (!igb_has_link(adapter)) 9552 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 9553 9554 return -EBUSY; 9555 } 9556 9557 static int __maybe_unused igb_runtime_suspend(struct device *dev) 9558 { 9559 return __igb_shutdown(to_pci_dev(dev), NULL, 1); 9560 } 9561 9562 static int __maybe_unused igb_runtime_resume(struct device *dev) 9563 { 9564 return __igb_resume(dev, true); 9565 } 9566 9567 static void igb_shutdown(struct pci_dev *pdev) 9568 { 9569 bool wake; 9570 9571 __igb_shutdown(pdev, &wake, 0); 9572 9573 if (system_state == SYSTEM_POWER_OFF) { 9574 pci_wake_from_d3(pdev, wake); 9575 pci_set_power_state(pdev, PCI_D3hot); 9576 } 9577 } 9578 9579 static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs) 9580 { 9581 #ifdef CONFIG_PCI_IOV 9582 int err; 9583 9584 if (num_vfs == 0) { 9585 return igb_disable_sriov(dev, true); 9586 } else { 9587 err = igb_enable_sriov(dev, num_vfs, true); 9588 return err ? err : num_vfs; 9589 } 9590 #endif 9591 return 0; 9592 } 9593 9594 /** 9595 * igb_io_error_detected - called when PCI error is detected 9596 * @pdev: Pointer to PCI device 9597 * @state: The current pci connection state 9598 * 9599 * This function is called after a PCI bus error affecting 9600 * this device has been detected. 9601 **/ 9602 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev, 9603 pci_channel_state_t state) 9604 { 9605 struct net_device *netdev = pci_get_drvdata(pdev); 9606 struct igb_adapter *adapter = netdev_priv(netdev); 9607 9608 if (state == pci_channel_io_normal) { 9609 dev_warn(&pdev->dev, "Non-correctable non-fatal error reported.\n"); 9610 return PCI_ERS_RESULT_CAN_RECOVER; 9611 } 9612 9613 netif_device_detach(netdev); 9614 9615 if (state == pci_channel_io_perm_failure) 9616 return PCI_ERS_RESULT_DISCONNECT; 9617 9618 if (netif_running(netdev)) 9619 igb_down(adapter); 9620 pci_disable_device(pdev); 9621 9622 /* Request a slot reset. */ 9623 return PCI_ERS_RESULT_NEED_RESET; 9624 } 9625 9626 /** 9627 * igb_io_slot_reset - called after the pci bus has been reset. 9628 * @pdev: Pointer to PCI device 9629 * 9630 * Restart the card from scratch, as if from a cold-boot. Implementation 9631 * resembles the first-half of the __igb_resume routine. 9632 **/ 9633 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev) 9634 { 9635 struct net_device *netdev = pci_get_drvdata(pdev); 9636 struct igb_adapter *adapter = netdev_priv(netdev); 9637 struct e1000_hw *hw = &adapter->hw; 9638 pci_ers_result_t result; 9639 9640 if (pci_enable_device_mem(pdev)) { 9641 dev_err(&pdev->dev, 9642 "Cannot re-enable PCI device after reset.\n"); 9643 result = PCI_ERS_RESULT_DISCONNECT; 9644 } else { 9645 pci_set_master(pdev); 9646 pci_restore_state(pdev); 9647 pci_save_state(pdev); 9648 9649 pci_enable_wake(pdev, PCI_D3hot, 0); 9650 pci_enable_wake(pdev, PCI_D3cold, 0); 9651 9652 /* In case of PCI error, adapter lose its HW address 9653 * so we should re-assign it here. 9654 */ 9655 hw->hw_addr = adapter->io_addr; 9656 9657 igb_reset(adapter); 9658 wr32(E1000_WUS, ~0); 9659 result = PCI_ERS_RESULT_RECOVERED; 9660 } 9661 9662 return result; 9663 } 9664 9665 /** 9666 * igb_io_resume - called when traffic can start flowing again. 9667 * @pdev: Pointer to PCI device 9668 * 9669 * This callback is called when the error recovery driver tells us that 9670 * its OK to resume normal operation. Implementation resembles the 9671 * second-half of the __igb_resume routine. 9672 */ 9673 static void igb_io_resume(struct pci_dev *pdev) 9674 { 9675 struct net_device *netdev = pci_get_drvdata(pdev); 9676 struct igb_adapter *adapter = netdev_priv(netdev); 9677 9678 if (netif_running(netdev)) { 9679 if (!test_bit(__IGB_DOWN, &adapter->state)) { 9680 dev_dbg(&pdev->dev, "Resuming from non-fatal error, do nothing.\n"); 9681 return; 9682 } 9683 if (igb_up(adapter)) { 9684 dev_err(&pdev->dev, "igb_up failed after reset\n"); 9685 return; 9686 } 9687 } 9688 9689 netif_device_attach(netdev); 9690 9691 /* let the f/w know that the h/w is now under the control of the 9692 * driver. 9693 */ 9694 igb_get_hw_control(adapter); 9695 } 9696 9697 /** 9698 * igb_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table 9699 * @adapter: Pointer to adapter structure 9700 * @index: Index of the RAR entry which need to be synced with MAC table 9701 **/ 9702 static void igb_rar_set_index(struct igb_adapter *adapter, u32 index) 9703 { 9704 struct e1000_hw *hw = &adapter->hw; 9705 u32 rar_low, rar_high; 9706 u8 *addr = adapter->mac_table[index].addr; 9707 9708 /* HW expects these to be in network order when they are plugged 9709 * into the registers which are little endian. In order to guarantee 9710 * that ordering we need to do an leXX_to_cpup here in order to be 9711 * ready for the byteswap that occurs with writel 9712 */ 9713 rar_low = le32_to_cpup((__le32 *)(addr)); 9714 rar_high = le16_to_cpup((__le16 *)(addr + 4)); 9715 9716 /* Indicate to hardware the Address is Valid. */ 9717 if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE) { 9718 if (is_valid_ether_addr(addr)) 9719 rar_high |= E1000_RAH_AV; 9720 9721 if (adapter->mac_table[index].state & IGB_MAC_STATE_SRC_ADDR) 9722 rar_high |= E1000_RAH_ASEL_SRC_ADDR; 9723 9724 switch (hw->mac.type) { 9725 case e1000_82575: 9726 case e1000_i210: 9727 if (adapter->mac_table[index].state & 9728 IGB_MAC_STATE_QUEUE_STEERING) 9729 rar_high |= E1000_RAH_QSEL_ENABLE; 9730 9731 rar_high |= E1000_RAH_POOL_1 * 9732 adapter->mac_table[index].queue; 9733 break; 9734 default: 9735 rar_high |= E1000_RAH_POOL_1 << 9736 adapter->mac_table[index].queue; 9737 break; 9738 } 9739 } 9740 9741 wr32(E1000_RAL(index), rar_low); 9742 wrfl(); 9743 wr32(E1000_RAH(index), rar_high); 9744 wrfl(); 9745 } 9746 9747 static int igb_set_vf_mac(struct igb_adapter *adapter, 9748 int vf, unsigned char *mac_addr) 9749 { 9750 struct e1000_hw *hw = &adapter->hw; 9751 /* VF MAC addresses start at end of receive addresses and moves 9752 * towards the first, as a result a collision should not be possible 9753 */ 9754 int rar_entry = hw->mac.rar_entry_count - (vf + 1); 9755 unsigned char *vf_mac_addr = adapter->vf_data[vf].vf_mac_addresses; 9756 9757 ether_addr_copy(vf_mac_addr, mac_addr); 9758 ether_addr_copy(adapter->mac_table[rar_entry].addr, mac_addr); 9759 adapter->mac_table[rar_entry].queue = vf; 9760 adapter->mac_table[rar_entry].state |= IGB_MAC_STATE_IN_USE; 9761 igb_rar_set_index(adapter, rar_entry); 9762 9763 return 0; 9764 } 9765 9766 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) 9767 { 9768 struct igb_adapter *adapter = netdev_priv(netdev); 9769 9770 if (vf >= adapter->vfs_allocated_count) 9771 return -EINVAL; 9772 9773 /* Setting the VF MAC to 0 reverts the IGB_VF_FLAG_PF_SET_MAC 9774 * flag and allows to overwrite the MAC via VF netdev. This 9775 * is necessary to allow libvirt a way to restore the original 9776 * MAC after unbinding vfio-pci and reloading igbvf after shutting 9777 * down a VM. 9778 */ 9779 if (is_zero_ether_addr(mac)) { 9780 adapter->vf_data[vf].flags &= ~IGB_VF_FLAG_PF_SET_MAC; 9781 dev_info(&adapter->pdev->dev, 9782 "remove administratively set MAC on VF %d\n", 9783 vf); 9784 } else if (is_valid_ether_addr(mac)) { 9785 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC; 9786 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", 9787 mac, vf); 9788 dev_info(&adapter->pdev->dev, 9789 "Reload the VF driver to make this change effective."); 9790 /* Generate additional warning if PF is down */ 9791 if (test_bit(__IGB_DOWN, &adapter->state)) { 9792 dev_warn(&adapter->pdev->dev, 9793 "The VF MAC address has been set, but the PF device is not up.\n"); 9794 dev_warn(&adapter->pdev->dev, 9795 "Bring the PF device up before attempting to use the VF device.\n"); 9796 } 9797 } else { 9798 return -EINVAL; 9799 } 9800 return igb_set_vf_mac(adapter, vf, mac); 9801 } 9802 9803 static int igb_link_mbps(int internal_link_speed) 9804 { 9805 switch (internal_link_speed) { 9806 case SPEED_100: 9807 return 100; 9808 case SPEED_1000: 9809 return 1000; 9810 default: 9811 return 0; 9812 } 9813 } 9814 9815 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate, 9816 int link_speed) 9817 { 9818 int rf_dec, rf_int; 9819 u32 bcnrc_val; 9820 9821 if (tx_rate != 0) { 9822 /* Calculate the rate factor values to set */ 9823 rf_int = link_speed / tx_rate; 9824 rf_dec = (link_speed - (rf_int * tx_rate)); 9825 rf_dec = (rf_dec * BIT(E1000_RTTBCNRC_RF_INT_SHIFT)) / 9826 tx_rate; 9827 9828 bcnrc_val = E1000_RTTBCNRC_RS_ENA; 9829 bcnrc_val |= ((rf_int << E1000_RTTBCNRC_RF_INT_SHIFT) & 9830 E1000_RTTBCNRC_RF_INT_MASK); 9831 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK); 9832 } else { 9833 bcnrc_val = 0; 9834 } 9835 9836 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */ 9837 /* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM 9838 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported. 9839 */ 9840 wr32(E1000_RTTBCNRM, 0x14); 9841 wr32(E1000_RTTBCNRC, bcnrc_val); 9842 } 9843 9844 static void igb_check_vf_rate_limit(struct igb_adapter *adapter) 9845 { 9846 int actual_link_speed, i; 9847 bool reset_rate = false; 9848 9849 /* VF TX rate limit was not set or not supported */ 9850 if ((adapter->vf_rate_link_speed == 0) || 9851 (adapter->hw.mac.type != e1000_82576)) 9852 return; 9853 9854 actual_link_speed = igb_link_mbps(adapter->link_speed); 9855 if (actual_link_speed != adapter->vf_rate_link_speed) { 9856 reset_rate = true; 9857 adapter->vf_rate_link_speed = 0; 9858 dev_info(&adapter->pdev->dev, 9859 "Link speed has been changed. VF Transmit rate is disabled\n"); 9860 } 9861 9862 for (i = 0; i < adapter->vfs_allocated_count; i++) { 9863 if (reset_rate) 9864 adapter->vf_data[i].tx_rate = 0; 9865 9866 igb_set_vf_rate_limit(&adapter->hw, i, 9867 adapter->vf_data[i].tx_rate, 9868 actual_link_speed); 9869 } 9870 } 9871 9872 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, 9873 int min_tx_rate, int max_tx_rate) 9874 { 9875 struct igb_adapter *adapter = netdev_priv(netdev); 9876 struct e1000_hw *hw = &adapter->hw; 9877 int actual_link_speed; 9878 9879 if (hw->mac.type != e1000_82576) 9880 return -EOPNOTSUPP; 9881 9882 if (min_tx_rate) 9883 return -EINVAL; 9884 9885 actual_link_speed = igb_link_mbps(adapter->link_speed); 9886 if ((vf >= adapter->vfs_allocated_count) || 9887 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) || 9888 (max_tx_rate < 0) || 9889 (max_tx_rate > actual_link_speed)) 9890 return -EINVAL; 9891 9892 adapter->vf_rate_link_speed = actual_link_speed; 9893 adapter->vf_data[vf].tx_rate = (u16)max_tx_rate; 9894 igb_set_vf_rate_limit(hw, vf, max_tx_rate, actual_link_speed); 9895 9896 return 0; 9897 } 9898 9899 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf, 9900 bool setting) 9901 { 9902 struct igb_adapter *adapter = netdev_priv(netdev); 9903 struct e1000_hw *hw = &adapter->hw; 9904 u32 reg_val, reg_offset; 9905 9906 if (!adapter->vfs_allocated_count) 9907 return -EOPNOTSUPP; 9908 9909 if (vf >= adapter->vfs_allocated_count) 9910 return -EINVAL; 9911 9912 reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC; 9913 reg_val = rd32(reg_offset); 9914 if (setting) 9915 reg_val |= (BIT(vf) | 9916 BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT)); 9917 else 9918 reg_val &= ~(BIT(vf) | 9919 BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT)); 9920 wr32(reg_offset, reg_val); 9921 9922 adapter->vf_data[vf].spoofchk_enabled = setting; 9923 return 0; 9924 } 9925 9926 static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, bool setting) 9927 { 9928 struct igb_adapter *adapter = netdev_priv(netdev); 9929 9930 if (vf >= adapter->vfs_allocated_count) 9931 return -EINVAL; 9932 if (adapter->vf_data[vf].trusted == setting) 9933 return 0; 9934 9935 adapter->vf_data[vf].trusted = setting; 9936 9937 dev_info(&adapter->pdev->dev, "VF %u is %strusted\n", 9938 vf, setting ? "" : "not "); 9939 return 0; 9940 } 9941 9942 static int igb_ndo_get_vf_config(struct net_device *netdev, 9943 int vf, struct ifla_vf_info *ivi) 9944 { 9945 struct igb_adapter *adapter = netdev_priv(netdev); 9946 if (vf >= adapter->vfs_allocated_count) 9947 return -EINVAL; 9948 ivi->vf = vf; 9949 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN); 9950 ivi->max_tx_rate = adapter->vf_data[vf].tx_rate; 9951 ivi->min_tx_rate = 0; 9952 ivi->vlan = adapter->vf_data[vf].pf_vlan; 9953 ivi->qos = adapter->vf_data[vf].pf_qos; 9954 ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled; 9955 ivi->trusted = adapter->vf_data[vf].trusted; 9956 return 0; 9957 } 9958 9959 static void igb_vmm_control(struct igb_adapter *adapter) 9960 { 9961 struct e1000_hw *hw = &adapter->hw; 9962 u32 reg; 9963 9964 switch (hw->mac.type) { 9965 case e1000_82575: 9966 case e1000_i210: 9967 case e1000_i211: 9968 case e1000_i354: 9969 default: 9970 /* replication is not supported for 82575 */ 9971 return; 9972 case e1000_82576: 9973 /* notify HW that the MAC is adding vlan tags */ 9974 reg = rd32(E1000_DTXCTL); 9975 reg |= E1000_DTXCTL_VLAN_ADDED; 9976 wr32(E1000_DTXCTL, reg); 9977 fallthrough; 9978 case e1000_82580: 9979 /* enable replication vlan tag stripping */ 9980 reg = rd32(E1000_RPLOLR); 9981 reg |= E1000_RPLOLR_STRVLAN; 9982 wr32(E1000_RPLOLR, reg); 9983 fallthrough; 9984 case e1000_i350: 9985 /* none of the above registers are supported by i350 */ 9986 break; 9987 } 9988 9989 if (adapter->vfs_allocated_count) { 9990 igb_vmdq_set_loopback_pf(hw, true); 9991 igb_vmdq_set_replication_pf(hw, true); 9992 igb_vmdq_set_anti_spoofing_pf(hw, true, 9993 adapter->vfs_allocated_count); 9994 } else { 9995 igb_vmdq_set_loopback_pf(hw, false); 9996 igb_vmdq_set_replication_pf(hw, false); 9997 } 9998 } 9999 10000 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba) 10001 { 10002 struct e1000_hw *hw = &adapter->hw; 10003 u32 dmac_thr; 10004 u16 hwm; 10005 u32 reg; 10006 10007 if (hw->mac.type > e1000_82580) { 10008 if (adapter->flags & IGB_FLAG_DMAC) { 10009 /* force threshold to 0. */ 10010 wr32(E1000_DMCTXTH, 0); 10011 10012 /* DMA Coalescing high water mark needs to be greater 10013 * than the Rx threshold. Set hwm to PBA - max frame 10014 * size in 16B units, capping it at PBA - 6KB. 10015 */ 10016 hwm = 64 * (pba - 6); 10017 reg = rd32(E1000_FCRTC); 10018 reg &= ~E1000_FCRTC_RTH_COAL_MASK; 10019 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT) 10020 & E1000_FCRTC_RTH_COAL_MASK); 10021 wr32(E1000_FCRTC, reg); 10022 10023 /* Set the DMA Coalescing Rx threshold to PBA - 2 * max 10024 * frame size, capping it at PBA - 10KB. 10025 */ 10026 dmac_thr = pba - 10; 10027 reg = rd32(E1000_DMACR); 10028 reg &= ~E1000_DMACR_DMACTHR_MASK; 10029 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT) 10030 & E1000_DMACR_DMACTHR_MASK); 10031 10032 /* transition to L0x or L1 if available..*/ 10033 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK); 10034 10035 /* watchdog timer= +-1000 usec in 32usec intervals */ 10036 reg |= (1000 >> 5); 10037 10038 /* Disable BMC-to-OS Watchdog Enable */ 10039 if (hw->mac.type != e1000_i354) 10040 reg &= ~E1000_DMACR_DC_BMC2OSW_EN; 10041 wr32(E1000_DMACR, reg); 10042 10043 /* no lower threshold to disable 10044 * coalescing(smart fifb)-UTRESH=0 10045 */ 10046 wr32(E1000_DMCRTRH, 0); 10047 10048 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4); 10049 10050 wr32(E1000_DMCTLX, reg); 10051 10052 /* free space in tx packet buffer to wake from 10053 * DMA coal 10054 */ 10055 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE - 10056 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6); 10057 } 10058 10059 if (hw->mac.type >= e1000_i210 || 10060 (adapter->flags & IGB_FLAG_DMAC)) { 10061 reg = rd32(E1000_PCIEMISC); 10062 reg |= E1000_PCIEMISC_LX_DECISION; 10063 wr32(E1000_PCIEMISC, reg); 10064 } /* endif adapter->dmac is not disabled */ 10065 } else if (hw->mac.type == e1000_82580) { 10066 u32 reg = rd32(E1000_PCIEMISC); 10067 10068 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION); 10069 wr32(E1000_DMACR, 0); 10070 } 10071 } 10072 10073 /** 10074 * igb_read_i2c_byte - Reads 8 bit word over I2C 10075 * @hw: pointer to hardware structure 10076 * @byte_offset: byte offset to read 10077 * @dev_addr: device address 10078 * @data: value read 10079 * 10080 * Performs byte read operation over I2C interface at 10081 * a specified device address. 10082 **/ 10083 s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset, 10084 u8 dev_addr, u8 *data) 10085 { 10086 struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw); 10087 struct i2c_client *this_client = adapter->i2c_client; 10088 s32 status; 10089 u16 swfw_mask = 0; 10090 10091 if (!this_client) 10092 return E1000_ERR_I2C; 10093 10094 swfw_mask = E1000_SWFW_PHY0_SM; 10095 10096 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) 10097 return E1000_ERR_SWFW_SYNC; 10098 10099 status = i2c_smbus_read_byte_data(this_client, byte_offset); 10100 hw->mac.ops.release_swfw_sync(hw, swfw_mask); 10101 10102 if (status < 0) 10103 return E1000_ERR_I2C; 10104 else { 10105 *data = status; 10106 return 0; 10107 } 10108 } 10109 10110 /** 10111 * igb_write_i2c_byte - Writes 8 bit word over I2C 10112 * @hw: pointer to hardware structure 10113 * @byte_offset: byte offset to write 10114 * @dev_addr: device address 10115 * @data: value to write 10116 * 10117 * Performs byte write operation over I2C interface at 10118 * a specified device address. 10119 **/ 10120 s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset, 10121 u8 dev_addr, u8 data) 10122 { 10123 struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw); 10124 struct i2c_client *this_client = adapter->i2c_client; 10125 s32 status; 10126 u16 swfw_mask = E1000_SWFW_PHY0_SM; 10127 10128 if (!this_client) 10129 return E1000_ERR_I2C; 10130 10131 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) 10132 return E1000_ERR_SWFW_SYNC; 10133 status = i2c_smbus_write_byte_data(this_client, byte_offset, data); 10134 hw->mac.ops.release_swfw_sync(hw, swfw_mask); 10135 10136 if (status) 10137 return E1000_ERR_I2C; 10138 else 10139 return 0; 10140 10141 } 10142 10143 int igb_reinit_queues(struct igb_adapter *adapter) 10144 { 10145 struct net_device *netdev = adapter->netdev; 10146 struct pci_dev *pdev = adapter->pdev; 10147 int err = 0; 10148 10149 if (netif_running(netdev)) 10150 igb_close(netdev); 10151 10152 igb_reset_interrupt_capability(adapter); 10153 10154 if (igb_init_interrupt_scheme(adapter, true)) { 10155 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 10156 return -ENOMEM; 10157 } 10158 10159 if (netif_running(netdev)) 10160 err = igb_open(netdev); 10161 10162 return err; 10163 } 10164 10165 static void igb_nfc_filter_exit(struct igb_adapter *adapter) 10166 { 10167 struct igb_nfc_filter *rule; 10168 10169 spin_lock(&adapter->nfc_lock); 10170 10171 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) 10172 igb_erase_filter(adapter, rule); 10173 10174 hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node) 10175 igb_erase_filter(adapter, rule); 10176 10177 spin_unlock(&adapter->nfc_lock); 10178 } 10179 10180 static void igb_nfc_filter_restore(struct igb_adapter *adapter) 10181 { 10182 struct igb_nfc_filter *rule; 10183 10184 spin_lock(&adapter->nfc_lock); 10185 10186 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) 10187 igb_add_filter(adapter, rule); 10188 10189 spin_unlock(&adapter->nfc_lock); 10190 } 10191 /* igb_main.c */ 10192