1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2007 - 2018 Intel Corporation. */
3
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5
6 #include <linux/module.h>
7 #include <linux/types.h>
8 #include <linux/init.h>
9 #include <linux/bitops.h>
10 #include <linux/vmalloc.h>
11 #include <linux/pagemap.h>
12 #include <linux/netdevice.h>
13 #include <linux/ipv6.h>
14 #include <linux/slab.h>
15 #include <net/checksum.h>
16 #include <net/ip6_checksum.h>
17 #include <net/pkt_sched.h>
18 #include <net/pkt_cls.h>
19 #include <linux/net_tstamp.h>
20 #include <linux/mii.h>
21 #include <linux/ethtool.h>
22 #include <linux/if.h>
23 #include <linux/if_vlan.h>
24 #include <linux/pci.h>
25 #include <linux/delay.h>
26 #include <linux/interrupt.h>
27 #include <linux/ip.h>
28 #include <linux/tcp.h>
29 #include <linux/sctp.h>
30 #include <linux/if_ether.h>
31 #include <linux/prefetch.h>
32 #include <linux/bpf.h>
33 #include <linux/bpf_trace.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/etherdevice.h>
36 #include <linux/lockdep.h>
37 #ifdef CONFIG_IGB_DCA
38 #include <linux/dca.h>
39 #endif
40 #include <linux/i2c.h>
41 #include "igb.h"
42
43 enum queue_mode {
44 QUEUE_MODE_STRICT_PRIORITY,
45 QUEUE_MODE_STREAM_RESERVATION,
46 };
47
48 enum tx_queue_prio {
49 TX_QUEUE_PRIO_HIGH,
50 TX_QUEUE_PRIO_LOW,
51 };
52
53 char igb_driver_name[] = "igb";
54 static const char igb_driver_string[] =
55 "Intel(R) Gigabit Ethernet Network Driver";
56 static const char igb_copyright[] =
57 "Copyright (c) 2007-2014 Intel Corporation.";
58
59 static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
61 };
62
63 static const struct pci_device_id igb_pci_tbl[] = {
64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 },
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
99 /* required last entry */
100 {0, }
101 };
102
103 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
104
105 static int igb_setup_all_tx_resources(struct igb_adapter *);
106 static int igb_setup_all_rx_resources(struct igb_adapter *);
107 static void igb_free_all_tx_resources(struct igb_adapter *);
108 static void igb_free_all_rx_resources(struct igb_adapter *);
109 static void igb_setup_mrqc(struct igb_adapter *);
110 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
111 static void igb_remove(struct pci_dev *pdev);
112 static void igb_init_queue_configuration(struct igb_adapter *adapter);
113 static int igb_sw_init(struct igb_adapter *);
114 int igb_open(struct net_device *);
115 int igb_close(struct net_device *);
116 static void igb_configure(struct igb_adapter *);
117 static void igb_configure_tx(struct igb_adapter *);
118 static void igb_configure_rx(struct igb_adapter *);
119 static void igb_clean_all_tx_rings(struct igb_adapter *);
120 static void igb_clean_all_rx_rings(struct igb_adapter *);
121 static void igb_clean_tx_ring(struct igb_ring *);
122 static void igb_clean_rx_ring(struct igb_ring *);
123 static void igb_set_rx_mode(struct net_device *);
124 static void igb_update_phy_info(struct timer_list *);
125 static void igb_watchdog(struct timer_list *);
126 static void igb_watchdog_task(struct work_struct *);
127 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
128 static void igb_get_stats64(struct net_device *dev,
129 struct rtnl_link_stats64 *stats);
130 static int igb_change_mtu(struct net_device *, int);
131 static int igb_set_mac(struct net_device *, void *);
132 static void igb_set_uta(struct igb_adapter *adapter, bool set);
133 static irqreturn_t igb_intr(int irq, void *);
134 static irqreturn_t igb_intr_msi(int irq, void *);
135 static irqreturn_t igb_msix_other(int irq, void *);
136 static irqreturn_t igb_msix_ring(int irq, void *);
137 #ifdef CONFIG_IGB_DCA
138 static void igb_update_dca(struct igb_q_vector *);
139 static void igb_setup_dca(struct igb_adapter *);
140 #endif /* CONFIG_IGB_DCA */
141 static int igb_poll(struct napi_struct *, int);
142 static bool igb_clean_tx_irq(struct igb_q_vector *, int);
143 static int igb_clean_rx_irq(struct igb_q_vector *, int);
144 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
145 static void igb_tx_timeout(struct net_device *, unsigned int txqueue);
146 static void igb_reset_task(struct work_struct *);
147 static void igb_vlan_mode(struct net_device *netdev,
148 netdev_features_t features);
149 static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
150 static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
151 static void igb_restore_vlan(struct igb_adapter *);
152 static void igb_rar_set_index(struct igb_adapter *, u32);
153 static void igb_ping_all_vfs(struct igb_adapter *);
154 static void igb_msg_task(struct igb_adapter *);
155 static void igb_vmm_control(struct igb_adapter *);
156 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
157 static void igb_flush_mac_table(struct igb_adapter *);
158 static int igb_available_rars(struct igb_adapter *, u8);
159 static void igb_set_default_mac_filter(struct igb_adapter *);
160 static int igb_uc_sync(struct net_device *, const unsigned char *);
161 static int igb_uc_unsync(struct net_device *, const unsigned char *);
162 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
163 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
164 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
165 int vf, u16 vlan, u8 qos, __be16 vlan_proto);
166 static int igb_ndo_set_vf_bw(struct net_device *, int, int, int);
167 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
168 bool setting);
169 static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf,
170 bool setting);
171 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
172 struct ifla_vf_info *ivi);
173 static void igb_check_vf_rate_limit(struct igb_adapter *);
174 static void igb_nfc_filter_exit(struct igb_adapter *adapter);
175 static void igb_nfc_filter_restore(struct igb_adapter *adapter);
176
177 #ifdef CONFIG_PCI_IOV
178 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
179 static int igb_disable_sriov(struct pci_dev *dev, bool reinit);
180 #endif
181
182 static int igb_suspend(struct device *);
183 static int igb_resume(struct device *);
184 static int igb_runtime_suspend(struct device *dev);
185 static int igb_runtime_resume(struct device *dev);
186 static int igb_runtime_idle(struct device *dev);
187 #ifdef CONFIG_PM
188 static const struct dev_pm_ops igb_pm_ops = {
189 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
190 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
191 igb_runtime_idle)
192 };
193 #endif
194 static void igb_shutdown(struct pci_dev *);
195 static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs);
196 #ifdef CONFIG_IGB_DCA
197 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
198 static struct notifier_block dca_notifier = {
199 .notifier_call = igb_notify_dca,
200 .next = NULL,
201 .priority = 0
202 };
203 #endif
204 #ifdef CONFIG_PCI_IOV
205 static unsigned int max_vfs;
206 module_param(max_vfs, uint, 0);
207 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function");
208 #endif /* CONFIG_PCI_IOV */
209
210 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
211 pci_channel_state_t);
212 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
213 static void igb_io_resume(struct pci_dev *);
214
215 static const struct pci_error_handlers igb_err_handler = {
216 .error_detected = igb_io_error_detected,
217 .slot_reset = igb_io_slot_reset,
218 .resume = igb_io_resume,
219 };
220
221 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
222
223 static struct pci_driver igb_driver = {
224 .name = igb_driver_name,
225 .id_table = igb_pci_tbl,
226 .probe = igb_probe,
227 .remove = igb_remove,
228 #ifdef CONFIG_PM
229 .driver.pm = &igb_pm_ops,
230 #endif
231 .shutdown = igb_shutdown,
232 .sriov_configure = igb_pci_sriov_configure,
233 .err_handler = &igb_err_handler
234 };
235
236 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
237 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
238 MODULE_LICENSE("GPL v2");
239
240 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
241 static int debug = -1;
242 module_param(debug, int, 0);
243 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
244
245 struct igb_reg_info {
246 u32 ofs;
247 char *name;
248 };
249
250 static const struct igb_reg_info igb_reg_info_tbl[] = {
251
252 /* General Registers */
253 {E1000_CTRL, "CTRL"},
254 {E1000_STATUS, "STATUS"},
255 {E1000_CTRL_EXT, "CTRL_EXT"},
256
257 /* Interrupt Registers */
258 {E1000_ICR, "ICR"},
259
260 /* RX Registers */
261 {E1000_RCTL, "RCTL"},
262 {E1000_RDLEN(0), "RDLEN"},
263 {E1000_RDH(0), "RDH"},
264 {E1000_RDT(0), "RDT"},
265 {E1000_RXDCTL(0), "RXDCTL"},
266 {E1000_RDBAL(0), "RDBAL"},
267 {E1000_RDBAH(0), "RDBAH"},
268
269 /* TX Registers */
270 {E1000_TCTL, "TCTL"},
271 {E1000_TDBAL(0), "TDBAL"},
272 {E1000_TDBAH(0), "TDBAH"},
273 {E1000_TDLEN(0), "TDLEN"},
274 {E1000_TDH(0), "TDH"},
275 {E1000_TDT(0), "TDT"},
276 {E1000_TXDCTL(0), "TXDCTL"},
277 {E1000_TDFH, "TDFH"},
278 {E1000_TDFT, "TDFT"},
279 {E1000_TDFHS, "TDFHS"},
280 {E1000_TDFPC, "TDFPC"},
281
282 /* List Terminator */
283 {}
284 };
285
286 /* igb_regdump - register printout routine */
igb_regdump(struct e1000_hw * hw,struct igb_reg_info * reginfo)287 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
288 {
289 int n = 0;
290 char rname[16];
291 u32 regs[8];
292
293 switch (reginfo->ofs) {
294 case E1000_RDLEN(0):
295 for (n = 0; n < 4; n++)
296 regs[n] = rd32(E1000_RDLEN(n));
297 break;
298 case E1000_RDH(0):
299 for (n = 0; n < 4; n++)
300 regs[n] = rd32(E1000_RDH(n));
301 break;
302 case E1000_RDT(0):
303 for (n = 0; n < 4; n++)
304 regs[n] = rd32(E1000_RDT(n));
305 break;
306 case E1000_RXDCTL(0):
307 for (n = 0; n < 4; n++)
308 regs[n] = rd32(E1000_RXDCTL(n));
309 break;
310 case E1000_RDBAL(0):
311 for (n = 0; n < 4; n++)
312 regs[n] = rd32(E1000_RDBAL(n));
313 break;
314 case E1000_RDBAH(0):
315 for (n = 0; n < 4; n++)
316 regs[n] = rd32(E1000_RDBAH(n));
317 break;
318 case E1000_TDBAL(0):
319 for (n = 0; n < 4; n++)
320 regs[n] = rd32(E1000_TDBAL(n));
321 break;
322 case E1000_TDBAH(0):
323 for (n = 0; n < 4; n++)
324 regs[n] = rd32(E1000_TDBAH(n));
325 break;
326 case E1000_TDLEN(0):
327 for (n = 0; n < 4; n++)
328 regs[n] = rd32(E1000_TDLEN(n));
329 break;
330 case E1000_TDH(0):
331 for (n = 0; n < 4; n++)
332 regs[n] = rd32(E1000_TDH(n));
333 break;
334 case E1000_TDT(0):
335 for (n = 0; n < 4; n++)
336 regs[n] = rd32(E1000_TDT(n));
337 break;
338 case E1000_TXDCTL(0):
339 for (n = 0; n < 4; n++)
340 regs[n] = rd32(E1000_TXDCTL(n));
341 break;
342 default:
343 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
344 return;
345 }
346
347 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
348 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
349 regs[2], regs[3]);
350 }
351
352 /* igb_dump - Print registers, Tx-rings and Rx-rings */
igb_dump(struct igb_adapter * adapter)353 static void igb_dump(struct igb_adapter *adapter)
354 {
355 struct net_device *netdev = adapter->netdev;
356 struct e1000_hw *hw = &adapter->hw;
357 struct igb_reg_info *reginfo;
358 struct igb_ring *tx_ring;
359 union e1000_adv_tx_desc *tx_desc;
360 struct my_u0 { __le64 a; __le64 b; } *u0;
361 struct igb_ring *rx_ring;
362 union e1000_adv_rx_desc *rx_desc;
363 u32 staterr;
364 u16 i, n;
365
366 if (!netif_msg_hw(adapter))
367 return;
368
369 /* Print netdevice Info */
370 if (netdev) {
371 dev_info(&adapter->pdev->dev, "Net device Info\n");
372 pr_info("Device Name state trans_start\n");
373 pr_info("%-15s %016lX %016lX\n", netdev->name,
374 netdev->state, dev_trans_start(netdev));
375 }
376
377 /* Print Registers */
378 dev_info(&adapter->pdev->dev, "Register Dump\n");
379 pr_info(" Register Name Value\n");
380 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
381 reginfo->name; reginfo++) {
382 igb_regdump(hw, reginfo);
383 }
384
385 /* Print TX Ring Summary */
386 if (!netdev || !netif_running(netdev))
387 goto exit;
388
389 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
390 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
391 for (n = 0; n < adapter->num_tx_queues; n++) {
392 struct igb_tx_buffer *buffer_info;
393 tx_ring = adapter->tx_ring[n];
394 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
395 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
396 n, tx_ring->next_to_use, tx_ring->next_to_clean,
397 (u64)dma_unmap_addr(buffer_info, dma),
398 dma_unmap_len(buffer_info, len),
399 buffer_info->next_to_watch,
400 (u64)buffer_info->time_stamp);
401 }
402
403 /* Print TX Rings */
404 if (!netif_msg_tx_done(adapter))
405 goto rx_ring_summary;
406
407 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
408
409 /* Transmit Descriptor Formats
410 *
411 * Advanced Transmit Descriptor
412 * +--------------------------------------------------------------+
413 * 0 | Buffer Address [63:0] |
414 * +--------------------------------------------------------------+
415 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
416 * +--------------------------------------------------------------+
417 * 63 46 45 40 39 38 36 35 32 31 24 15 0
418 */
419
420 for (n = 0; n < adapter->num_tx_queues; n++) {
421 tx_ring = adapter->tx_ring[n];
422 pr_info("------------------------------------\n");
423 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
424 pr_info("------------------------------------\n");
425 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n");
426
427 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
428 const char *next_desc;
429 struct igb_tx_buffer *buffer_info;
430 tx_desc = IGB_TX_DESC(tx_ring, i);
431 buffer_info = &tx_ring->tx_buffer_info[i];
432 u0 = (struct my_u0 *)tx_desc;
433 if (i == tx_ring->next_to_use &&
434 i == tx_ring->next_to_clean)
435 next_desc = " NTC/U";
436 else if (i == tx_ring->next_to_use)
437 next_desc = " NTU";
438 else if (i == tx_ring->next_to_clean)
439 next_desc = " NTC";
440 else
441 next_desc = "";
442
443 pr_info("T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n",
444 i, le64_to_cpu(u0->a),
445 le64_to_cpu(u0->b),
446 (u64)dma_unmap_addr(buffer_info, dma),
447 dma_unmap_len(buffer_info, len),
448 buffer_info->next_to_watch,
449 (u64)buffer_info->time_stamp,
450 buffer_info->skb, next_desc);
451
452 if (netif_msg_pktdata(adapter) && buffer_info->skb)
453 print_hex_dump(KERN_INFO, "",
454 DUMP_PREFIX_ADDRESS,
455 16, 1, buffer_info->skb->data,
456 dma_unmap_len(buffer_info, len),
457 true);
458 }
459 }
460
461 /* Print RX Rings Summary */
462 rx_ring_summary:
463 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
464 pr_info("Queue [NTU] [NTC]\n");
465 for (n = 0; n < adapter->num_rx_queues; n++) {
466 rx_ring = adapter->rx_ring[n];
467 pr_info(" %5d %5X %5X\n",
468 n, rx_ring->next_to_use, rx_ring->next_to_clean);
469 }
470
471 /* Print RX Rings */
472 if (!netif_msg_rx_status(adapter))
473 goto exit;
474
475 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
476
477 /* Advanced Receive Descriptor (Read) Format
478 * 63 1 0
479 * +-----------------------------------------------------+
480 * 0 | Packet Buffer Address [63:1] |A0/NSE|
481 * +----------------------------------------------+------+
482 * 8 | Header Buffer Address [63:1] | DD |
483 * +-----------------------------------------------------+
484 *
485 *
486 * Advanced Receive Descriptor (Write-Back) Format
487 *
488 * 63 48 47 32 31 30 21 20 17 16 4 3 0
489 * +------------------------------------------------------+
490 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
491 * | Checksum Ident | | | | Type | Type |
492 * +------------------------------------------------------+
493 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
494 * +------------------------------------------------------+
495 * 63 48 47 32 31 20 19 0
496 */
497
498 for (n = 0; n < adapter->num_rx_queues; n++) {
499 rx_ring = adapter->rx_ring[n];
500 pr_info("------------------------------------\n");
501 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
502 pr_info("------------------------------------\n");
503 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
504 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n");
505
506 for (i = 0; i < rx_ring->count; i++) {
507 const char *next_desc;
508 struct igb_rx_buffer *buffer_info;
509 buffer_info = &rx_ring->rx_buffer_info[i];
510 rx_desc = IGB_RX_DESC(rx_ring, i);
511 u0 = (struct my_u0 *)rx_desc;
512 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
513
514 if (i == rx_ring->next_to_use)
515 next_desc = " NTU";
516 else if (i == rx_ring->next_to_clean)
517 next_desc = " NTC";
518 else
519 next_desc = "";
520
521 if (staterr & E1000_RXD_STAT_DD) {
522 /* Descriptor Done */
523 pr_info("%s[0x%03X] %016llX %016llX ---------------- %s\n",
524 "RWB", i,
525 le64_to_cpu(u0->a),
526 le64_to_cpu(u0->b),
527 next_desc);
528 } else {
529 pr_info("%s[0x%03X] %016llX %016llX %016llX %s\n",
530 "R ", i,
531 le64_to_cpu(u0->a),
532 le64_to_cpu(u0->b),
533 (u64)buffer_info->dma,
534 next_desc);
535
536 if (netif_msg_pktdata(adapter) &&
537 buffer_info->dma && buffer_info->page) {
538 print_hex_dump(KERN_INFO, "",
539 DUMP_PREFIX_ADDRESS,
540 16, 1,
541 page_address(buffer_info->page) +
542 buffer_info->page_offset,
543 igb_rx_bufsz(rx_ring), true);
544 }
545 }
546 }
547 }
548
549 exit:
550 return;
551 }
552
553 /**
554 * igb_get_i2c_data - Reads the I2C SDA data bit
555 * @data: opaque pointer to adapter struct
556 *
557 * Returns the I2C data bit value
558 **/
igb_get_i2c_data(void * data)559 static int igb_get_i2c_data(void *data)
560 {
561 struct igb_adapter *adapter = (struct igb_adapter *)data;
562 struct e1000_hw *hw = &adapter->hw;
563 s32 i2cctl = rd32(E1000_I2CPARAMS);
564
565 return !!(i2cctl & E1000_I2C_DATA_IN);
566 }
567
568 /**
569 * igb_set_i2c_data - Sets the I2C data bit
570 * @data: pointer to hardware structure
571 * @state: I2C data value (0 or 1) to set
572 *
573 * Sets the I2C data bit
574 **/
igb_set_i2c_data(void * data,int state)575 static void igb_set_i2c_data(void *data, int state)
576 {
577 struct igb_adapter *adapter = (struct igb_adapter *)data;
578 struct e1000_hw *hw = &adapter->hw;
579 s32 i2cctl = rd32(E1000_I2CPARAMS);
580
581 if (state) {
582 i2cctl |= E1000_I2C_DATA_OUT | E1000_I2C_DATA_OE_N;
583 } else {
584 i2cctl &= ~E1000_I2C_DATA_OE_N;
585 i2cctl &= ~E1000_I2C_DATA_OUT;
586 }
587
588 wr32(E1000_I2CPARAMS, i2cctl);
589 wrfl();
590 }
591
592 /**
593 * igb_set_i2c_clk - Sets the I2C SCL clock
594 * @data: pointer to hardware structure
595 * @state: state to set clock
596 *
597 * Sets the I2C clock line to state
598 **/
igb_set_i2c_clk(void * data,int state)599 static void igb_set_i2c_clk(void *data, int state)
600 {
601 struct igb_adapter *adapter = (struct igb_adapter *)data;
602 struct e1000_hw *hw = &adapter->hw;
603 s32 i2cctl = rd32(E1000_I2CPARAMS);
604
605 if (state) {
606 i2cctl |= E1000_I2C_CLK_OUT | E1000_I2C_CLK_OE_N;
607 } else {
608 i2cctl &= ~E1000_I2C_CLK_OUT;
609 i2cctl &= ~E1000_I2C_CLK_OE_N;
610 }
611 wr32(E1000_I2CPARAMS, i2cctl);
612 wrfl();
613 }
614
615 /**
616 * igb_get_i2c_clk - Gets the I2C SCL clock state
617 * @data: pointer to hardware structure
618 *
619 * Gets the I2C clock state
620 **/
igb_get_i2c_clk(void * data)621 static int igb_get_i2c_clk(void *data)
622 {
623 struct igb_adapter *adapter = (struct igb_adapter *)data;
624 struct e1000_hw *hw = &adapter->hw;
625 s32 i2cctl = rd32(E1000_I2CPARAMS);
626
627 return !!(i2cctl & E1000_I2C_CLK_IN);
628 }
629
630 static const struct i2c_algo_bit_data igb_i2c_algo = {
631 .setsda = igb_set_i2c_data,
632 .setscl = igb_set_i2c_clk,
633 .getsda = igb_get_i2c_data,
634 .getscl = igb_get_i2c_clk,
635 .udelay = 5,
636 .timeout = 20,
637 };
638
639 /**
640 * igb_get_hw_dev - return device
641 * @hw: pointer to hardware structure
642 *
643 * used by hardware layer to print debugging information
644 **/
igb_get_hw_dev(struct e1000_hw * hw)645 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
646 {
647 struct igb_adapter *adapter = hw->back;
648 return adapter->netdev;
649 }
650
651 /**
652 * igb_init_module - Driver Registration Routine
653 *
654 * igb_init_module is the first routine called when the driver is
655 * loaded. All it does is register with the PCI subsystem.
656 **/
igb_init_module(void)657 static int __init igb_init_module(void)
658 {
659 int ret;
660
661 pr_info("%s\n", igb_driver_string);
662 pr_info("%s\n", igb_copyright);
663
664 #ifdef CONFIG_IGB_DCA
665 dca_register_notify(&dca_notifier);
666 #endif
667 ret = pci_register_driver(&igb_driver);
668 return ret;
669 }
670
671 module_init(igb_init_module);
672
673 /**
674 * igb_exit_module - Driver Exit Cleanup Routine
675 *
676 * igb_exit_module is called just before the driver is removed
677 * from memory.
678 **/
igb_exit_module(void)679 static void __exit igb_exit_module(void)
680 {
681 #ifdef CONFIG_IGB_DCA
682 dca_unregister_notify(&dca_notifier);
683 #endif
684 pci_unregister_driver(&igb_driver);
685 }
686
687 module_exit(igb_exit_module);
688
689 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
690 /**
691 * igb_cache_ring_register - Descriptor ring to register mapping
692 * @adapter: board private structure to initialize
693 *
694 * Once we know the feature-set enabled for the device, we'll cache
695 * the register offset the descriptor ring is assigned to.
696 **/
igb_cache_ring_register(struct igb_adapter * adapter)697 static void igb_cache_ring_register(struct igb_adapter *adapter)
698 {
699 int i = 0, j = 0;
700 u32 rbase_offset = adapter->vfs_allocated_count;
701
702 switch (adapter->hw.mac.type) {
703 case e1000_82576:
704 /* The queues are allocated for virtualization such that VF 0
705 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
706 * In order to avoid collision we start at the first free queue
707 * and continue consuming queues in the same sequence
708 */
709 if (adapter->vfs_allocated_count) {
710 for (; i < adapter->rss_queues; i++)
711 adapter->rx_ring[i]->reg_idx = rbase_offset +
712 Q_IDX_82576(i);
713 }
714 fallthrough;
715 case e1000_82575:
716 case e1000_82580:
717 case e1000_i350:
718 case e1000_i354:
719 case e1000_i210:
720 case e1000_i211:
721 default:
722 for (; i < adapter->num_rx_queues; i++)
723 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
724 for (; j < adapter->num_tx_queues; j++)
725 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
726 break;
727 }
728 }
729
igb_rd32(struct e1000_hw * hw,u32 reg)730 u32 igb_rd32(struct e1000_hw *hw, u32 reg)
731 {
732 struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw);
733 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
734 u32 value = 0;
735
736 if (E1000_REMOVED(hw_addr))
737 return ~value;
738
739 value = readl(&hw_addr[reg]);
740
741 /* reads should not return all F's */
742 if (!(~value) && (!reg || !(~readl(hw_addr)))) {
743 struct net_device *netdev = igb->netdev;
744 hw->hw_addr = NULL;
745 netdev_err(netdev, "PCIe link lost\n");
746 WARN(pci_device_is_present(igb->pdev),
747 "igb: Failed to read reg 0x%x!\n", reg);
748 }
749
750 return value;
751 }
752
753 /**
754 * igb_write_ivar - configure ivar for given MSI-X vector
755 * @hw: pointer to the HW structure
756 * @msix_vector: vector number we are allocating to a given ring
757 * @index: row index of IVAR register to write within IVAR table
758 * @offset: column offset of in IVAR, should be multiple of 8
759 *
760 * This function is intended to handle the writing of the IVAR register
761 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
762 * each containing an cause allocation for an Rx and Tx ring, and a
763 * variable number of rows depending on the number of queues supported.
764 **/
igb_write_ivar(struct e1000_hw * hw,int msix_vector,int index,int offset)765 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
766 int index, int offset)
767 {
768 u32 ivar = array_rd32(E1000_IVAR0, index);
769
770 /* clear any bits that are currently set */
771 ivar &= ~((u32)0xFF << offset);
772
773 /* write vector and valid bit */
774 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
775
776 array_wr32(E1000_IVAR0, index, ivar);
777 }
778
779 #define IGB_N0_QUEUE -1
igb_assign_vector(struct igb_q_vector * q_vector,int msix_vector)780 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
781 {
782 struct igb_adapter *adapter = q_vector->adapter;
783 struct e1000_hw *hw = &adapter->hw;
784 int rx_queue = IGB_N0_QUEUE;
785 int tx_queue = IGB_N0_QUEUE;
786 u32 msixbm = 0;
787
788 if (q_vector->rx.ring)
789 rx_queue = q_vector->rx.ring->reg_idx;
790 if (q_vector->tx.ring)
791 tx_queue = q_vector->tx.ring->reg_idx;
792
793 switch (hw->mac.type) {
794 case e1000_82575:
795 /* The 82575 assigns vectors using a bitmask, which matches the
796 * bitmask for the EICR/EIMS/EIMC registers. To assign one
797 * or more queues to a vector, we write the appropriate bits
798 * into the MSIXBM register for that vector.
799 */
800 if (rx_queue > IGB_N0_QUEUE)
801 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
802 if (tx_queue > IGB_N0_QUEUE)
803 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
804 if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0)
805 msixbm |= E1000_EIMS_OTHER;
806 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
807 q_vector->eims_value = msixbm;
808 break;
809 case e1000_82576:
810 /* 82576 uses a table that essentially consists of 2 columns
811 * with 8 rows. The ordering is column-major so we use the
812 * lower 3 bits as the row index, and the 4th bit as the
813 * column offset.
814 */
815 if (rx_queue > IGB_N0_QUEUE)
816 igb_write_ivar(hw, msix_vector,
817 rx_queue & 0x7,
818 (rx_queue & 0x8) << 1);
819 if (tx_queue > IGB_N0_QUEUE)
820 igb_write_ivar(hw, msix_vector,
821 tx_queue & 0x7,
822 ((tx_queue & 0x8) << 1) + 8);
823 q_vector->eims_value = BIT(msix_vector);
824 break;
825 case e1000_82580:
826 case e1000_i350:
827 case e1000_i354:
828 case e1000_i210:
829 case e1000_i211:
830 /* On 82580 and newer adapters the scheme is similar to 82576
831 * however instead of ordering column-major we have things
832 * ordered row-major. So we traverse the table by using
833 * bit 0 as the column offset, and the remaining bits as the
834 * row index.
835 */
836 if (rx_queue > IGB_N0_QUEUE)
837 igb_write_ivar(hw, msix_vector,
838 rx_queue >> 1,
839 (rx_queue & 0x1) << 4);
840 if (tx_queue > IGB_N0_QUEUE)
841 igb_write_ivar(hw, msix_vector,
842 tx_queue >> 1,
843 ((tx_queue & 0x1) << 4) + 8);
844 q_vector->eims_value = BIT(msix_vector);
845 break;
846 default:
847 BUG();
848 break;
849 }
850
851 /* add q_vector eims value to global eims_enable_mask */
852 adapter->eims_enable_mask |= q_vector->eims_value;
853
854 /* configure q_vector to set itr on first interrupt */
855 q_vector->set_itr = 1;
856 }
857
858 /**
859 * igb_configure_msix - Configure MSI-X hardware
860 * @adapter: board private structure to initialize
861 *
862 * igb_configure_msix sets up the hardware to properly
863 * generate MSI-X interrupts.
864 **/
igb_configure_msix(struct igb_adapter * adapter)865 static void igb_configure_msix(struct igb_adapter *adapter)
866 {
867 u32 tmp;
868 int i, vector = 0;
869 struct e1000_hw *hw = &adapter->hw;
870
871 adapter->eims_enable_mask = 0;
872
873 /* set vector for other causes, i.e. link changes */
874 switch (hw->mac.type) {
875 case e1000_82575:
876 tmp = rd32(E1000_CTRL_EXT);
877 /* enable MSI-X PBA support*/
878 tmp |= E1000_CTRL_EXT_PBA_CLR;
879
880 /* Auto-Mask interrupts upon ICR read. */
881 tmp |= E1000_CTRL_EXT_EIAME;
882 tmp |= E1000_CTRL_EXT_IRCA;
883
884 wr32(E1000_CTRL_EXT, tmp);
885
886 /* enable msix_other interrupt */
887 array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER);
888 adapter->eims_other = E1000_EIMS_OTHER;
889
890 break;
891
892 case e1000_82576:
893 case e1000_82580:
894 case e1000_i350:
895 case e1000_i354:
896 case e1000_i210:
897 case e1000_i211:
898 /* Turn on MSI-X capability first, or our settings
899 * won't stick. And it will take days to debug.
900 */
901 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
902 E1000_GPIE_PBA | E1000_GPIE_EIAME |
903 E1000_GPIE_NSICR);
904
905 /* enable msix_other interrupt */
906 adapter->eims_other = BIT(vector);
907 tmp = (vector++ | E1000_IVAR_VALID) << 8;
908
909 wr32(E1000_IVAR_MISC, tmp);
910 break;
911 default:
912 /* do nothing, since nothing else supports MSI-X */
913 break;
914 } /* switch (hw->mac.type) */
915
916 adapter->eims_enable_mask |= adapter->eims_other;
917
918 for (i = 0; i < adapter->num_q_vectors; i++)
919 igb_assign_vector(adapter->q_vector[i], vector++);
920
921 wrfl();
922 }
923
924 /**
925 * igb_request_msix - Initialize MSI-X interrupts
926 * @adapter: board private structure to initialize
927 *
928 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
929 * kernel.
930 **/
igb_request_msix(struct igb_adapter * adapter)931 static int igb_request_msix(struct igb_adapter *adapter)
932 {
933 unsigned int num_q_vectors = adapter->num_q_vectors;
934 struct net_device *netdev = adapter->netdev;
935 int i, err = 0, vector = 0, free_vector = 0;
936
937 err = request_irq(adapter->msix_entries[vector].vector,
938 igb_msix_other, 0, netdev->name, adapter);
939 if (err)
940 goto err_out;
941
942 if (num_q_vectors > MAX_Q_VECTORS) {
943 num_q_vectors = MAX_Q_VECTORS;
944 dev_warn(&adapter->pdev->dev,
945 "The number of queue vectors (%d) is higher than max allowed (%d)\n",
946 adapter->num_q_vectors, MAX_Q_VECTORS);
947 }
948 for (i = 0; i < num_q_vectors; i++) {
949 struct igb_q_vector *q_vector = adapter->q_vector[i];
950
951 vector++;
952
953 q_vector->itr_register = adapter->io_addr + E1000_EITR(vector);
954
955 if (q_vector->rx.ring && q_vector->tx.ring)
956 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
957 q_vector->rx.ring->queue_index);
958 else if (q_vector->tx.ring)
959 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
960 q_vector->tx.ring->queue_index);
961 else if (q_vector->rx.ring)
962 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
963 q_vector->rx.ring->queue_index);
964 else
965 sprintf(q_vector->name, "%s-unused", netdev->name);
966
967 err = request_irq(adapter->msix_entries[vector].vector,
968 igb_msix_ring, 0, q_vector->name,
969 q_vector);
970 if (err)
971 goto err_free;
972 }
973
974 igb_configure_msix(adapter);
975 return 0;
976
977 err_free:
978 /* free already assigned IRQs */
979 free_irq(adapter->msix_entries[free_vector++].vector, adapter);
980
981 vector--;
982 for (i = 0; i < vector; i++) {
983 free_irq(adapter->msix_entries[free_vector++].vector,
984 adapter->q_vector[i]);
985 }
986 err_out:
987 return err;
988 }
989
990 /**
991 * igb_free_q_vector - Free memory allocated for specific interrupt vector
992 * @adapter: board private structure to initialize
993 * @v_idx: Index of vector to be freed
994 *
995 * This function frees the memory allocated to the q_vector.
996 **/
igb_free_q_vector(struct igb_adapter * adapter,int v_idx)997 static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx)
998 {
999 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1000
1001 adapter->q_vector[v_idx] = NULL;
1002
1003 /* igb_get_stats64() might access the rings on this vector,
1004 * we must wait a grace period before freeing it.
1005 */
1006 if (q_vector)
1007 kfree_rcu(q_vector, rcu);
1008 }
1009
1010 /**
1011 * igb_reset_q_vector - Reset config for interrupt vector
1012 * @adapter: board private structure to initialize
1013 * @v_idx: Index of vector to be reset
1014 *
1015 * If NAPI is enabled it will delete any references to the
1016 * NAPI struct. This is preparation for igb_free_q_vector.
1017 **/
igb_reset_q_vector(struct igb_adapter * adapter,int v_idx)1018 static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx)
1019 {
1020 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1021
1022 /* Coming from igb_set_interrupt_capability, the vectors are not yet
1023 * allocated. So, q_vector is NULL so we should stop here.
1024 */
1025 if (!q_vector)
1026 return;
1027
1028 if (q_vector->tx.ring)
1029 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
1030
1031 if (q_vector->rx.ring)
1032 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
1033
1034 netif_napi_del(&q_vector->napi);
1035
1036 }
1037
igb_reset_interrupt_capability(struct igb_adapter * adapter)1038 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
1039 {
1040 int v_idx = adapter->num_q_vectors;
1041
1042 if (adapter->flags & IGB_FLAG_HAS_MSIX)
1043 pci_disable_msix(adapter->pdev);
1044 else if (adapter->flags & IGB_FLAG_HAS_MSI)
1045 pci_disable_msi(adapter->pdev);
1046
1047 while (v_idx--)
1048 igb_reset_q_vector(adapter, v_idx);
1049 }
1050
1051 /**
1052 * igb_free_q_vectors - Free memory allocated for interrupt vectors
1053 * @adapter: board private structure to initialize
1054 *
1055 * This function frees the memory allocated to the q_vectors. In addition if
1056 * NAPI is enabled it will delete any references to the NAPI struct prior
1057 * to freeing the q_vector.
1058 **/
igb_free_q_vectors(struct igb_adapter * adapter)1059 static void igb_free_q_vectors(struct igb_adapter *adapter)
1060 {
1061 int v_idx = adapter->num_q_vectors;
1062
1063 adapter->num_tx_queues = 0;
1064 adapter->num_rx_queues = 0;
1065 adapter->num_q_vectors = 0;
1066
1067 while (v_idx--) {
1068 igb_reset_q_vector(adapter, v_idx);
1069 igb_free_q_vector(adapter, v_idx);
1070 }
1071 }
1072
1073 /**
1074 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1075 * @adapter: board private structure to initialize
1076 *
1077 * This function resets the device so that it has 0 Rx queues, Tx queues, and
1078 * MSI-X interrupts allocated.
1079 */
igb_clear_interrupt_scheme(struct igb_adapter * adapter)1080 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1081 {
1082 igb_free_q_vectors(adapter);
1083 igb_reset_interrupt_capability(adapter);
1084 }
1085
1086 /**
1087 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1088 * @adapter: board private structure to initialize
1089 * @msix: boolean value of MSIX capability
1090 *
1091 * Attempt to configure interrupts using the best available
1092 * capabilities of the hardware and kernel.
1093 **/
igb_set_interrupt_capability(struct igb_adapter * adapter,bool msix)1094 static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix)
1095 {
1096 int err;
1097 int numvecs, i;
1098
1099 if (!msix)
1100 goto msi_only;
1101 adapter->flags |= IGB_FLAG_HAS_MSIX;
1102
1103 /* Number of supported queues. */
1104 adapter->num_rx_queues = adapter->rss_queues;
1105 if (adapter->vfs_allocated_count)
1106 adapter->num_tx_queues = 1;
1107 else
1108 adapter->num_tx_queues = adapter->rss_queues;
1109
1110 /* start with one vector for every Rx queue */
1111 numvecs = adapter->num_rx_queues;
1112
1113 /* if Tx handler is separate add 1 for every Tx queue */
1114 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1115 numvecs += adapter->num_tx_queues;
1116
1117 /* store the number of vectors reserved for queues */
1118 adapter->num_q_vectors = numvecs;
1119
1120 /* add 1 vector for link status interrupts */
1121 numvecs++;
1122 for (i = 0; i < numvecs; i++)
1123 adapter->msix_entries[i].entry = i;
1124
1125 err = pci_enable_msix_range(adapter->pdev,
1126 adapter->msix_entries,
1127 numvecs,
1128 numvecs);
1129 if (err > 0)
1130 return;
1131
1132 igb_reset_interrupt_capability(adapter);
1133
1134 /* If we can't do MSI-X, try MSI */
1135 msi_only:
1136 adapter->flags &= ~IGB_FLAG_HAS_MSIX;
1137 #ifdef CONFIG_PCI_IOV
1138 /* disable SR-IOV for non MSI-X configurations */
1139 if (adapter->vf_data) {
1140 struct e1000_hw *hw = &adapter->hw;
1141 /* disable iov and allow time for transactions to clear */
1142 pci_disable_sriov(adapter->pdev);
1143 msleep(500);
1144
1145 kfree(adapter->vf_mac_list);
1146 adapter->vf_mac_list = NULL;
1147 kfree(adapter->vf_data);
1148 adapter->vf_data = NULL;
1149 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1150 wrfl();
1151 msleep(100);
1152 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1153 }
1154 #endif
1155 adapter->vfs_allocated_count = 0;
1156 adapter->rss_queues = 1;
1157 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1158 adapter->num_rx_queues = 1;
1159 adapter->num_tx_queues = 1;
1160 adapter->num_q_vectors = 1;
1161 if (!pci_enable_msi(adapter->pdev))
1162 adapter->flags |= IGB_FLAG_HAS_MSI;
1163 }
1164
igb_add_ring(struct igb_ring * ring,struct igb_ring_container * head)1165 static void igb_add_ring(struct igb_ring *ring,
1166 struct igb_ring_container *head)
1167 {
1168 head->ring = ring;
1169 head->count++;
1170 }
1171
1172 /**
1173 * igb_alloc_q_vector - Allocate memory for a single interrupt vector
1174 * @adapter: board private structure to initialize
1175 * @v_count: q_vectors allocated on adapter, used for ring interleaving
1176 * @v_idx: index of vector in adapter struct
1177 * @txr_count: total number of Tx rings to allocate
1178 * @txr_idx: index of first Tx ring to allocate
1179 * @rxr_count: total number of Rx rings to allocate
1180 * @rxr_idx: index of first Rx ring to allocate
1181 *
1182 * We allocate one q_vector. If allocation fails we return -ENOMEM.
1183 **/
igb_alloc_q_vector(struct igb_adapter * adapter,int v_count,int v_idx,int txr_count,int txr_idx,int rxr_count,int rxr_idx)1184 static int igb_alloc_q_vector(struct igb_adapter *adapter,
1185 int v_count, int v_idx,
1186 int txr_count, int txr_idx,
1187 int rxr_count, int rxr_idx)
1188 {
1189 struct igb_q_vector *q_vector;
1190 struct igb_ring *ring;
1191 int ring_count;
1192 size_t size;
1193
1194 /* igb only supports 1 Tx and/or 1 Rx queue per vector */
1195 if (txr_count > 1 || rxr_count > 1)
1196 return -ENOMEM;
1197
1198 ring_count = txr_count + rxr_count;
1199 size = kmalloc_size_roundup(struct_size(q_vector, ring, ring_count));
1200
1201 /* allocate q_vector and rings */
1202 q_vector = adapter->q_vector[v_idx];
1203 if (!q_vector) {
1204 q_vector = kzalloc(size, GFP_KERNEL);
1205 } else if (size > ksize(q_vector)) {
1206 struct igb_q_vector *new_q_vector;
1207
1208 new_q_vector = kzalloc(size, GFP_KERNEL);
1209 if (new_q_vector)
1210 kfree_rcu(q_vector, rcu);
1211 q_vector = new_q_vector;
1212 } else {
1213 memset(q_vector, 0, size);
1214 }
1215 if (!q_vector)
1216 return -ENOMEM;
1217
1218 /* initialize NAPI */
1219 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll);
1220
1221 /* tie q_vector and adapter together */
1222 adapter->q_vector[v_idx] = q_vector;
1223 q_vector->adapter = adapter;
1224
1225 /* initialize work limits */
1226 q_vector->tx.work_limit = adapter->tx_work_limit;
1227
1228 /* initialize ITR configuration */
1229 q_vector->itr_register = adapter->io_addr + E1000_EITR(0);
1230 q_vector->itr_val = IGB_START_ITR;
1231
1232 /* initialize pointer to rings */
1233 ring = q_vector->ring;
1234
1235 /* intialize ITR */
1236 if (rxr_count) {
1237 /* rx or rx/tx vector */
1238 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
1239 q_vector->itr_val = adapter->rx_itr_setting;
1240 } else {
1241 /* tx only vector */
1242 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
1243 q_vector->itr_val = adapter->tx_itr_setting;
1244 }
1245
1246 if (txr_count) {
1247 /* assign generic ring traits */
1248 ring->dev = &adapter->pdev->dev;
1249 ring->netdev = adapter->netdev;
1250
1251 /* configure backlink on ring */
1252 ring->q_vector = q_vector;
1253
1254 /* update q_vector Tx values */
1255 igb_add_ring(ring, &q_vector->tx);
1256
1257 /* For 82575, context index must be unique per ring. */
1258 if (adapter->hw.mac.type == e1000_82575)
1259 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
1260
1261 /* apply Tx specific ring traits */
1262 ring->count = adapter->tx_ring_count;
1263 ring->queue_index = txr_idx;
1264
1265 ring->cbs_enable = false;
1266 ring->idleslope = 0;
1267 ring->sendslope = 0;
1268 ring->hicredit = 0;
1269 ring->locredit = 0;
1270
1271 u64_stats_init(&ring->tx_syncp);
1272 u64_stats_init(&ring->tx_syncp2);
1273
1274 /* assign ring to adapter */
1275 adapter->tx_ring[txr_idx] = ring;
1276
1277 /* push pointer to next ring */
1278 ring++;
1279 }
1280
1281 if (rxr_count) {
1282 /* assign generic ring traits */
1283 ring->dev = &adapter->pdev->dev;
1284 ring->netdev = adapter->netdev;
1285
1286 /* configure backlink on ring */
1287 ring->q_vector = q_vector;
1288
1289 /* update q_vector Rx values */
1290 igb_add_ring(ring, &q_vector->rx);
1291
1292 /* set flag indicating ring supports SCTP checksum offload */
1293 if (adapter->hw.mac.type >= e1000_82576)
1294 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
1295
1296 /* On i350, i354, i210, and i211, loopback VLAN packets
1297 * have the tag byte-swapped.
1298 */
1299 if (adapter->hw.mac.type >= e1000_i350)
1300 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
1301
1302 /* apply Rx specific ring traits */
1303 ring->count = adapter->rx_ring_count;
1304 ring->queue_index = rxr_idx;
1305
1306 u64_stats_init(&ring->rx_syncp);
1307
1308 /* assign ring to adapter */
1309 adapter->rx_ring[rxr_idx] = ring;
1310 }
1311
1312 return 0;
1313 }
1314
1315
1316 /**
1317 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1318 * @adapter: board private structure to initialize
1319 *
1320 * We allocate one q_vector per queue interrupt. If allocation fails we
1321 * return -ENOMEM.
1322 **/
igb_alloc_q_vectors(struct igb_adapter * adapter)1323 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1324 {
1325 int q_vectors = adapter->num_q_vectors;
1326 int rxr_remaining = adapter->num_rx_queues;
1327 int txr_remaining = adapter->num_tx_queues;
1328 int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1329 int err;
1330
1331 if (q_vectors >= (rxr_remaining + txr_remaining)) {
1332 for (; rxr_remaining; v_idx++) {
1333 err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1334 0, 0, 1, rxr_idx);
1335
1336 if (err)
1337 goto err_out;
1338
1339 /* update counts and index */
1340 rxr_remaining--;
1341 rxr_idx++;
1342 }
1343 }
1344
1345 for (; v_idx < q_vectors; v_idx++) {
1346 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1347 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1348
1349 err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1350 tqpv, txr_idx, rqpv, rxr_idx);
1351
1352 if (err)
1353 goto err_out;
1354
1355 /* update counts and index */
1356 rxr_remaining -= rqpv;
1357 txr_remaining -= tqpv;
1358 rxr_idx++;
1359 txr_idx++;
1360 }
1361
1362 return 0;
1363
1364 err_out:
1365 adapter->num_tx_queues = 0;
1366 adapter->num_rx_queues = 0;
1367 adapter->num_q_vectors = 0;
1368
1369 while (v_idx--)
1370 igb_free_q_vector(adapter, v_idx);
1371
1372 return -ENOMEM;
1373 }
1374
1375 /**
1376 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1377 * @adapter: board private structure to initialize
1378 * @msix: boolean value of MSIX capability
1379 *
1380 * This function initializes the interrupts and allocates all of the queues.
1381 **/
igb_init_interrupt_scheme(struct igb_adapter * adapter,bool msix)1382 static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix)
1383 {
1384 struct pci_dev *pdev = adapter->pdev;
1385 int err;
1386
1387 igb_set_interrupt_capability(adapter, msix);
1388
1389 err = igb_alloc_q_vectors(adapter);
1390 if (err) {
1391 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1392 goto err_alloc_q_vectors;
1393 }
1394
1395 igb_cache_ring_register(adapter);
1396
1397 return 0;
1398
1399 err_alloc_q_vectors:
1400 igb_reset_interrupt_capability(adapter);
1401 return err;
1402 }
1403
1404 /**
1405 * igb_request_irq - initialize interrupts
1406 * @adapter: board private structure to initialize
1407 *
1408 * Attempts to configure interrupts using the best available
1409 * capabilities of the hardware and kernel.
1410 **/
igb_request_irq(struct igb_adapter * adapter)1411 static int igb_request_irq(struct igb_adapter *adapter)
1412 {
1413 struct net_device *netdev = adapter->netdev;
1414 struct pci_dev *pdev = adapter->pdev;
1415 int err = 0;
1416
1417 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1418 err = igb_request_msix(adapter);
1419 if (!err)
1420 goto request_done;
1421 /* fall back to MSI */
1422 igb_free_all_tx_resources(adapter);
1423 igb_free_all_rx_resources(adapter);
1424
1425 igb_clear_interrupt_scheme(adapter);
1426 err = igb_init_interrupt_scheme(adapter, false);
1427 if (err)
1428 goto request_done;
1429
1430 igb_setup_all_tx_resources(adapter);
1431 igb_setup_all_rx_resources(adapter);
1432 igb_configure(adapter);
1433 }
1434
1435 igb_assign_vector(adapter->q_vector[0], 0);
1436
1437 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1438 err = request_irq(pdev->irq, igb_intr_msi, 0,
1439 netdev->name, adapter);
1440 if (!err)
1441 goto request_done;
1442
1443 /* fall back to legacy interrupts */
1444 igb_reset_interrupt_capability(adapter);
1445 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1446 }
1447
1448 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1449 netdev->name, adapter);
1450
1451 if (err)
1452 dev_err(&pdev->dev, "Error %d getting interrupt\n",
1453 err);
1454
1455 request_done:
1456 return err;
1457 }
1458
igb_free_irq(struct igb_adapter * adapter)1459 static void igb_free_irq(struct igb_adapter *adapter)
1460 {
1461 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1462 int vector = 0, i;
1463
1464 free_irq(adapter->msix_entries[vector++].vector, adapter);
1465
1466 for (i = 0; i < adapter->num_q_vectors; i++)
1467 free_irq(adapter->msix_entries[vector++].vector,
1468 adapter->q_vector[i]);
1469 } else {
1470 free_irq(adapter->pdev->irq, adapter);
1471 }
1472 }
1473
1474 /**
1475 * igb_irq_disable - Mask off interrupt generation on the NIC
1476 * @adapter: board private structure
1477 **/
igb_irq_disable(struct igb_adapter * adapter)1478 static void igb_irq_disable(struct igb_adapter *adapter)
1479 {
1480 struct e1000_hw *hw = &adapter->hw;
1481
1482 /* we need to be careful when disabling interrupts. The VFs are also
1483 * mapped into these registers and so clearing the bits can cause
1484 * issues on the VF drivers so we only need to clear what we set
1485 */
1486 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1487 u32 regval = rd32(E1000_EIAM);
1488
1489 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1490 wr32(E1000_EIMC, adapter->eims_enable_mask);
1491 regval = rd32(E1000_EIAC);
1492 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1493 }
1494
1495 wr32(E1000_IAM, 0);
1496 wr32(E1000_IMC, ~0);
1497 wrfl();
1498 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1499 int i;
1500
1501 for (i = 0; i < adapter->num_q_vectors; i++)
1502 synchronize_irq(adapter->msix_entries[i].vector);
1503 } else {
1504 synchronize_irq(adapter->pdev->irq);
1505 }
1506 }
1507
1508 /**
1509 * igb_irq_enable - Enable default interrupt generation settings
1510 * @adapter: board private structure
1511 **/
igb_irq_enable(struct igb_adapter * adapter)1512 static void igb_irq_enable(struct igb_adapter *adapter)
1513 {
1514 struct e1000_hw *hw = &adapter->hw;
1515
1516 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1517 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1518 u32 regval = rd32(E1000_EIAC);
1519
1520 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1521 regval = rd32(E1000_EIAM);
1522 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1523 wr32(E1000_EIMS, adapter->eims_enable_mask);
1524 if (adapter->vfs_allocated_count) {
1525 wr32(E1000_MBVFIMR, 0xFF);
1526 ims |= E1000_IMS_VMMB;
1527 }
1528 wr32(E1000_IMS, ims);
1529 } else {
1530 wr32(E1000_IMS, IMS_ENABLE_MASK |
1531 E1000_IMS_DRSTA);
1532 wr32(E1000_IAM, IMS_ENABLE_MASK |
1533 E1000_IMS_DRSTA);
1534 }
1535 }
1536
igb_update_mng_vlan(struct igb_adapter * adapter)1537 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1538 {
1539 struct e1000_hw *hw = &adapter->hw;
1540 u16 pf_id = adapter->vfs_allocated_count;
1541 u16 vid = adapter->hw.mng_cookie.vlan_id;
1542 u16 old_vid = adapter->mng_vlan_id;
1543
1544 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1545 /* add VID to filter table */
1546 igb_vfta_set(hw, vid, pf_id, true, true);
1547 adapter->mng_vlan_id = vid;
1548 } else {
1549 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1550 }
1551
1552 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1553 (vid != old_vid) &&
1554 !test_bit(old_vid, adapter->active_vlans)) {
1555 /* remove VID from filter table */
1556 igb_vfta_set(hw, vid, pf_id, false, true);
1557 }
1558 }
1559
1560 /**
1561 * igb_release_hw_control - release control of the h/w to f/w
1562 * @adapter: address of board private structure
1563 *
1564 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1565 * For ASF and Pass Through versions of f/w this means that the
1566 * driver is no longer loaded.
1567 **/
igb_release_hw_control(struct igb_adapter * adapter)1568 static void igb_release_hw_control(struct igb_adapter *adapter)
1569 {
1570 struct e1000_hw *hw = &adapter->hw;
1571 u32 ctrl_ext;
1572
1573 /* Let firmware take over control of h/w */
1574 ctrl_ext = rd32(E1000_CTRL_EXT);
1575 wr32(E1000_CTRL_EXT,
1576 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1577 }
1578
1579 /**
1580 * igb_get_hw_control - get control of the h/w from f/w
1581 * @adapter: address of board private structure
1582 *
1583 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1584 * For ASF and Pass Through versions of f/w this means that
1585 * the driver is loaded.
1586 **/
igb_get_hw_control(struct igb_adapter * adapter)1587 static void igb_get_hw_control(struct igb_adapter *adapter)
1588 {
1589 struct e1000_hw *hw = &adapter->hw;
1590 u32 ctrl_ext;
1591
1592 /* Let firmware know the driver has taken over */
1593 ctrl_ext = rd32(E1000_CTRL_EXT);
1594 wr32(E1000_CTRL_EXT,
1595 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1596 }
1597
enable_fqtss(struct igb_adapter * adapter,bool enable)1598 static void enable_fqtss(struct igb_adapter *adapter, bool enable)
1599 {
1600 struct net_device *netdev = adapter->netdev;
1601 struct e1000_hw *hw = &adapter->hw;
1602
1603 WARN_ON(hw->mac.type != e1000_i210);
1604
1605 if (enable)
1606 adapter->flags |= IGB_FLAG_FQTSS;
1607 else
1608 adapter->flags &= ~IGB_FLAG_FQTSS;
1609
1610 if (netif_running(netdev))
1611 schedule_work(&adapter->reset_task);
1612 }
1613
is_fqtss_enabled(struct igb_adapter * adapter)1614 static bool is_fqtss_enabled(struct igb_adapter *adapter)
1615 {
1616 return (adapter->flags & IGB_FLAG_FQTSS) ? true : false;
1617 }
1618
set_tx_desc_fetch_prio(struct e1000_hw * hw,int queue,enum tx_queue_prio prio)1619 static void set_tx_desc_fetch_prio(struct e1000_hw *hw, int queue,
1620 enum tx_queue_prio prio)
1621 {
1622 u32 val;
1623
1624 WARN_ON(hw->mac.type != e1000_i210);
1625 WARN_ON(queue < 0 || queue > 4);
1626
1627 val = rd32(E1000_I210_TXDCTL(queue));
1628
1629 if (prio == TX_QUEUE_PRIO_HIGH)
1630 val |= E1000_TXDCTL_PRIORITY;
1631 else
1632 val &= ~E1000_TXDCTL_PRIORITY;
1633
1634 wr32(E1000_I210_TXDCTL(queue), val);
1635 }
1636
set_queue_mode(struct e1000_hw * hw,int queue,enum queue_mode mode)1637 static void set_queue_mode(struct e1000_hw *hw, int queue, enum queue_mode mode)
1638 {
1639 u32 val;
1640
1641 WARN_ON(hw->mac.type != e1000_i210);
1642 WARN_ON(queue < 0 || queue > 1);
1643
1644 val = rd32(E1000_I210_TQAVCC(queue));
1645
1646 if (mode == QUEUE_MODE_STREAM_RESERVATION)
1647 val |= E1000_TQAVCC_QUEUEMODE;
1648 else
1649 val &= ~E1000_TQAVCC_QUEUEMODE;
1650
1651 wr32(E1000_I210_TQAVCC(queue), val);
1652 }
1653
is_any_cbs_enabled(struct igb_adapter * adapter)1654 static bool is_any_cbs_enabled(struct igb_adapter *adapter)
1655 {
1656 int i;
1657
1658 for (i = 0; i < adapter->num_tx_queues; i++) {
1659 if (adapter->tx_ring[i]->cbs_enable)
1660 return true;
1661 }
1662
1663 return false;
1664 }
1665
is_any_txtime_enabled(struct igb_adapter * adapter)1666 static bool is_any_txtime_enabled(struct igb_adapter *adapter)
1667 {
1668 int i;
1669
1670 for (i = 0; i < adapter->num_tx_queues; i++) {
1671 if (adapter->tx_ring[i]->launchtime_enable)
1672 return true;
1673 }
1674
1675 return false;
1676 }
1677
1678 /**
1679 * igb_config_tx_modes - Configure "Qav Tx mode" features on igb
1680 * @adapter: pointer to adapter struct
1681 * @queue: queue number
1682 *
1683 * Configure CBS and Launchtime for a given hardware queue.
1684 * Parameters are retrieved from the correct Tx ring, so
1685 * igb_save_cbs_params() and igb_save_txtime_params() should be used
1686 * for setting those correctly prior to this function being called.
1687 **/
igb_config_tx_modes(struct igb_adapter * adapter,int queue)1688 static void igb_config_tx_modes(struct igb_adapter *adapter, int queue)
1689 {
1690 struct net_device *netdev = adapter->netdev;
1691 struct e1000_hw *hw = &adapter->hw;
1692 struct igb_ring *ring;
1693 u32 tqavcc, tqavctrl;
1694 u16 value;
1695
1696 WARN_ON(hw->mac.type != e1000_i210);
1697 WARN_ON(queue < 0 || queue > 1);
1698 ring = adapter->tx_ring[queue];
1699
1700 /* If any of the Qav features is enabled, configure queues as SR and
1701 * with HIGH PRIO. If none is, then configure them with LOW PRIO and
1702 * as SP.
1703 */
1704 if (ring->cbs_enable || ring->launchtime_enable) {
1705 set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_HIGH);
1706 set_queue_mode(hw, queue, QUEUE_MODE_STREAM_RESERVATION);
1707 } else {
1708 set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_LOW);
1709 set_queue_mode(hw, queue, QUEUE_MODE_STRICT_PRIORITY);
1710 }
1711
1712 /* If CBS is enabled, set DataTranARB and config its parameters. */
1713 if (ring->cbs_enable || queue == 0) {
1714 /* i210 does not allow the queue 0 to be in the Strict
1715 * Priority mode while the Qav mode is enabled, so,
1716 * instead of disabling strict priority mode, we give
1717 * queue 0 the maximum of credits possible.
1718 *
1719 * See section 8.12.19 of the i210 datasheet, "Note:
1720 * Queue0 QueueMode must be set to 1b when
1721 * TransmitMode is set to Qav."
1722 */
1723 if (queue == 0 && !ring->cbs_enable) {
1724 /* max "linkspeed" idleslope in kbps */
1725 ring->idleslope = 1000000;
1726 ring->hicredit = ETH_FRAME_LEN;
1727 }
1728
1729 /* Always set data transfer arbitration to credit-based
1730 * shaper algorithm on TQAVCTRL if CBS is enabled for any of
1731 * the queues.
1732 */
1733 tqavctrl = rd32(E1000_I210_TQAVCTRL);
1734 tqavctrl |= E1000_TQAVCTRL_DATATRANARB;
1735 wr32(E1000_I210_TQAVCTRL, tqavctrl);
1736
1737 /* According to i210 datasheet section 7.2.7.7, we should set
1738 * the 'idleSlope' field from TQAVCC register following the
1739 * equation:
1740 *
1741 * For 100 Mbps link speed:
1742 *
1743 * value = BW * 0x7735 * 0.2 (E1)
1744 *
1745 * For 1000Mbps link speed:
1746 *
1747 * value = BW * 0x7735 * 2 (E2)
1748 *
1749 * E1 and E2 can be merged into one equation as shown below.
1750 * Note that 'link-speed' is in Mbps.
1751 *
1752 * value = BW * 0x7735 * 2 * link-speed
1753 * -------------- (E3)
1754 * 1000
1755 *
1756 * 'BW' is the percentage bandwidth out of full link speed
1757 * which can be found with the following equation. Note that
1758 * idleSlope here is the parameter from this function which
1759 * is in kbps.
1760 *
1761 * BW = idleSlope
1762 * ----------------- (E4)
1763 * link-speed * 1000
1764 *
1765 * That said, we can come up with a generic equation to
1766 * calculate the value we should set it TQAVCC register by
1767 * replacing 'BW' in E3 by E4. The resulting equation is:
1768 *
1769 * value = idleSlope * 0x7735 * 2 * link-speed
1770 * ----------------- -------------- (E5)
1771 * link-speed * 1000 1000
1772 *
1773 * 'link-speed' is present in both sides of the fraction so
1774 * it is canceled out. The final equation is the following:
1775 *
1776 * value = idleSlope * 61034
1777 * ----------------- (E6)
1778 * 1000000
1779 *
1780 * NOTE: For i210, given the above, we can see that idleslope
1781 * is represented in 16.38431 kbps units by the value at
1782 * the TQAVCC register (1Gbps / 61034), which reduces
1783 * the granularity for idleslope increments.
1784 * For instance, if you want to configure a 2576kbps
1785 * idleslope, the value to be written on the register
1786 * would have to be 157.23. If rounded down, you end
1787 * up with less bandwidth available than originally
1788 * required (~2572 kbps). If rounded up, you end up
1789 * with a higher bandwidth (~2589 kbps). Below the
1790 * approach we take is to always round up the
1791 * calculated value, so the resulting bandwidth might
1792 * be slightly higher for some configurations.
1793 */
1794 value = DIV_ROUND_UP_ULL(ring->idleslope * 61034ULL, 1000000);
1795
1796 tqavcc = rd32(E1000_I210_TQAVCC(queue));
1797 tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK;
1798 tqavcc |= value;
1799 wr32(E1000_I210_TQAVCC(queue), tqavcc);
1800
1801 wr32(E1000_I210_TQAVHC(queue),
1802 0x80000000 + ring->hicredit * 0x7735);
1803 } else {
1804
1805 /* Set idleSlope to zero. */
1806 tqavcc = rd32(E1000_I210_TQAVCC(queue));
1807 tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK;
1808 wr32(E1000_I210_TQAVCC(queue), tqavcc);
1809
1810 /* Set hiCredit to zero. */
1811 wr32(E1000_I210_TQAVHC(queue), 0);
1812
1813 /* If CBS is not enabled for any queues anymore, then return to
1814 * the default state of Data Transmission Arbitration on
1815 * TQAVCTRL.
1816 */
1817 if (!is_any_cbs_enabled(adapter)) {
1818 tqavctrl = rd32(E1000_I210_TQAVCTRL);
1819 tqavctrl &= ~E1000_TQAVCTRL_DATATRANARB;
1820 wr32(E1000_I210_TQAVCTRL, tqavctrl);
1821 }
1822 }
1823
1824 /* If LaunchTime is enabled, set DataTranTIM. */
1825 if (ring->launchtime_enable) {
1826 /* Always set DataTranTIM on TQAVCTRL if LaunchTime is enabled
1827 * for any of the SR queues, and configure fetchtime delta.
1828 * XXX NOTE:
1829 * - LaunchTime will be enabled for all SR queues.
1830 * - A fixed offset can be added relative to the launch
1831 * time of all packets if configured at reg LAUNCH_OS0.
1832 * We are keeping it as 0 for now (default value).
1833 */
1834 tqavctrl = rd32(E1000_I210_TQAVCTRL);
1835 tqavctrl |= E1000_TQAVCTRL_DATATRANTIM |
1836 E1000_TQAVCTRL_FETCHTIME_DELTA;
1837 wr32(E1000_I210_TQAVCTRL, tqavctrl);
1838 } else {
1839 /* If Launchtime is not enabled for any SR queues anymore,
1840 * then clear DataTranTIM on TQAVCTRL and clear fetchtime delta,
1841 * effectively disabling Launchtime.
1842 */
1843 if (!is_any_txtime_enabled(adapter)) {
1844 tqavctrl = rd32(E1000_I210_TQAVCTRL);
1845 tqavctrl &= ~E1000_TQAVCTRL_DATATRANTIM;
1846 tqavctrl &= ~E1000_TQAVCTRL_FETCHTIME_DELTA;
1847 wr32(E1000_I210_TQAVCTRL, tqavctrl);
1848 }
1849 }
1850
1851 /* XXX: In i210 controller the sendSlope and loCredit parameters from
1852 * CBS are not configurable by software so we don't do any 'controller
1853 * configuration' in respect to these parameters.
1854 */
1855
1856 netdev_dbg(netdev, "Qav Tx mode: cbs %s, launchtime %s, queue %d idleslope %d sendslope %d hiCredit %d locredit %d\n",
1857 ring->cbs_enable ? "enabled" : "disabled",
1858 ring->launchtime_enable ? "enabled" : "disabled",
1859 queue,
1860 ring->idleslope, ring->sendslope,
1861 ring->hicredit, ring->locredit);
1862 }
1863
igb_save_txtime_params(struct igb_adapter * adapter,int queue,bool enable)1864 static int igb_save_txtime_params(struct igb_adapter *adapter, int queue,
1865 bool enable)
1866 {
1867 struct igb_ring *ring;
1868
1869 if (queue < 0 || queue > adapter->num_tx_queues)
1870 return -EINVAL;
1871
1872 ring = adapter->tx_ring[queue];
1873 ring->launchtime_enable = enable;
1874
1875 return 0;
1876 }
1877
igb_save_cbs_params(struct igb_adapter * adapter,int queue,bool enable,int idleslope,int sendslope,int hicredit,int locredit)1878 static int igb_save_cbs_params(struct igb_adapter *adapter, int queue,
1879 bool enable, int idleslope, int sendslope,
1880 int hicredit, int locredit)
1881 {
1882 struct igb_ring *ring;
1883
1884 if (queue < 0 || queue > adapter->num_tx_queues)
1885 return -EINVAL;
1886
1887 ring = adapter->tx_ring[queue];
1888
1889 ring->cbs_enable = enable;
1890 ring->idleslope = idleslope;
1891 ring->sendslope = sendslope;
1892 ring->hicredit = hicredit;
1893 ring->locredit = locredit;
1894
1895 return 0;
1896 }
1897
1898 /**
1899 * igb_setup_tx_mode - Switch to/from Qav Tx mode when applicable
1900 * @adapter: pointer to adapter struct
1901 *
1902 * Configure TQAVCTRL register switching the controller's Tx mode
1903 * if FQTSS mode is enabled or disabled. Additionally, will issue
1904 * a call to igb_config_tx_modes() per queue so any previously saved
1905 * Tx parameters are applied.
1906 **/
igb_setup_tx_mode(struct igb_adapter * adapter)1907 static void igb_setup_tx_mode(struct igb_adapter *adapter)
1908 {
1909 struct net_device *netdev = adapter->netdev;
1910 struct e1000_hw *hw = &adapter->hw;
1911 u32 val;
1912
1913 /* Only i210 controller supports changing the transmission mode. */
1914 if (hw->mac.type != e1000_i210)
1915 return;
1916
1917 if (is_fqtss_enabled(adapter)) {
1918 int i, max_queue;
1919
1920 /* Configure TQAVCTRL register: set transmit mode to 'Qav',
1921 * set data fetch arbitration to 'round robin', set SP_WAIT_SR
1922 * so SP queues wait for SR ones.
1923 */
1924 val = rd32(E1000_I210_TQAVCTRL);
1925 val |= E1000_TQAVCTRL_XMIT_MODE | E1000_TQAVCTRL_SP_WAIT_SR;
1926 val &= ~E1000_TQAVCTRL_DATAFETCHARB;
1927 wr32(E1000_I210_TQAVCTRL, val);
1928
1929 /* Configure Tx and Rx packet buffers sizes as described in
1930 * i210 datasheet section 7.2.7.7.
1931 */
1932 val = rd32(E1000_TXPBS);
1933 val &= ~I210_TXPBSIZE_MASK;
1934 val |= I210_TXPBSIZE_PB0_6KB | I210_TXPBSIZE_PB1_6KB |
1935 I210_TXPBSIZE_PB2_6KB | I210_TXPBSIZE_PB3_6KB;
1936 wr32(E1000_TXPBS, val);
1937
1938 val = rd32(E1000_RXPBS);
1939 val &= ~I210_RXPBSIZE_MASK;
1940 val |= I210_RXPBSIZE_PB_30KB;
1941 wr32(E1000_RXPBS, val);
1942
1943 /* Section 8.12.9 states that MAX_TPKT_SIZE from DTXMXPKTSZ
1944 * register should not exceed the buffer size programmed in
1945 * TXPBS. The smallest buffer size programmed in TXPBS is 4kB
1946 * so according to the datasheet we should set MAX_TPKT_SIZE to
1947 * 4kB / 64.
1948 *
1949 * However, when we do so, no frame from queue 2 and 3 are
1950 * transmitted. It seems the MAX_TPKT_SIZE should not be great
1951 * or _equal_ to the buffer size programmed in TXPBS. For this
1952 * reason, we set MAX_ TPKT_SIZE to (4kB - 1) / 64.
1953 */
1954 val = (4096 - 1) / 64;
1955 wr32(E1000_I210_DTXMXPKTSZ, val);
1956
1957 /* Since FQTSS mode is enabled, apply any CBS configuration
1958 * previously set. If no previous CBS configuration has been
1959 * done, then the initial configuration is applied, which means
1960 * CBS is disabled.
1961 */
1962 max_queue = (adapter->num_tx_queues < I210_SR_QUEUES_NUM) ?
1963 adapter->num_tx_queues : I210_SR_QUEUES_NUM;
1964
1965 for (i = 0; i < max_queue; i++) {
1966 igb_config_tx_modes(adapter, i);
1967 }
1968 } else {
1969 wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
1970 wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
1971 wr32(E1000_I210_DTXMXPKTSZ, I210_DTXMXPKTSZ_DEFAULT);
1972
1973 val = rd32(E1000_I210_TQAVCTRL);
1974 /* According to Section 8.12.21, the other flags we've set when
1975 * enabling FQTSS are not relevant when disabling FQTSS so we
1976 * don't set they here.
1977 */
1978 val &= ~E1000_TQAVCTRL_XMIT_MODE;
1979 wr32(E1000_I210_TQAVCTRL, val);
1980 }
1981
1982 netdev_dbg(netdev, "FQTSS %s\n", (is_fqtss_enabled(adapter)) ?
1983 "enabled" : "disabled");
1984 }
1985
1986 /**
1987 * igb_configure - configure the hardware for RX and TX
1988 * @adapter: private board structure
1989 **/
igb_configure(struct igb_adapter * adapter)1990 static void igb_configure(struct igb_adapter *adapter)
1991 {
1992 struct net_device *netdev = adapter->netdev;
1993 int i;
1994
1995 igb_get_hw_control(adapter);
1996 igb_set_rx_mode(netdev);
1997 igb_setup_tx_mode(adapter);
1998
1999 igb_restore_vlan(adapter);
2000
2001 igb_setup_tctl(adapter);
2002 igb_setup_mrqc(adapter);
2003 igb_setup_rctl(adapter);
2004
2005 igb_nfc_filter_restore(adapter);
2006 igb_configure_tx(adapter);
2007 igb_configure_rx(adapter);
2008
2009 igb_rx_fifo_flush_82575(&adapter->hw);
2010
2011 /* call igb_desc_unused which always leaves
2012 * at least 1 descriptor unused to make sure
2013 * next_to_use != next_to_clean
2014 */
2015 for (i = 0; i < adapter->num_rx_queues; i++) {
2016 struct igb_ring *ring = adapter->rx_ring[i];
2017 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
2018 }
2019 }
2020
2021 /**
2022 * igb_power_up_link - Power up the phy/serdes link
2023 * @adapter: address of board private structure
2024 **/
igb_power_up_link(struct igb_adapter * adapter)2025 void igb_power_up_link(struct igb_adapter *adapter)
2026 {
2027 igb_reset_phy(&adapter->hw);
2028
2029 if (adapter->hw.phy.media_type == e1000_media_type_copper)
2030 igb_power_up_phy_copper(&adapter->hw);
2031 else
2032 igb_power_up_serdes_link_82575(&adapter->hw);
2033
2034 igb_setup_link(&adapter->hw);
2035 }
2036
2037 /**
2038 * igb_power_down_link - Power down the phy/serdes link
2039 * @adapter: address of board private structure
2040 */
igb_power_down_link(struct igb_adapter * adapter)2041 static void igb_power_down_link(struct igb_adapter *adapter)
2042 {
2043 if (adapter->hw.phy.media_type == e1000_media_type_copper)
2044 igb_power_down_phy_copper_82575(&adapter->hw);
2045 else
2046 igb_shutdown_serdes_link_82575(&adapter->hw);
2047 }
2048
2049 /**
2050 * igb_check_swap_media - Detect and switch function for Media Auto Sense
2051 * @adapter: address of the board private structure
2052 **/
igb_check_swap_media(struct igb_adapter * adapter)2053 static void igb_check_swap_media(struct igb_adapter *adapter)
2054 {
2055 struct e1000_hw *hw = &adapter->hw;
2056 u32 ctrl_ext, connsw;
2057 bool swap_now = false;
2058
2059 ctrl_ext = rd32(E1000_CTRL_EXT);
2060 connsw = rd32(E1000_CONNSW);
2061
2062 /* need to live swap if current media is copper and we have fiber/serdes
2063 * to go to.
2064 */
2065
2066 if ((hw->phy.media_type == e1000_media_type_copper) &&
2067 (!(connsw & E1000_CONNSW_AUTOSENSE_EN))) {
2068 swap_now = true;
2069 } else if ((hw->phy.media_type != e1000_media_type_copper) &&
2070 !(connsw & E1000_CONNSW_SERDESD)) {
2071 /* copper signal takes time to appear */
2072 if (adapter->copper_tries < 4) {
2073 adapter->copper_tries++;
2074 connsw |= E1000_CONNSW_AUTOSENSE_CONF;
2075 wr32(E1000_CONNSW, connsw);
2076 return;
2077 } else {
2078 adapter->copper_tries = 0;
2079 if ((connsw & E1000_CONNSW_PHYSD) &&
2080 (!(connsw & E1000_CONNSW_PHY_PDN))) {
2081 swap_now = true;
2082 connsw &= ~E1000_CONNSW_AUTOSENSE_CONF;
2083 wr32(E1000_CONNSW, connsw);
2084 }
2085 }
2086 }
2087
2088 if (!swap_now)
2089 return;
2090
2091 switch (hw->phy.media_type) {
2092 case e1000_media_type_copper:
2093 netdev_info(adapter->netdev,
2094 "MAS: changing media to fiber/serdes\n");
2095 ctrl_ext |=
2096 E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
2097 adapter->flags |= IGB_FLAG_MEDIA_RESET;
2098 adapter->copper_tries = 0;
2099 break;
2100 case e1000_media_type_internal_serdes:
2101 case e1000_media_type_fiber:
2102 netdev_info(adapter->netdev,
2103 "MAS: changing media to copper\n");
2104 ctrl_ext &=
2105 ~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
2106 adapter->flags |= IGB_FLAG_MEDIA_RESET;
2107 break;
2108 default:
2109 /* shouldn't get here during regular operation */
2110 netdev_err(adapter->netdev,
2111 "AMS: Invalid media type found, returning\n");
2112 break;
2113 }
2114 wr32(E1000_CTRL_EXT, ctrl_ext);
2115 }
2116
2117 /**
2118 * igb_up - Open the interface and prepare it to handle traffic
2119 * @adapter: board private structure
2120 **/
igb_up(struct igb_adapter * adapter)2121 int igb_up(struct igb_adapter *adapter)
2122 {
2123 struct e1000_hw *hw = &adapter->hw;
2124 int i;
2125
2126 /* hardware has been reset, we need to reload some things */
2127 igb_configure(adapter);
2128
2129 clear_bit(__IGB_DOWN, &adapter->state);
2130
2131 for (i = 0; i < adapter->num_q_vectors; i++)
2132 napi_enable(&(adapter->q_vector[i]->napi));
2133
2134 if (adapter->flags & IGB_FLAG_HAS_MSIX)
2135 igb_configure_msix(adapter);
2136 else
2137 igb_assign_vector(adapter->q_vector[0], 0);
2138
2139 /* Clear any pending interrupts. */
2140 rd32(E1000_TSICR);
2141 rd32(E1000_ICR);
2142 igb_irq_enable(adapter);
2143
2144 /* notify VFs that reset has been completed */
2145 if (adapter->vfs_allocated_count) {
2146 u32 reg_data = rd32(E1000_CTRL_EXT);
2147
2148 reg_data |= E1000_CTRL_EXT_PFRSTD;
2149 wr32(E1000_CTRL_EXT, reg_data);
2150 }
2151
2152 netif_tx_start_all_queues(adapter->netdev);
2153
2154 /* start the watchdog. */
2155 hw->mac.get_link_status = 1;
2156 schedule_work(&adapter->watchdog_task);
2157
2158 if ((adapter->flags & IGB_FLAG_EEE) &&
2159 (!hw->dev_spec._82575.eee_disable))
2160 adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T;
2161
2162 return 0;
2163 }
2164
igb_down(struct igb_adapter * adapter)2165 void igb_down(struct igb_adapter *adapter)
2166 {
2167 struct net_device *netdev = adapter->netdev;
2168 struct e1000_hw *hw = &adapter->hw;
2169 u32 tctl, rctl;
2170 int i;
2171
2172 /* signal that we're down so the interrupt handler does not
2173 * reschedule our watchdog timer
2174 */
2175 set_bit(__IGB_DOWN, &adapter->state);
2176
2177 /* disable receives in the hardware */
2178 rctl = rd32(E1000_RCTL);
2179 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
2180 /* flush and sleep below */
2181
2182 igb_nfc_filter_exit(adapter);
2183
2184 netif_carrier_off(netdev);
2185 netif_tx_stop_all_queues(netdev);
2186
2187 /* disable transmits in the hardware */
2188 tctl = rd32(E1000_TCTL);
2189 tctl &= ~E1000_TCTL_EN;
2190 wr32(E1000_TCTL, tctl);
2191 /* flush both disables and wait for them to finish */
2192 wrfl();
2193 usleep_range(10000, 11000);
2194
2195 igb_irq_disable(adapter);
2196
2197 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
2198
2199 for (i = 0; i < adapter->num_q_vectors; i++) {
2200 if (adapter->q_vector[i]) {
2201 napi_synchronize(&adapter->q_vector[i]->napi);
2202 napi_disable(&adapter->q_vector[i]->napi);
2203 }
2204 }
2205
2206 del_timer_sync(&adapter->watchdog_timer);
2207 del_timer_sync(&adapter->phy_info_timer);
2208
2209 /* record the stats before reset*/
2210 spin_lock(&adapter->stats64_lock);
2211 igb_update_stats(adapter);
2212 spin_unlock(&adapter->stats64_lock);
2213
2214 adapter->link_speed = 0;
2215 adapter->link_duplex = 0;
2216
2217 if (!pci_channel_offline(adapter->pdev))
2218 igb_reset(adapter);
2219
2220 /* clear VLAN promisc flag so VFTA will be updated if necessary */
2221 adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
2222
2223 igb_clean_all_tx_rings(adapter);
2224 igb_clean_all_rx_rings(adapter);
2225 #ifdef CONFIG_IGB_DCA
2226
2227 /* since we reset the hardware DCA settings were cleared */
2228 igb_setup_dca(adapter);
2229 #endif
2230 }
2231
igb_reinit_locked(struct igb_adapter * adapter)2232 void igb_reinit_locked(struct igb_adapter *adapter)
2233 {
2234 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
2235 usleep_range(1000, 2000);
2236 igb_down(adapter);
2237 igb_up(adapter);
2238 clear_bit(__IGB_RESETTING, &adapter->state);
2239 }
2240
2241 /** igb_enable_mas - Media Autosense re-enable after swap
2242 *
2243 * @adapter: adapter struct
2244 **/
igb_enable_mas(struct igb_adapter * adapter)2245 static void igb_enable_mas(struct igb_adapter *adapter)
2246 {
2247 struct e1000_hw *hw = &adapter->hw;
2248 u32 connsw = rd32(E1000_CONNSW);
2249
2250 /* configure for SerDes media detect */
2251 if ((hw->phy.media_type == e1000_media_type_copper) &&
2252 (!(connsw & E1000_CONNSW_SERDESD))) {
2253 connsw |= E1000_CONNSW_ENRGSRC;
2254 connsw |= E1000_CONNSW_AUTOSENSE_EN;
2255 wr32(E1000_CONNSW, connsw);
2256 wrfl();
2257 }
2258 }
2259
2260 #ifdef CONFIG_IGB_HWMON
2261 /**
2262 * igb_set_i2c_bb - Init I2C interface
2263 * @hw: pointer to hardware structure
2264 **/
igb_set_i2c_bb(struct e1000_hw * hw)2265 static void igb_set_i2c_bb(struct e1000_hw *hw)
2266 {
2267 u32 ctrl_ext;
2268 s32 i2cctl;
2269
2270 ctrl_ext = rd32(E1000_CTRL_EXT);
2271 ctrl_ext |= E1000_CTRL_I2C_ENA;
2272 wr32(E1000_CTRL_EXT, ctrl_ext);
2273 wrfl();
2274
2275 i2cctl = rd32(E1000_I2CPARAMS);
2276 i2cctl |= E1000_I2CBB_EN
2277 | E1000_I2C_CLK_OE_N
2278 | E1000_I2C_DATA_OE_N;
2279 wr32(E1000_I2CPARAMS, i2cctl);
2280 wrfl();
2281 }
2282 #endif
2283
igb_reset(struct igb_adapter * adapter)2284 void igb_reset(struct igb_adapter *adapter)
2285 {
2286 struct pci_dev *pdev = adapter->pdev;
2287 struct e1000_hw *hw = &adapter->hw;
2288 struct e1000_mac_info *mac = &hw->mac;
2289 struct e1000_fc_info *fc = &hw->fc;
2290 u32 pba, hwm;
2291
2292 /* Repartition Pba for greater than 9k mtu
2293 * To take effect CTRL.RST is required.
2294 */
2295 switch (mac->type) {
2296 case e1000_i350:
2297 case e1000_i354:
2298 case e1000_82580:
2299 pba = rd32(E1000_RXPBS);
2300 pba = igb_rxpbs_adjust_82580(pba);
2301 break;
2302 case e1000_82576:
2303 pba = rd32(E1000_RXPBS);
2304 pba &= E1000_RXPBS_SIZE_MASK_82576;
2305 break;
2306 case e1000_82575:
2307 case e1000_i210:
2308 case e1000_i211:
2309 default:
2310 pba = E1000_PBA_34K;
2311 break;
2312 }
2313
2314 if (mac->type == e1000_82575) {
2315 u32 min_rx_space, min_tx_space, needed_tx_space;
2316
2317 /* write Rx PBA so that hardware can report correct Tx PBA */
2318 wr32(E1000_PBA, pba);
2319
2320 /* To maintain wire speed transmits, the Tx FIFO should be
2321 * large enough to accommodate two full transmit packets,
2322 * rounded up to the next 1KB and expressed in KB. Likewise,
2323 * the Rx FIFO should be large enough to accommodate at least
2324 * one full receive packet and is similarly rounded up and
2325 * expressed in KB.
2326 */
2327 min_rx_space = DIV_ROUND_UP(MAX_JUMBO_FRAME_SIZE, 1024);
2328
2329 /* The Tx FIFO also stores 16 bytes of information about the Tx
2330 * but don't include Ethernet FCS because hardware appends it.
2331 * We only need to round down to the nearest 512 byte block
2332 * count since the value we care about is 2 frames, not 1.
2333 */
2334 min_tx_space = adapter->max_frame_size;
2335 min_tx_space += sizeof(union e1000_adv_tx_desc) - ETH_FCS_LEN;
2336 min_tx_space = DIV_ROUND_UP(min_tx_space, 512);
2337
2338 /* upper 16 bits has Tx packet buffer allocation size in KB */
2339 needed_tx_space = min_tx_space - (rd32(E1000_PBA) >> 16);
2340
2341 /* If current Tx allocation is less than the min Tx FIFO size,
2342 * and the min Tx FIFO size is less than the current Rx FIFO
2343 * allocation, take space away from current Rx allocation.
2344 */
2345 if (needed_tx_space < pba) {
2346 pba -= needed_tx_space;
2347
2348 /* if short on Rx space, Rx wins and must trump Tx
2349 * adjustment
2350 */
2351 if (pba < min_rx_space)
2352 pba = min_rx_space;
2353 }
2354
2355 /* adjust PBA for jumbo frames */
2356 wr32(E1000_PBA, pba);
2357 }
2358
2359 /* flow control settings
2360 * The high water mark must be low enough to fit one full frame
2361 * after transmitting the pause frame. As such we must have enough
2362 * space to allow for us to complete our current transmit and then
2363 * receive the frame that is in progress from the link partner.
2364 * Set it to:
2365 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
2366 */
2367 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
2368
2369 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */
2370 fc->low_water = fc->high_water - 16;
2371 fc->pause_time = 0xFFFF;
2372 fc->send_xon = 1;
2373 fc->current_mode = fc->requested_mode;
2374
2375 /* disable receive for all VFs and wait one second */
2376 if (adapter->vfs_allocated_count) {
2377 int i;
2378
2379 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
2380 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
2381
2382 /* ping all the active vfs to let them know we are going down */
2383 igb_ping_all_vfs(adapter);
2384
2385 /* disable transmits and receives */
2386 wr32(E1000_VFRE, 0);
2387 wr32(E1000_VFTE, 0);
2388 }
2389
2390 /* Allow time for pending master requests to run */
2391 hw->mac.ops.reset_hw(hw);
2392 wr32(E1000_WUC, 0);
2393
2394 if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
2395 /* need to resetup here after media swap */
2396 adapter->ei.get_invariants(hw);
2397 adapter->flags &= ~IGB_FLAG_MEDIA_RESET;
2398 }
2399 if ((mac->type == e1000_82575 || mac->type == e1000_i350) &&
2400 (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
2401 igb_enable_mas(adapter);
2402 }
2403 if (hw->mac.ops.init_hw(hw))
2404 dev_err(&pdev->dev, "Hardware Error\n");
2405
2406 /* RAR registers were cleared during init_hw, clear mac table */
2407 igb_flush_mac_table(adapter);
2408 __dev_uc_unsync(adapter->netdev, NULL);
2409
2410 /* Recover default RAR entry */
2411 igb_set_default_mac_filter(adapter);
2412
2413 /* Flow control settings reset on hardware reset, so guarantee flow
2414 * control is off when forcing speed.
2415 */
2416 if (!hw->mac.autoneg)
2417 igb_force_mac_fc(hw);
2418
2419 igb_init_dmac(adapter, pba);
2420 #ifdef CONFIG_IGB_HWMON
2421 /* Re-initialize the thermal sensor on i350 devices. */
2422 if (!test_bit(__IGB_DOWN, &adapter->state)) {
2423 if (mac->type == e1000_i350 && hw->bus.func == 0) {
2424 /* If present, re-initialize the external thermal sensor
2425 * interface.
2426 */
2427 if (adapter->ets)
2428 igb_set_i2c_bb(hw);
2429 mac->ops.init_thermal_sensor_thresh(hw);
2430 }
2431 }
2432 #endif
2433 /* Re-establish EEE setting */
2434 if (hw->phy.media_type == e1000_media_type_copper) {
2435 switch (mac->type) {
2436 case e1000_i350:
2437 case e1000_i210:
2438 case e1000_i211:
2439 igb_set_eee_i350(hw, true, true);
2440 break;
2441 case e1000_i354:
2442 igb_set_eee_i354(hw, true, true);
2443 break;
2444 default:
2445 break;
2446 }
2447 }
2448 if (!netif_running(adapter->netdev))
2449 igb_power_down_link(adapter);
2450
2451 igb_update_mng_vlan(adapter);
2452
2453 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
2454 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
2455
2456 /* Re-enable PTP, where applicable. */
2457 if (adapter->ptp_flags & IGB_PTP_ENABLED)
2458 igb_ptp_reset(adapter);
2459
2460 igb_get_phy_info(hw);
2461 }
2462
igb_fix_features(struct net_device * netdev,netdev_features_t features)2463 static netdev_features_t igb_fix_features(struct net_device *netdev,
2464 netdev_features_t features)
2465 {
2466 /* Since there is no support for separate Rx/Tx vlan accel
2467 * enable/disable make sure Tx flag is always in same state as Rx.
2468 */
2469 if (features & NETIF_F_HW_VLAN_CTAG_RX)
2470 features |= NETIF_F_HW_VLAN_CTAG_TX;
2471 else
2472 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2473
2474 return features;
2475 }
2476
igb_set_features(struct net_device * netdev,netdev_features_t features)2477 static int igb_set_features(struct net_device *netdev,
2478 netdev_features_t features)
2479 {
2480 netdev_features_t changed = netdev->features ^ features;
2481 struct igb_adapter *adapter = netdev_priv(netdev);
2482
2483 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2484 igb_vlan_mode(netdev, features);
2485
2486 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
2487 return 0;
2488
2489 if (!(features & NETIF_F_NTUPLE)) {
2490 struct hlist_node *node2;
2491 struct igb_nfc_filter *rule;
2492
2493 spin_lock(&adapter->nfc_lock);
2494 hlist_for_each_entry_safe(rule, node2,
2495 &adapter->nfc_filter_list, nfc_node) {
2496 igb_erase_filter(adapter, rule);
2497 hlist_del(&rule->nfc_node);
2498 kfree(rule);
2499 }
2500 spin_unlock(&adapter->nfc_lock);
2501 adapter->nfc_filter_count = 0;
2502 }
2503
2504 netdev->features = features;
2505
2506 if (netif_running(netdev))
2507 igb_reinit_locked(adapter);
2508 else
2509 igb_reset(adapter);
2510
2511 return 1;
2512 }
2513
igb_ndo_fdb_add(struct ndmsg * ndm,struct nlattr * tb[],struct net_device * dev,const unsigned char * addr,u16 vid,u16 flags,struct netlink_ext_ack * extack)2514 static int igb_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
2515 struct net_device *dev,
2516 const unsigned char *addr, u16 vid,
2517 u16 flags,
2518 struct netlink_ext_ack *extack)
2519 {
2520 /* guarantee we can provide a unique filter for the unicast address */
2521 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
2522 struct igb_adapter *adapter = netdev_priv(dev);
2523 int vfn = adapter->vfs_allocated_count;
2524
2525 if (netdev_uc_count(dev) >= igb_available_rars(adapter, vfn))
2526 return -ENOMEM;
2527 }
2528
2529 return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags);
2530 }
2531
2532 #define IGB_MAX_MAC_HDR_LEN 127
2533 #define IGB_MAX_NETWORK_HDR_LEN 511
2534
2535 static netdev_features_t
igb_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)2536 igb_features_check(struct sk_buff *skb, struct net_device *dev,
2537 netdev_features_t features)
2538 {
2539 unsigned int network_hdr_len, mac_hdr_len;
2540
2541 /* Make certain the headers can be described by a context descriptor */
2542 mac_hdr_len = skb_network_header(skb) - skb->data;
2543 if (unlikely(mac_hdr_len > IGB_MAX_MAC_HDR_LEN))
2544 return features & ~(NETIF_F_HW_CSUM |
2545 NETIF_F_SCTP_CRC |
2546 NETIF_F_GSO_UDP_L4 |
2547 NETIF_F_HW_VLAN_CTAG_TX |
2548 NETIF_F_TSO |
2549 NETIF_F_TSO6);
2550
2551 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2552 if (unlikely(network_hdr_len > IGB_MAX_NETWORK_HDR_LEN))
2553 return features & ~(NETIF_F_HW_CSUM |
2554 NETIF_F_SCTP_CRC |
2555 NETIF_F_GSO_UDP_L4 |
2556 NETIF_F_TSO |
2557 NETIF_F_TSO6);
2558
2559 /* We can only support IPV4 TSO in tunnels if we can mangle the
2560 * inner IP ID field, so strip TSO if MANGLEID is not supported.
2561 */
2562 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2563 features &= ~NETIF_F_TSO;
2564
2565 return features;
2566 }
2567
igb_offload_apply(struct igb_adapter * adapter,s32 queue)2568 static void igb_offload_apply(struct igb_adapter *adapter, s32 queue)
2569 {
2570 if (!is_fqtss_enabled(adapter)) {
2571 enable_fqtss(adapter, true);
2572 return;
2573 }
2574
2575 igb_config_tx_modes(adapter, queue);
2576
2577 if (!is_any_cbs_enabled(adapter) && !is_any_txtime_enabled(adapter))
2578 enable_fqtss(adapter, false);
2579 }
2580
igb_offload_cbs(struct igb_adapter * adapter,struct tc_cbs_qopt_offload * qopt)2581 static int igb_offload_cbs(struct igb_adapter *adapter,
2582 struct tc_cbs_qopt_offload *qopt)
2583 {
2584 struct e1000_hw *hw = &adapter->hw;
2585 int err;
2586
2587 /* CBS offloading is only supported by i210 controller. */
2588 if (hw->mac.type != e1000_i210)
2589 return -EOPNOTSUPP;
2590
2591 /* CBS offloading is only supported by queue 0 and queue 1. */
2592 if (qopt->queue < 0 || qopt->queue > 1)
2593 return -EINVAL;
2594
2595 err = igb_save_cbs_params(adapter, qopt->queue, qopt->enable,
2596 qopt->idleslope, qopt->sendslope,
2597 qopt->hicredit, qopt->locredit);
2598 if (err)
2599 return err;
2600
2601 igb_offload_apply(adapter, qopt->queue);
2602
2603 return 0;
2604 }
2605
2606 #define ETHER_TYPE_FULL_MASK ((__force __be16)~0)
2607 #define VLAN_PRIO_FULL_MASK (0x07)
2608
igb_parse_cls_flower(struct igb_adapter * adapter,struct flow_cls_offload * f,int traffic_class,struct igb_nfc_filter * input)2609 static int igb_parse_cls_flower(struct igb_adapter *adapter,
2610 struct flow_cls_offload *f,
2611 int traffic_class,
2612 struct igb_nfc_filter *input)
2613 {
2614 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
2615 struct flow_dissector *dissector = rule->match.dissector;
2616 struct netlink_ext_ack *extack = f->common.extack;
2617
2618 if (dissector->used_keys &
2619 ~(BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
2620 BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
2621 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
2622 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN))) {
2623 NL_SET_ERR_MSG_MOD(extack,
2624 "Unsupported key used, only BASIC, CONTROL, ETH_ADDRS and VLAN are supported");
2625 return -EOPNOTSUPP;
2626 }
2627
2628 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
2629 struct flow_match_eth_addrs match;
2630
2631 flow_rule_match_eth_addrs(rule, &match);
2632 if (!is_zero_ether_addr(match.mask->dst)) {
2633 if (!is_broadcast_ether_addr(match.mask->dst)) {
2634 NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for destination MAC address");
2635 return -EINVAL;
2636 }
2637
2638 input->filter.match_flags |=
2639 IGB_FILTER_FLAG_DST_MAC_ADDR;
2640 ether_addr_copy(input->filter.dst_addr, match.key->dst);
2641 }
2642
2643 if (!is_zero_ether_addr(match.mask->src)) {
2644 if (!is_broadcast_ether_addr(match.mask->src)) {
2645 NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for source MAC address");
2646 return -EINVAL;
2647 }
2648
2649 input->filter.match_flags |=
2650 IGB_FILTER_FLAG_SRC_MAC_ADDR;
2651 ether_addr_copy(input->filter.src_addr, match.key->src);
2652 }
2653 }
2654
2655 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
2656 struct flow_match_basic match;
2657
2658 flow_rule_match_basic(rule, &match);
2659 if (match.mask->n_proto) {
2660 if (match.mask->n_proto != ETHER_TYPE_FULL_MASK) {
2661 NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for EtherType filter");
2662 return -EINVAL;
2663 }
2664
2665 input->filter.match_flags |= IGB_FILTER_FLAG_ETHER_TYPE;
2666 input->filter.etype = match.key->n_proto;
2667 }
2668 }
2669
2670 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
2671 struct flow_match_vlan match;
2672
2673 flow_rule_match_vlan(rule, &match);
2674 if (match.mask->vlan_priority) {
2675 if (match.mask->vlan_priority != VLAN_PRIO_FULL_MASK) {
2676 NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for VLAN priority");
2677 return -EINVAL;
2678 }
2679
2680 input->filter.match_flags |= IGB_FILTER_FLAG_VLAN_TCI;
2681 input->filter.vlan_tci =
2682 (__force __be16)match.key->vlan_priority;
2683 }
2684 }
2685
2686 input->action = traffic_class;
2687 input->cookie = f->cookie;
2688
2689 return 0;
2690 }
2691
igb_configure_clsflower(struct igb_adapter * adapter,struct flow_cls_offload * cls_flower)2692 static int igb_configure_clsflower(struct igb_adapter *adapter,
2693 struct flow_cls_offload *cls_flower)
2694 {
2695 struct netlink_ext_ack *extack = cls_flower->common.extack;
2696 struct igb_nfc_filter *filter, *f;
2697 int err, tc;
2698
2699 tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
2700 if (tc < 0) {
2701 NL_SET_ERR_MSG_MOD(extack, "Invalid traffic class");
2702 return -EINVAL;
2703 }
2704
2705 filter = kzalloc(sizeof(*filter), GFP_KERNEL);
2706 if (!filter)
2707 return -ENOMEM;
2708
2709 err = igb_parse_cls_flower(adapter, cls_flower, tc, filter);
2710 if (err < 0)
2711 goto err_parse;
2712
2713 spin_lock(&adapter->nfc_lock);
2714
2715 hlist_for_each_entry(f, &adapter->nfc_filter_list, nfc_node) {
2716 if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
2717 err = -EEXIST;
2718 NL_SET_ERR_MSG_MOD(extack,
2719 "This filter is already set in ethtool");
2720 goto err_locked;
2721 }
2722 }
2723
2724 hlist_for_each_entry(f, &adapter->cls_flower_list, nfc_node) {
2725 if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
2726 err = -EEXIST;
2727 NL_SET_ERR_MSG_MOD(extack,
2728 "This filter is already set in cls_flower");
2729 goto err_locked;
2730 }
2731 }
2732
2733 err = igb_add_filter(adapter, filter);
2734 if (err < 0) {
2735 NL_SET_ERR_MSG_MOD(extack, "Could not add filter to the adapter");
2736 goto err_locked;
2737 }
2738
2739 hlist_add_head(&filter->nfc_node, &adapter->cls_flower_list);
2740
2741 spin_unlock(&adapter->nfc_lock);
2742
2743 return 0;
2744
2745 err_locked:
2746 spin_unlock(&adapter->nfc_lock);
2747
2748 err_parse:
2749 kfree(filter);
2750
2751 return err;
2752 }
2753
igb_delete_clsflower(struct igb_adapter * adapter,struct flow_cls_offload * cls_flower)2754 static int igb_delete_clsflower(struct igb_adapter *adapter,
2755 struct flow_cls_offload *cls_flower)
2756 {
2757 struct igb_nfc_filter *filter;
2758 int err;
2759
2760 spin_lock(&adapter->nfc_lock);
2761
2762 hlist_for_each_entry(filter, &adapter->cls_flower_list, nfc_node)
2763 if (filter->cookie == cls_flower->cookie)
2764 break;
2765
2766 if (!filter) {
2767 err = -ENOENT;
2768 goto out;
2769 }
2770
2771 err = igb_erase_filter(adapter, filter);
2772 if (err < 0)
2773 goto out;
2774
2775 hlist_del(&filter->nfc_node);
2776 kfree(filter);
2777
2778 out:
2779 spin_unlock(&adapter->nfc_lock);
2780
2781 return err;
2782 }
2783
igb_setup_tc_cls_flower(struct igb_adapter * adapter,struct flow_cls_offload * cls_flower)2784 static int igb_setup_tc_cls_flower(struct igb_adapter *adapter,
2785 struct flow_cls_offload *cls_flower)
2786 {
2787 switch (cls_flower->command) {
2788 case FLOW_CLS_REPLACE:
2789 return igb_configure_clsflower(adapter, cls_flower);
2790 case FLOW_CLS_DESTROY:
2791 return igb_delete_clsflower(adapter, cls_flower);
2792 case FLOW_CLS_STATS:
2793 return -EOPNOTSUPP;
2794 default:
2795 return -EOPNOTSUPP;
2796 }
2797 }
2798
igb_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)2799 static int igb_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
2800 void *cb_priv)
2801 {
2802 struct igb_adapter *adapter = cb_priv;
2803
2804 if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
2805 return -EOPNOTSUPP;
2806
2807 switch (type) {
2808 case TC_SETUP_CLSFLOWER:
2809 return igb_setup_tc_cls_flower(adapter, type_data);
2810
2811 default:
2812 return -EOPNOTSUPP;
2813 }
2814 }
2815
igb_offload_txtime(struct igb_adapter * adapter,struct tc_etf_qopt_offload * qopt)2816 static int igb_offload_txtime(struct igb_adapter *adapter,
2817 struct tc_etf_qopt_offload *qopt)
2818 {
2819 struct e1000_hw *hw = &adapter->hw;
2820 int err;
2821
2822 /* Launchtime offloading is only supported by i210 controller. */
2823 if (hw->mac.type != e1000_i210)
2824 return -EOPNOTSUPP;
2825
2826 /* Launchtime offloading is only supported by queues 0 and 1. */
2827 if (qopt->queue < 0 || qopt->queue > 1)
2828 return -EINVAL;
2829
2830 err = igb_save_txtime_params(adapter, qopt->queue, qopt->enable);
2831 if (err)
2832 return err;
2833
2834 igb_offload_apply(adapter, qopt->queue);
2835
2836 return 0;
2837 }
2838
igb_tc_query_caps(struct igb_adapter * adapter,struct tc_query_caps_base * base)2839 static int igb_tc_query_caps(struct igb_adapter *adapter,
2840 struct tc_query_caps_base *base)
2841 {
2842 switch (base->type) {
2843 case TC_SETUP_QDISC_TAPRIO: {
2844 struct tc_taprio_caps *caps = base->caps;
2845
2846 caps->broken_mqprio = true;
2847
2848 return 0;
2849 }
2850 default:
2851 return -EOPNOTSUPP;
2852 }
2853 }
2854
2855 static LIST_HEAD(igb_block_cb_list);
2856
igb_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)2857 static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type,
2858 void *type_data)
2859 {
2860 struct igb_adapter *adapter = netdev_priv(dev);
2861
2862 switch (type) {
2863 case TC_QUERY_CAPS:
2864 return igb_tc_query_caps(adapter, type_data);
2865 case TC_SETUP_QDISC_CBS:
2866 return igb_offload_cbs(adapter, type_data);
2867 case TC_SETUP_BLOCK:
2868 return flow_block_cb_setup_simple(type_data,
2869 &igb_block_cb_list,
2870 igb_setup_tc_block_cb,
2871 adapter, adapter, true);
2872
2873 case TC_SETUP_QDISC_ETF:
2874 return igb_offload_txtime(adapter, type_data);
2875
2876 default:
2877 return -EOPNOTSUPP;
2878 }
2879 }
2880
igb_xdp_setup(struct net_device * dev,struct netdev_bpf * bpf)2881 static int igb_xdp_setup(struct net_device *dev, struct netdev_bpf *bpf)
2882 {
2883 int i, frame_size = dev->mtu + IGB_ETH_PKT_HDR_PAD;
2884 struct igb_adapter *adapter = netdev_priv(dev);
2885 struct bpf_prog *prog = bpf->prog, *old_prog;
2886 bool running = netif_running(dev);
2887 bool need_reset;
2888
2889 /* verify igb ring attributes are sufficient for XDP */
2890 for (i = 0; i < adapter->num_rx_queues; i++) {
2891 struct igb_ring *ring = adapter->rx_ring[i];
2892
2893 if (frame_size > igb_rx_bufsz(ring)) {
2894 NL_SET_ERR_MSG_MOD(bpf->extack,
2895 "The RX buffer size is too small for the frame size");
2896 netdev_warn(dev, "XDP RX buffer size %d is too small for the frame size %d\n",
2897 igb_rx_bufsz(ring), frame_size);
2898 return -EINVAL;
2899 }
2900 }
2901
2902 old_prog = xchg(&adapter->xdp_prog, prog);
2903 need_reset = (!!prog != !!old_prog);
2904
2905 /* device is up and bpf is added/removed, must setup the RX queues */
2906 if (need_reset && running) {
2907 igb_close(dev);
2908 } else {
2909 for (i = 0; i < adapter->num_rx_queues; i++)
2910 (void)xchg(&adapter->rx_ring[i]->xdp_prog,
2911 adapter->xdp_prog);
2912 }
2913
2914 if (old_prog)
2915 bpf_prog_put(old_prog);
2916
2917 /* bpf is just replaced, RXQ and MTU are already setup */
2918 if (!need_reset) {
2919 return 0;
2920 } else {
2921 if (prog)
2922 xdp_features_set_redirect_target(dev, true);
2923 else
2924 xdp_features_clear_redirect_target(dev);
2925 }
2926
2927 if (running)
2928 igb_open(dev);
2929
2930 return 0;
2931 }
2932
igb_xdp(struct net_device * dev,struct netdev_bpf * xdp)2933 static int igb_xdp(struct net_device *dev, struct netdev_bpf *xdp)
2934 {
2935 switch (xdp->command) {
2936 case XDP_SETUP_PROG:
2937 return igb_xdp_setup(dev, xdp);
2938 default:
2939 return -EINVAL;
2940 }
2941 }
2942
2943 /* This function assumes __netif_tx_lock is held by the caller. */
igb_xdp_ring_update_tail(struct igb_ring * ring)2944 static void igb_xdp_ring_update_tail(struct igb_ring *ring)
2945 {
2946 lockdep_assert_held(&txring_txq(ring)->_xmit_lock);
2947
2948 /* Force memory writes to complete before letting h/w know there
2949 * are new descriptors to fetch.
2950 */
2951 wmb();
2952 writel(ring->next_to_use, ring->tail);
2953 }
2954
igb_xdp_tx_queue_mapping(struct igb_adapter * adapter)2955 static struct igb_ring *igb_xdp_tx_queue_mapping(struct igb_adapter *adapter)
2956 {
2957 unsigned int r_idx = smp_processor_id();
2958
2959 if (r_idx >= adapter->num_tx_queues)
2960 r_idx = r_idx % adapter->num_tx_queues;
2961
2962 return adapter->tx_ring[r_idx];
2963 }
2964
igb_xdp_xmit_back(struct igb_adapter * adapter,struct xdp_buff * xdp)2965 static int igb_xdp_xmit_back(struct igb_adapter *adapter, struct xdp_buff *xdp)
2966 {
2967 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
2968 int cpu = smp_processor_id();
2969 struct igb_ring *tx_ring;
2970 struct netdev_queue *nq;
2971 u32 ret;
2972
2973 if (unlikely(!xdpf))
2974 return IGB_XDP_CONSUMED;
2975
2976 /* During program transitions its possible adapter->xdp_prog is assigned
2977 * but ring has not been configured yet. In this case simply abort xmit.
2978 */
2979 tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL;
2980 if (unlikely(!tx_ring))
2981 return IGB_XDP_CONSUMED;
2982
2983 nq = txring_txq(tx_ring);
2984 __netif_tx_lock(nq, cpu);
2985 /* Avoid transmit queue timeout since we share it with the slow path */
2986 txq_trans_cond_update(nq);
2987 ret = igb_xmit_xdp_ring(adapter, tx_ring, xdpf);
2988 __netif_tx_unlock(nq);
2989
2990 return ret;
2991 }
2992
igb_xdp_xmit(struct net_device * dev,int n,struct xdp_frame ** frames,u32 flags)2993 static int igb_xdp_xmit(struct net_device *dev, int n,
2994 struct xdp_frame **frames, u32 flags)
2995 {
2996 struct igb_adapter *adapter = netdev_priv(dev);
2997 int cpu = smp_processor_id();
2998 struct igb_ring *tx_ring;
2999 struct netdev_queue *nq;
3000 int nxmit = 0;
3001 int i;
3002
3003 if (unlikely(test_bit(__IGB_DOWN, &adapter->state)))
3004 return -ENETDOWN;
3005
3006 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
3007 return -EINVAL;
3008
3009 /* During program transitions its possible adapter->xdp_prog is assigned
3010 * but ring has not been configured yet. In this case simply abort xmit.
3011 */
3012 tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL;
3013 if (unlikely(!tx_ring))
3014 return -ENXIO;
3015
3016 nq = txring_txq(tx_ring);
3017 __netif_tx_lock(nq, cpu);
3018
3019 /* Avoid transmit queue timeout since we share it with the slow path */
3020 txq_trans_cond_update(nq);
3021
3022 for (i = 0; i < n; i++) {
3023 struct xdp_frame *xdpf = frames[i];
3024 int err;
3025
3026 err = igb_xmit_xdp_ring(adapter, tx_ring, xdpf);
3027 if (err != IGB_XDP_TX)
3028 break;
3029 nxmit++;
3030 }
3031
3032 if (unlikely(flags & XDP_XMIT_FLUSH))
3033 igb_xdp_ring_update_tail(tx_ring);
3034
3035 __netif_tx_unlock(nq);
3036
3037 return nxmit;
3038 }
3039
3040 static const struct net_device_ops igb_netdev_ops = {
3041 .ndo_open = igb_open,
3042 .ndo_stop = igb_close,
3043 .ndo_start_xmit = igb_xmit_frame,
3044 .ndo_get_stats64 = igb_get_stats64,
3045 .ndo_set_rx_mode = igb_set_rx_mode,
3046 .ndo_set_mac_address = igb_set_mac,
3047 .ndo_change_mtu = igb_change_mtu,
3048 .ndo_eth_ioctl = igb_ioctl,
3049 .ndo_tx_timeout = igb_tx_timeout,
3050 .ndo_validate_addr = eth_validate_addr,
3051 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
3052 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
3053 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
3054 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
3055 .ndo_set_vf_rate = igb_ndo_set_vf_bw,
3056 .ndo_set_vf_spoofchk = igb_ndo_set_vf_spoofchk,
3057 .ndo_set_vf_trust = igb_ndo_set_vf_trust,
3058 .ndo_get_vf_config = igb_ndo_get_vf_config,
3059 .ndo_fix_features = igb_fix_features,
3060 .ndo_set_features = igb_set_features,
3061 .ndo_fdb_add = igb_ndo_fdb_add,
3062 .ndo_features_check = igb_features_check,
3063 .ndo_setup_tc = igb_setup_tc,
3064 .ndo_bpf = igb_xdp,
3065 .ndo_xdp_xmit = igb_xdp_xmit,
3066 };
3067
3068 /**
3069 * igb_set_fw_version - Configure version string for ethtool
3070 * @adapter: adapter struct
3071 **/
igb_set_fw_version(struct igb_adapter * adapter)3072 void igb_set_fw_version(struct igb_adapter *adapter)
3073 {
3074 struct e1000_hw *hw = &adapter->hw;
3075 struct e1000_fw_version fw;
3076
3077 igb_get_fw_version(hw, &fw);
3078
3079 switch (hw->mac.type) {
3080 case e1000_i210:
3081 case e1000_i211:
3082 if (!(igb_get_flash_presence_i210(hw))) {
3083 snprintf(adapter->fw_version,
3084 sizeof(adapter->fw_version),
3085 "%2d.%2d-%d",
3086 fw.invm_major, fw.invm_minor,
3087 fw.invm_img_type);
3088 break;
3089 }
3090 fallthrough;
3091 default:
3092 /* if option is rom valid, display its version too */
3093 if (fw.or_valid) {
3094 snprintf(adapter->fw_version,
3095 sizeof(adapter->fw_version),
3096 "%d.%d, 0x%08x, %d.%d.%d",
3097 fw.eep_major, fw.eep_minor, fw.etrack_id,
3098 fw.or_major, fw.or_build, fw.or_patch);
3099 /* no option rom */
3100 } else if (fw.etrack_id != 0X0000) {
3101 snprintf(adapter->fw_version,
3102 sizeof(adapter->fw_version),
3103 "%d.%d, 0x%08x",
3104 fw.eep_major, fw.eep_minor, fw.etrack_id);
3105 } else {
3106 snprintf(adapter->fw_version,
3107 sizeof(adapter->fw_version),
3108 "%d.%d.%d",
3109 fw.eep_major, fw.eep_minor, fw.eep_build);
3110 }
3111 break;
3112 }
3113 }
3114
3115 /**
3116 * igb_init_mas - init Media Autosense feature if enabled in the NVM
3117 *
3118 * @adapter: adapter struct
3119 **/
igb_init_mas(struct igb_adapter * adapter)3120 static void igb_init_mas(struct igb_adapter *adapter)
3121 {
3122 struct e1000_hw *hw = &adapter->hw;
3123 u16 eeprom_data;
3124
3125 hw->nvm.ops.read(hw, NVM_COMPAT, 1, &eeprom_data);
3126 switch (hw->bus.func) {
3127 case E1000_FUNC_0:
3128 if (eeprom_data & IGB_MAS_ENABLE_0) {
3129 adapter->flags |= IGB_FLAG_MAS_ENABLE;
3130 netdev_info(adapter->netdev,
3131 "MAS: Enabling Media Autosense for port %d\n",
3132 hw->bus.func);
3133 }
3134 break;
3135 case E1000_FUNC_1:
3136 if (eeprom_data & IGB_MAS_ENABLE_1) {
3137 adapter->flags |= IGB_FLAG_MAS_ENABLE;
3138 netdev_info(adapter->netdev,
3139 "MAS: Enabling Media Autosense for port %d\n",
3140 hw->bus.func);
3141 }
3142 break;
3143 case E1000_FUNC_2:
3144 if (eeprom_data & IGB_MAS_ENABLE_2) {
3145 adapter->flags |= IGB_FLAG_MAS_ENABLE;
3146 netdev_info(adapter->netdev,
3147 "MAS: Enabling Media Autosense for port %d\n",
3148 hw->bus.func);
3149 }
3150 break;
3151 case E1000_FUNC_3:
3152 if (eeprom_data & IGB_MAS_ENABLE_3) {
3153 adapter->flags |= IGB_FLAG_MAS_ENABLE;
3154 netdev_info(adapter->netdev,
3155 "MAS: Enabling Media Autosense for port %d\n",
3156 hw->bus.func);
3157 }
3158 break;
3159 default:
3160 /* Shouldn't get here */
3161 netdev_err(adapter->netdev,
3162 "MAS: Invalid port configuration, returning\n");
3163 break;
3164 }
3165 }
3166
3167 /**
3168 * igb_init_i2c - Init I2C interface
3169 * @adapter: pointer to adapter structure
3170 **/
igb_init_i2c(struct igb_adapter * adapter)3171 static s32 igb_init_i2c(struct igb_adapter *adapter)
3172 {
3173 s32 status = 0;
3174
3175 /* I2C interface supported on i350 devices */
3176 if (adapter->hw.mac.type != e1000_i350)
3177 return 0;
3178
3179 /* Initialize the i2c bus which is controlled by the registers.
3180 * This bus will use the i2c_algo_bit structure that implements
3181 * the protocol through toggling of the 4 bits in the register.
3182 */
3183 adapter->i2c_adap.owner = THIS_MODULE;
3184 adapter->i2c_algo = igb_i2c_algo;
3185 adapter->i2c_algo.data = adapter;
3186 adapter->i2c_adap.algo_data = &adapter->i2c_algo;
3187 adapter->i2c_adap.dev.parent = &adapter->pdev->dev;
3188 strscpy(adapter->i2c_adap.name, "igb BB",
3189 sizeof(adapter->i2c_adap.name));
3190 status = i2c_bit_add_bus(&adapter->i2c_adap);
3191 return status;
3192 }
3193
3194 /**
3195 * igb_probe - Device Initialization Routine
3196 * @pdev: PCI device information struct
3197 * @ent: entry in igb_pci_tbl
3198 *
3199 * Returns 0 on success, negative on failure
3200 *
3201 * igb_probe initializes an adapter identified by a pci_dev structure.
3202 * The OS initialization, configuring of the adapter private structure,
3203 * and a hardware reset occur.
3204 **/
igb_probe(struct pci_dev * pdev,const struct pci_device_id * ent)3205 static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3206 {
3207 struct net_device *netdev;
3208 struct igb_adapter *adapter;
3209 struct e1000_hw *hw;
3210 u16 eeprom_data = 0;
3211 s32 ret_val;
3212 static int global_quad_port_a; /* global quad port a indication */
3213 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
3214 u8 part_str[E1000_PBANUM_LENGTH];
3215 int err;
3216
3217 /* Catch broken hardware that put the wrong VF device ID in
3218 * the PCIe SR-IOV capability.
3219 */
3220 if (pdev->is_virtfn) {
3221 WARN(1, KERN_ERR "%s (%x:%x) should not be a VF!\n",
3222 pci_name(pdev), pdev->vendor, pdev->device);
3223 return -EINVAL;
3224 }
3225
3226 err = pci_enable_device_mem(pdev);
3227 if (err)
3228 return err;
3229
3230 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3231 if (err) {
3232 dev_err(&pdev->dev,
3233 "No usable DMA configuration, aborting\n");
3234 goto err_dma;
3235 }
3236
3237 err = pci_request_mem_regions(pdev, igb_driver_name);
3238 if (err)
3239 goto err_pci_reg;
3240
3241 pci_set_master(pdev);
3242 pci_save_state(pdev);
3243
3244 err = -ENOMEM;
3245 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
3246 IGB_MAX_TX_QUEUES);
3247 if (!netdev)
3248 goto err_alloc_etherdev;
3249
3250 SET_NETDEV_DEV(netdev, &pdev->dev);
3251
3252 pci_set_drvdata(pdev, netdev);
3253 adapter = netdev_priv(netdev);
3254 adapter->netdev = netdev;
3255 adapter->pdev = pdev;
3256 hw = &adapter->hw;
3257 hw->back = adapter;
3258 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3259
3260 err = -EIO;
3261 adapter->io_addr = pci_iomap(pdev, 0, 0);
3262 if (!adapter->io_addr)
3263 goto err_ioremap;
3264 /* hw->hw_addr can be altered, we'll use adapter->io_addr for unmap */
3265 hw->hw_addr = adapter->io_addr;
3266
3267 netdev->netdev_ops = &igb_netdev_ops;
3268 igb_set_ethtool_ops(netdev);
3269 netdev->watchdog_timeo = 5 * HZ;
3270
3271 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
3272
3273 netdev->mem_start = pci_resource_start(pdev, 0);
3274 netdev->mem_end = pci_resource_end(pdev, 0);
3275
3276 /* PCI config space info */
3277 hw->vendor_id = pdev->vendor;
3278 hw->device_id = pdev->device;
3279 hw->revision_id = pdev->revision;
3280 hw->subsystem_vendor_id = pdev->subsystem_vendor;
3281 hw->subsystem_device_id = pdev->subsystem_device;
3282
3283 /* Copy the default MAC, PHY and NVM function pointers */
3284 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
3285 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
3286 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
3287 /* Initialize skew-specific constants */
3288 err = ei->get_invariants(hw);
3289 if (err)
3290 goto err_sw_init;
3291
3292 /* setup the private structure */
3293 err = igb_sw_init(adapter);
3294 if (err)
3295 goto err_sw_init;
3296
3297 igb_get_bus_info_pcie(hw);
3298
3299 hw->phy.autoneg_wait_to_complete = false;
3300
3301 /* Copper options */
3302 if (hw->phy.media_type == e1000_media_type_copper) {
3303 hw->phy.mdix = AUTO_ALL_MODES;
3304 hw->phy.disable_polarity_correction = false;
3305 hw->phy.ms_type = e1000_ms_hw_default;
3306 }
3307
3308 if (igb_check_reset_block(hw))
3309 dev_info(&pdev->dev,
3310 "PHY reset is blocked due to SOL/IDER session.\n");
3311
3312 /* features is initialized to 0 in allocation, it might have bits
3313 * set by igb_sw_init so we should use an or instead of an
3314 * assignment.
3315 */
3316 netdev->features |= NETIF_F_SG |
3317 NETIF_F_TSO |
3318 NETIF_F_TSO6 |
3319 NETIF_F_RXHASH |
3320 NETIF_F_RXCSUM |
3321 NETIF_F_HW_CSUM;
3322
3323 if (hw->mac.type >= e1000_82576)
3324 netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4;
3325
3326 if (hw->mac.type >= e1000_i350)
3327 netdev->features |= NETIF_F_HW_TC;
3328
3329 #define IGB_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
3330 NETIF_F_GSO_GRE_CSUM | \
3331 NETIF_F_GSO_IPXIP4 | \
3332 NETIF_F_GSO_IPXIP6 | \
3333 NETIF_F_GSO_UDP_TUNNEL | \
3334 NETIF_F_GSO_UDP_TUNNEL_CSUM)
3335
3336 netdev->gso_partial_features = IGB_GSO_PARTIAL_FEATURES;
3337 netdev->features |= NETIF_F_GSO_PARTIAL | IGB_GSO_PARTIAL_FEATURES;
3338
3339 /* copy netdev features into list of user selectable features */
3340 netdev->hw_features |= netdev->features |
3341 NETIF_F_HW_VLAN_CTAG_RX |
3342 NETIF_F_HW_VLAN_CTAG_TX |
3343 NETIF_F_RXALL;
3344
3345 if (hw->mac.type >= e1000_i350)
3346 netdev->hw_features |= NETIF_F_NTUPLE;
3347
3348 netdev->features |= NETIF_F_HIGHDMA;
3349
3350 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
3351 netdev->mpls_features |= NETIF_F_HW_CSUM;
3352 netdev->hw_enc_features |= netdev->vlan_features;
3353
3354 /* set this bit last since it cannot be part of vlan_features */
3355 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
3356 NETIF_F_HW_VLAN_CTAG_RX |
3357 NETIF_F_HW_VLAN_CTAG_TX;
3358
3359 netdev->priv_flags |= IFF_SUPP_NOFCS;
3360
3361 netdev->priv_flags |= IFF_UNICAST_FLT;
3362 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT;
3363
3364 /* MTU range: 68 - 9216 */
3365 netdev->min_mtu = ETH_MIN_MTU;
3366 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
3367
3368 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
3369
3370 /* before reading the NVM, reset the controller to put the device in a
3371 * known good starting state
3372 */
3373 hw->mac.ops.reset_hw(hw);
3374
3375 /* make sure the NVM is good , i211/i210 parts can have special NVM
3376 * that doesn't contain a checksum
3377 */
3378 switch (hw->mac.type) {
3379 case e1000_i210:
3380 case e1000_i211:
3381 if (igb_get_flash_presence_i210(hw)) {
3382 if (hw->nvm.ops.validate(hw) < 0) {
3383 dev_err(&pdev->dev,
3384 "The NVM Checksum Is Not Valid\n");
3385 err = -EIO;
3386 goto err_eeprom;
3387 }
3388 }
3389 break;
3390 default:
3391 if (hw->nvm.ops.validate(hw) < 0) {
3392 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
3393 err = -EIO;
3394 goto err_eeprom;
3395 }
3396 break;
3397 }
3398
3399 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
3400 /* copy the MAC address out of the NVM */
3401 if (hw->mac.ops.read_mac_addr(hw))
3402 dev_err(&pdev->dev, "NVM Read Error\n");
3403 }
3404
3405 eth_hw_addr_set(netdev, hw->mac.addr);
3406
3407 if (!is_valid_ether_addr(netdev->dev_addr)) {
3408 dev_err(&pdev->dev, "Invalid MAC Address\n");
3409 err = -EIO;
3410 goto err_eeprom;
3411 }
3412
3413 igb_set_default_mac_filter(adapter);
3414
3415 /* get firmware version for ethtool -i */
3416 igb_set_fw_version(adapter);
3417
3418 /* configure RXPBSIZE and TXPBSIZE */
3419 if (hw->mac.type == e1000_i210) {
3420 wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
3421 wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
3422 }
3423
3424 timer_setup(&adapter->watchdog_timer, igb_watchdog, 0);
3425 timer_setup(&adapter->phy_info_timer, igb_update_phy_info, 0);
3426
3427 INIT_WORK(&adapter->reset_task, igb_reset_task);
3428 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
3429
3430 /* Initialize link properties that are user-changeable */
3431 adapter->fc_autoneg = true;
3432 hw->mac.autoneg = true;
3433 hw->phy.autoneg_advertised = 0x2f;
3434
3435 hw->fc.requested_mode = e1000_fc_default;
3436 hw->fc.current_mode = e1000_fc_default;
3437
3438 igb_validate_mdi_setting(hw);
3439
3440 /* By default, support wake on port A */
3441 if (hw->bus.func == 0)
3442 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3443
3444 /* Check the NVM for wake support on non-port A ports */
3445 if (hw->mac.type >= e1000_82580)
3446 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
3447 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
3448 &eeprom_data);
3449 else if (hw->bus.func == 1)
3450 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
3451
3452 if (eeprom_data & IGB_EEPROM_APME)
3453 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3454
3455 /* now that we have the eeprom settings, apply the special cases where
3456 * the eeprom may be wrong or the board simply won't support wake on
3457 * lan on a particular port
3458 */
3459 switch (pdev->device) {
3460 case E1000_DEV_ID_82575GB_QUAD_COPPER:
3461 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3462 break;
3463 case E1000_DEV_ID_82575EB_FIBER_SERDES:
3464 case E1000_DEV_ID_82576_FIBER:
3465 case E1000_DEV_ID_82576_SERDES:
3466 /* Wake events only supported on port A for dual fiber
3467 * regardless of eeprom setting
3468 */
3469 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
3470 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3471 break;
3472 case E1000_DEV_ID_82576_QUAD_COPPER:
3473 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
3474 /* if quad port adapter, disable WoL on all but port A */
3475 if (global_quad_port_a != 0)
3476 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3477 else
3478 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
3479 /* Reset for multiple quad port adapters */
3480 if (++global_quad_port_a == 4)
3481 global_quad_port_a = 0;
3482 break;
3483 default:
3484 /* If the device can't wake, don't set software support */
3485 if (!device_can_wakeup(&adapter->pdev->dev))
3486 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3487 }
3488
3489 /* initialize the wol settings based on the eeprom settings */
3490 if (adapter->flags & IGB_FLAG_WOL_SUPPORTED)
3491 adapter->wol |= E1000_WUFC_MAG;
3492
3493 /* Some vendors want WoL disabled by default, but still supported */
3494 if ((hw->mac.type == e1000_i350) &&
3495 (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) {
3496 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3497 adapter->wol = 0;
3498 }
3499
3500 /* Some vendors want the ability to Use the EEPROM setting as
3501 * enable/disable only, and not for capability
3502 */
3503 if (((hw->mac.type == e1000_i350) ||
3504 (hw->mac.type == e1000_i354)) &&
3505 (pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)) {
3506 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3507 adapter->wol = 0;
3508 }
3509 if (hw->mac.type == e1000_i350) {
3510 if (((pdev->subsystem_device == 0x5001) ||
3511 (pdev->subsystem_device == 0x5002)) &&
3512 (hw->bus.func == 0)) {
3513 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3514 adapter->wol = 0;
3515 }
3516 if (pdev->subsystem_device == 0x1F52)
3517 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3518 }
3519
3520 device_set_wakeup_enable(&adapter->pdev->dev,
3521 adapter->flags & IGB_FLAG_WOL_SUPPORTED);
3522
3523 /* reset the hardware with the new settings */
3524 igb_reset(adapter);
3525
3526 /* Init the I2C interface */
3527 err = igb_init_i2c(adapter);
3528 if (err) {
3529 dev_err(&pdev->dev, "failed to init i2c interface\n");
3530 goto err_eeprom;
3531 }
3532
3533 /* let the f/w know that the h/w is now under the control of the
3534 * driver.
3535 */
3536 igb_get_hw_control(adapter);
3537
3538 strcpy(netdev->name, "eth%d");
3539 err = register_netdev(netdev);
3540 if (err)
3541 goto err_register;
3542
3543 /* carrier off reporting is important to ethtool even BEFORE open */
3544 netif_carrier_off(netdev);
3545
3546 #ifdef CONFIG_IGB_DCA
3547 if (dca_add_requester(&pdev->dev) == 0) {
3548 adapter->flags |= IGB_FLAG_DCA_ENABLED;
3549 dev_info(&pdev->dev, "DCA enabled\n");
3550 igb_setup_dca(adapter);
3551 }
3552
3553 #endif
3554 #ifdef CONFIG_IGB_HWMON
3555 /* Initialize the thermal sensor on i350 devices. */
3556 if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
3557 u16 ets_word;
3558
3559 /* Read the NVM to determine if this i350 device supports an
3560 * external thermal sensor.
3561 */
3562 hw->nvm.ops.read(hw, NVM_ETS_CFG, 1, &ets_word);
3563 if (ets_word != 0x0000 && ets_word != 0xFFFF)
3564 adapter->ets = true;
3565 else
3566 adapter->ets = false;
3567 /* Only enable I2C bit banging if an external thermal
3568 * sensor is supported.
3569 */
3570 if (adapter->ets)
3571 igb_set_i2c_bb(hw);
3572 hw->mac.ops.init_thermal_sensor_thresh(hw);
3573 if (igb_sysfs_init(adapter))
3574 dev_err(&pdev->dev,
3575 "failed to allocate sysfs resources\n");
3576 } else {
3577 adapter->ets = false;
3578 }
3579 #endif
3580 /* Check if Media Autosense is enabled */
3581 adapter->ei = *ei;
3582 if (hw->dev_spec._82575.mas_capable)
3583 igb_init_mas(adapter);
3584
3585 /* do hw tstamp init after resetting */
3586 igb_ptp_init(adapter);
3587
3588 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
3589 /* print bus type/speed/width info, not applicable to i354 */
3590 if (hw->mac.type != e1000_i354) {
3591 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
3592 netdev->name,
3593 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
3594 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
3595 "unknown"),
3596 ((hw->bus.width == e1000_bus_width_pcie_x4) ?
3597 "Width x4" :
3598 (hw->bus.width == e1000_bus_width_pcie_x2) ?
3599 "Width x2" :
3600 (hw->bus.width == e1000_bus_width_pcie_x1) ?
3601 "Width x1" : "unknown"), netdev->dev_addr);
3602 }
3603
3604 if ((hw->mac.type == e1000_82576 &&
3605 rd32(E1000_EECD) & E1000_EECD_PRES) ||
3606 (hw->mac.type >= e1000_i210 ||
3607 igb_get_flash_presence_i210(hw))) {
3608 ret_val = igb_read_part_string(hw, part_str,
3609 E1000_PBANUM_LENGTH);
3610 } else {
3611 ret_val = -E1000_ERR_INVM_VALUE_NOT_FOUND;
3612 }
3613
3614 if (ret_val)
3615 strcpy(part_str, "Unknown");
3616 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
3617 dev_info(&pdev->dev,
3618 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
3619 (adapter->flags & IGB_FLAG_HAS_MSIX) ? "MSI-X" :
3620 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
3621 adapter->num_rx_queues, adapter->num_tx_queues);
3622 if (hw->phy.media_type == e1000_media_type_copper) {
3623 switch (hw->mac.type) {
3624 case e1000_i350:
3625 case e1000_i210:
3626 case e1000_i211:
3627 /* Enable EEE for internal copper PHY devices */
3628 err = igb_set_eee_i350(hw, true, true);
3629 if ((!err) &&
3630 (!hw->dev_spec._82575.eee_disable)) {
3631 adapter->eee_advert =
3632 MDIO_EEE_100TX | MDIO_EEE_1000T;
3633 adapter->flags |= IGB_FLAG_EEE;
3634 }
3635 break;
3636 case e1000_i354:
3637 if ((rd32(E1000_CTRL_EXT) &
3638 E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3639 err = igb_set_eee_i354(hw, true, true);
3640 if ((!err) &&
3641 (!hw->dev_spec._82575.eee_disable)) {
3642 adapter->eee_advert =
3643 MDIO_EEE_100TX | MDIO_EEE_1000T;
3644 adapter->flags |= IGB_FLAG_EEE;
3645 }
3646 }
3647 break;
3648 default:
3649 break;
3650 }
3651 }
3652
3653 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
3654
3655 pm_runtime_put_noidle(&pdev->dev);
3656 return 0;
3657
3658 err_register:
3659 igb_release_hw_control(adapter);
3660 memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap));
3661 err_eeprom:
3662 if (!igb_check_reset_block(hw))
3663 igb_reset_phy(hw);
3664
3665 if (hw->flash_address)
3666 iounmap(hw->flash_address);
3667 err_sw_init:
3668 kfree(adapter->mac_table);
3669 kfree(adapter->shadow_vfta);
3670 igb_clear_interrupt_scheme(adapter);
3671 #ifdef CONFIG_PCI_IOV
3672 igb_disable_sriov(pdev, false);
3673 #endif
3674 pci_iounmap(pdev, adapter->io_addr);
3675 err_ioremap:
3676 free_netdev(netdev);
3677 err_alloc_etherdev:
3678 pci_release_mem_regions(pdev);
3679 err_pci_reg:
3680 err_dma:
3681 pci_disable_device(pdev);
3682 return err;
3683 }
3684
3685 #ifdef CONFIG_PCI_IOV
igb_sriov_reinit(struct pci_dev * dev)3686 static int igb_sriov_reinit(struct pci_dev *dev)
3687 {
3688 struct net_device *netdev = pci_get_drvdata(dev);
3689 struct igb_adapter *adapter = netdev_priv(netdev);
3690 struct pci_dev *pdev = adapter->pdev;
3691
3692 rtnl_lock();
3693
3694 if (netif_running(netdev))
3695 igb_close(netdev);
3696 else
3697 igb_reset(adapter);
3698
3699 igb_clear_interrupt_scheme(adapter);
3700
3701 igb_init_queue_configuration(adapter);
3702
3703 if (igb_init_interrupt_scheme(adapter, true)) {
3704 rtnl_unlock();
3705 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
3706 return -ENOMEM;
3707 }
3708
3709 if (netif_running(netdev))
3710 igb_open(netdev);
3711
3712 rtnl_unlock();
3713
3714 return 0;
3715 }
3716
igb_disable_sriov(struct pci_dev * pdev,bool reinit)3717 static int igb_disable_sriov(struct pci_dev *pdev, bool reinit)
3718 {
3719 struct net_device *netdev = pci_get_drvdata(pdev);
3720 struct igb_adapter *adapter = netdev_priv(netdev);
3721 struct e1000_hw *hw = &adapter->hw;
3722 unsigned long flags;
3723
3724 /* reclaim resources allocated to VFs */
3725 if (adapter->vf_data) {
3726 /* disable iov and allow time for transactions to clear */
3727 if (pci_vfs_assigned(pdev)) {
3728 dev_warn(&pdev->dev,
3729 "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n");
3730 return -EPERM;
3731 } else {
3732 pci_disable_sriov(pdev);
3733 msleep(500);
3734 }
3735 spin_lock_irqsave(&adapter->vfs_lock, flags);
3736 kfree(adapter->vf_mac_list);
3737 adapter->vf_mac_list = NULL;
3738 kfree(adapter->vf_data);
3739 adapter->vf_data = NULL;
3740 adapter->vfs_allocated_count = 0;
3741 spin_unlock_irqrestore(&adapter->vfs_lock, flags);
3742 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
3743 wrfl();
3744 msleep(100);
3745 dev_info(&pdev->dev, "IOV Disabled\n");
3746
3747 /* Re-enable DMA Coalescing flag since IOV is turned off */
3748 adapter->flags |= IGB_FLAG_DMAC;
3749 }
3750
3751 return reinit ? igb_sriov_reinit(pdev) : 0;
3752 }
3753
igb_enable_sriov(struct pci_dev * pdev,int num_vfs,bool reinit)3754 static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs, bool reinit)
3755 {
3756 struct net_device *netdev = pci_get_drvdata(pdev);
3757 struct igb_adapter *adapter = netdev_priv(netdev);
3758 int old_vfs = pci_num_vf(pdev);
3759 struct vf_mac_filter *mac_list;
3760 int err = 0;
3761 int num_vf_mac_filters, i;
3762
3763 if (!(adapter->flags & IGB_FLAG_HAS_MSIX) || num_vfs > 7) {
3764 err = -EPERM;
3765 goto out;
3766 }
3767 if (!num_vfs)
3768 goto out;
3769
3770 if (old_vfs) {
3771 dev_info(&pdev->dev, "%d pre-allocated VFs found - override max_vfs setting of %d\n",
3772 old_vfs, max_vfs);
3773 adapter->vfs_allocated_count = old_vfs;
3774 } else
3775 adapter->vfs_allocated_count = num_vfs;
3776
3777 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
3778 sizeof(struct vf_data_storage), GFP_KERNEL);
3779
3780 /* if allocation failed then we do not support SR-IOV */
3781 if (!adapter->vf_data) {
3782 adapter->vfs_allocated_count = 0;
3783 err = -ENOMEM;
3784 goto out;
3785 }
3786
3787 /* Due to the limited number of RAR entries calculate potential
3788 * number of MAC filters available for the VFs. Reserve entries
3789 * for PF default MAC, PF MAC filters and at least one RAR entry
3790 * for each VF for VF MAC.
3791 */
3792 num_vf_mac_filters = adapter->hw.mac.rar_entry_count -
3793 (1 + IGB_PF_MAC_FILTERS_RESERVED +
3794 adapter->vfs_allocated_count);
3795
3796 adapter->vf_mac_list = kcalloc(num_vf_mac_filters,
3797 sizeof(struct vf_mac_filter),
3798 GFP_KERNEL);
3799
3800 mac_list = adapter->vf_mac_list;
3801 INIT_LIST_HEAD(&adapter->vf_macs.l);
3802
3803 if (adapter->vf_mac_list) {
3804 /* Initialize list of VF MAC filters */
3805 for (i = 0; i < num_vf_mac_filters; i++) {
3806 mac_list->vf = -1;
3807 mac_list->free = true;
3808 list_add(&mac_list->l, &adapter->vf_macs.l);
3809 mac_list++;
3810 }
3811 } else {
3812 /* If we could not allocate memory for the VF MAC filters
3813 * we can continue without this feature but warn user.
3814 */
3815 dev_err(&pdev->dev,
3816 "Unable to allocate memory for VF MAC filter list\n");
3817 }
3818
3819 dev_info(&pdev->dev, "%d VFs allocated\n",
3820 adapter->vfs_allocated_count);
3821 for (i = 0; i < adapter->vfs_allocated_count; i++)
3822 igb_vf_configure(adapter, i);
3823
3824 /* DMA Coalescing is not supported in IOV mode. */
3825 adapter->flags &= ~IGB_FLAG_DMAC;
3826
3827 if (reinit) {
3828 err = igb_sriov_reinit(pdev);
3829 if (err)
3830 goto err_out;
3831 }
3832
3833 /* only call pci_enable_sriov() if no VFs are allocated already */
3834 if (!old_vfs) {
3835 err = pci_enable_sriov(pdev, adapter->vfs_allocated_count);
3836 if (err)
3837 goto err_out;
3838 }
3839
3840 goto out;
3841
3842 err_out:
3843 kfree(adapter->vf_mac_list);
3844 adapter->vf_mac_list = NULL;
3845 kfree(adapter->vf_data);
3846 adapter->vf_data = NULL;
3847 adapter->vfs_allocated_count = 0;
3848 out:
3849 return err;
3850 }
3851
3852 #endif
3853 /**
3854 * igb_remove_i2c - Cleanup I2C interface
3855 * @adapter: pointer to adapter structure
3856 **/
igb_remove_i2c(struct igb_adapter * adapter)3857 static void igb_remove_i2c(struct igb_adapter *adapter)
3858 {
3859 /* free the adapter bus structure */
3860 i2c_del_adapter(&adapter->i2c_adap);
3861 }
3862
3863 /**
3864 * igb_remove - Device Removal Routine
3865 * @pdev: PCI device information struct
3866 *
3867 * igb_remove is called by the PCI subsystem to alert the driver
3868 * that it should release a PCI device. The could be caused by a
3869 * Hot-Plug event, or because the driver is going to be removed from
3870 * memory.
3871 **/
igb_remove(struct pci_dev * pdev)3872 static void igb_remove(struct pci_dev *pdev)
3873 {
3874 struct net_device *netdev = pci_get_drvdata(pdev);
3875 struct igb_adapter *adapter = netdev_priv(netdev);
3876 struct e1000_hw *hw = &adapter->hw;
3877
3878 pm_runtime_get_noresume(&pdev->dev);
3879 #ifdef CONFIG_IGB_HWMON
3880 igb_sysfs_exit(adapter);
3881 #endif
3882 igb_remove_i2c(adapter);
3883 igb_ptp_stop(adapter);
3884 /* The watchdog timer may be rescheduled, so explicitly
3885 * disable watchdog from being rescheduled.
3886 */
3887 set_bit(__IGB_DOWN, &adapter->state);
3888 del_timer_sync(&adapter->watchdog_timer);
3889 del_timer_sync(&adapter->phy_info_timer);
3890
3891 cancel_work_sync(&adapter->reset_task);
3892 cancel_work_sync(&adapter->watchdog_task);
3893
3894 #ifdef CONFIG_IGB_DCA
3895 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
3896 dev_info(&pdev->dev, "DCA disabled\n");
3897 dca_remove_requester(&pdev->dev);
3898 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
3899 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
3900 }
3901 #endif
3902
3903 /* Release control of h/w to f/w. If f/w is AMT enabled, this
3904 * would have already happened in close and is redundant.
3905 */
3906 igb_release_hw_control(adapter);
3907
3908 #ifdef CONFIG_PCI_IOV
3909 igb_disable_sriov(pdev, false);
3910 #endif
3911
3912 unregister_netdev(netdev);
3913
3914 igb_clear_interrupt_scheme(adapter);
3915
3916 pci_iounmap(pdev, adapter->io_addr);
3917 if (hw->flash_address)
3918 iounmap(hw->flash_address);
3919 pci_release_mem_regions(pdev);
3920
3921 kfree(adapter->mac_table);
3922 kfree(adapter->shadow_vfta);
3923 free_netdev(netdev);
3924
3925 pci_disable_device(pdev);
3926 }
3927
3928 /**
3929 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
3930 * @adapter: board private structure to initialize
3931 *
3932 * This function initializes the vf specific data storage and then attempts to
3933 * allocate the VFs. The reason for ordering it this way is because it is much
3934 * mor expensive time wise to disable SR-IOV than it is to allocate and free
3935 * the memory for the VFs.
3936 **/
igb_probe_vfs(struct igb_adapter * adapter)3937 static void igb_probe_vfs(struct igb_adapter *adapter)
3938 {
3939 #ifdef CONFIG_PCI_IOV
3940 struct pci_dev *pdev = adapter->pdev;
3941 struct e1000_hw *hw = &adapter->hw;
3942
3943 /* Virtualization features not supported on i210 and 82580 family. */
3944 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211) ||
3945 (hw->mac.type == e1000_82580))
3946 return;
3947
3948 /* Of the below we really only want the effect of getting
3949 * IGB_FLAG_HAS_MSIX set (if available), without which
3950 * igb_enable_sriov() has no effect.
3951 */
3952 igb_set_interrupt_capability(adapter, true);
3953 igb_reset_interrupt_capability(adapter);
3954
3955 pci_sriov_set_totalvfs(pdev, 7);
3956 igb_enable_sriov(pdev, max_vfs, false);
3957
3958 #endif /* CONFIG_PCI_IOV */
3959 }
3960
igb_get_max_rss_queues(struct igb_adapter * adapter)3961 unsigned int igb_get_max_rss_queues(struct igb_adapter *adapter)
3962 {
3963 struct e1000_hw *hw = &adapter->hw;
3964 unsigned int max_rss_queues;
3965
3966 /* Determine the maximum number of RSS queues supported. */
3967 switch (hw->mac.type) {
3968 case e1000_i211:
3969 max_rss_queues = IGB_MAX_RX_QUEUES_I211;
3970 break;
3971 case e1000_82575:
3972 case e1000_i210:
3973 max_rss_queues = IGB_MAX_RX_QUEUES_82575;
3974 break;
3975 case e1000_i350:
3976 /* I350 cannot do RSS and SR-IOV at the same time */
3977 if (!!adapter->vfs_allocated_count) {
3978 max_rss_queues = 1;
3979 break;
3980 }
3981 fallthrough;
3982 case e1000_82576:
3983 if (!!adapter->vfs_allocated_count) {
3984 max_rss_queues = 2;
3985 break;
3986 }
3987 fallthrough;
3988 case e1000_82580:
3989 case e1000_i354:
3990 default:
3991 max_rss_queues = IGB_MAX_RX_QUEUES;
3992 break;
3993 }
3994
3995 return max_rss_queues;
3996 }
3997
igb_init_queue_configuration(struct igb_adapter * adapter)3998 static void igb_init_queue_configuration(struct igb_adapter *adapter)
3999 {
4000 u32 max_rss_queues;
4001
4002 max_rss_queues = igb_get_max_rss_queues(adapter);
4003 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
4004
4005 igb_set_flag_queue_pairs(adapter, max_rss_queues);
4006 }
4007
igb_set_flag_queue_pairs(struct igb_adapter * adapter,const u32 max_rss_queues)4008 void igb_set_flag_queue_pairs(struct igb_adapter *adapter,
4009 const u32 max_rss_queues)
4010 {
4011 struct e1000_hw *hw = &adapter->hw;
4012
4013 /* Determine if we need to pair queues. */
4014 switch (hw->mac.type) {
4015 case e1000_82575:
4016 case e1000_i211:
4017 /* Device supports enough interrupts without queue pairing. */
4018 break;
4019 case e1000_82576:
4020 case e1000_82580:
4021 case e1000_i350:
4022 case e1000_i354:
4023 case e1000_i210:
4024 default:
4025 /* If rss_queues > half of max_rss_queues, pair the queues in
4026 * order to conserve interrupts due to limited supply.
4027 */
4028 if (adapter->rss_queues > (max_rss_queues / 2))
4029 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
4030 else
4031 adapter->flags &= ~IGB_FLAG_QUEUE_PAIRS;
4032 break;
4033 }
4034 }
4035
4036 /**
4037 * igb_sw_init - Initialize general software structures (struct igb_adapter)
4038 * @adapter: board private structure to initialize
4039 *
4040 * igb_sw_init initializes the Adapter private data structure.
4041 * Fields are initialized based on PCI device information and
4042 * OS network device settings (MTU size).
4043 **/
igb_sw_init(struct igb_adapter * adapter)4044 static int igb_sw_init(struct igb_adapter *adapter)
4045 {
4046 struct e1000_hw *hw = &adapter->hw;
4047 struct net_device *netdev = adapter->netdev;
4048 struct pci_dev *pdev = adapter->pdev;
4049
4050 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
4051
4052 /* set default ring sizes */
4053 adapter->tx_ring_count = IGB_DEFAULT_TXD;
4054 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4055
4056 /* set default ITR values */
4057 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
4058 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
4059
4060 /* set default work limits */
4061 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
4062
4063 adapter->max_frame_size = netdev->mtu + IGB_ETH_PKT_HDR_PAD;
4064 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
4065
4066 spin_lock_init(&adapter->nfc_lock);
4067 spin_lock_init(&adapter->stats64_lock);
4068
4069 /* init spinlock to avoid concurrency of VF resources */
4070 spin_lock_init(&adapter->vfs_lock);
4071 #ifdef CONFIG_PCI_IOV
4072 switch (hw->mac.type) {
4073 case e1000_82576:
4074 case e1000_i350:
4075 if (max_vfs > 7) {
4076 dev_warn(&pdev->dev,
4077 "Maximum of 7 VFs per PF, using max\n");
4078 max_vfs = adapter->vfs_allocated_count = 7;
4079 } else
4080 adapter->vfs_allocated_count = max_vfs;
4081 if (adapter->vfs_allocated_count)
4082 dev_warn(&pdev->dev,
4083 "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n");
4084 break;
4085 default:
4086 break;
4087 }
4088 #endif /* CONFIG_PCI_IOV */
4089
4090 /* Assume MSI-X interrupts, will be checked during IRQ allocation */
4091 adapter->flags |= IGB_FLAG_HAS_MSIX;
4092
4093 adapter->mac_table = kcalloc(hw->mac.rar_entry_count,
4094 sizeof(struct igb_mac_addr),
4095 GFP_KERNEL);
4096 if (!adapter->mac_table)
4097 return -ENOMEM;
4098
4099 igb_probe_vfs(adapter);
4100
4101 igb_init_queue_configuration(adapter);
4102
4103 /* Setup and initialize a copy of the hw vlan table array */
4104 adapter->shadow_vfta = kcalloc(E1000_VLAN_FILTER_TBL_SIZE, sizeof(u32),
4105 GFP_KERNEL);
4106 if (!adapter->shadow_vfta)
4107 return -ENOMEM;
4108
4109 /* This call may decrease the number of queues */
4110 if (igb_init_interrupt_scheme(adapter, true)) {
4111 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
4112 return -ENOMEM;
4113 }
4114
4115 /* Explicitly disable IRQ since the NIC can be in any state. */
4116 igb_irq_disable(adapter);
4117
4118 if (hw->mac.type >= e1000_i350)
4119 adapter->flags &= ~IGB_FLAG_DMAC;
4120
4121 set_bit(__IGB_DOWN, &adapter->state);
4122 return 0;
4123 }
4124
4125 /**
4126 * __igb_open - Called when a network interface is made active
4127 * @netdev: network interface device structure
4128 * @resuming: indicates whether we are in a resume call
4129 *
4130 * Returns 0 on success, negative value on failure
4131 *
4132 * The open entry point is called when a network interface is made
4133 * active by the system (IFF_UP). At this point all resources needed
4134 * for transmit and receive operations are allocated, the interrupt
4135 * handler is registered with the OS, the watchdog timer is started,
4136 * and the stack is notified that the interface is ready.
4137 **/
__igb_open(struct net_device * netdev,bool resuming)4138 static int __igb_open(struct net_device *netdev, bool resuming)
4139 {
4140 struct igb_adapter *adapter = netdev_priv(netdev);
4141 struct e1000_hw *hw = &adapter->hw;
4142 struct pci_dev *pdev = adapter->pdev;
4143 int err;
4144 int i;
4145
4146 /* disallow open during test */
4147 if (test_bit(__IGB_TESTING, &adapter->state)) {
4148 WARN_ON(resuming);
4149 return -EBUSY;
4150 }
4151
4152 if (!resuming)
4153 pm_runtime_get_sync(&pdev->dev);
4154
4155 netif_carrier_off(netdev);
4156
4157 /* allocate transmit descriptors */
4158 err = igb_setup_all_tx_resources(adapter);
4159 if (err)
4160 goto err_setup_tx;
4161
4162 /* allocate receive descriptors */
4163 err = igb_setup_all_rx_resources(adapter);
4164 if (err)
4165 goto err_setup_rx;
4166
4167 igb_power_up_link(adapter);
4168
4169 /* before we allocate an interrupt, we must be ready to handle it.
4170 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
4171 * as soon as we call pci_request_irq, so we have to setup our
4172 * clean_rx handler before we do so.
4173 */
4174 igb_configure(adapter);
4175
4176 err = igb_request_irq(adapter);
4177 if (err)
4178 goto err_req_irq;
4179
4180 /* Notify the stack of the actual queue counts. */
4181 err = netif_set_real_num_tx_queues(adapter->netdev,
4182 adapter->num_tx_queues);
4183 if (err)
4184 goto err_set_queues;
4185
4186 err = netif_set_real_num_rx_queues(adapter->netdev,
4187 adapter->num_rx_queues);
4188 if (err)
4189 goto err_set_queues;
4190
4191 /* From here on the code is the same as igb_up() */
4192 clear_bit(__IGB_DOWN, &adapter->state);
4193
4194 for (i = 0; i < adapter->num_q_vectors; i++)
4195 napi_enable(&(adapter->q_vector[i]->napi));
4196
4197 /* Clear any pending interrupts. */
4198 rd32(E1000_TSICR);
4199 rd32(E1000_ICR);
4200
4201 igb_irq_enable(adapter);
4202
4203 /* notify VFs that reset has been completed */
4204 if (adapter->vfs_allocated_count) {
4205 u32 reg_data = rd32(E1000_CTRL_EXT);
4206
4207 reg_data |= E1000_CTRL_EXT_PFRSTD;
4208 wr32(E1000_CTRL_EXT, reg_data);
4209 }
4210
4211 netif_tx_start_all_queues(netdev);
4212
4213 if (!resuming)
4214 pm_runtime_put(&pdev->dev);
4215
4216 /* start the watchdog. */
4217 hw->mac.get_link_status = 1;
4218 schedule_work(&adapter->watchdog_task);
4219
4220 return 0;
4221
4222 err_set_queues:
4223 igb_free_irq(adapter);
4224 err_req_irq:
4225 igb_release_hw_control(adapter);
4226 igb_power_down_link(adapter);
4227 igb_free_all_rx_resources(adapter);
4228 err_setup_rx:
4229 igb_free_all_tx_resources(adapter);
4230 err_setup_tx:
4231 igb_reset(adapter);
4232 if (!resuming)
4233 pm_runtime_put(&pdev->dev);
4234
4235 return err;
4236 }
4237
igb_open(struct net_device * netdev)4238 int igb_open(struct net_device *netdev)
4239 {
4240 return __igb_open(netdev, false);
4241 }
4242
4243 /**
4244 * __igb_close - Disables a network interface
4245 * @netdev: network interface device structure
4246 * @suspending: indicates we are in a suspend call
4247 *
4248 * Returns 0, this is not allowed to fail
4249 *
4250 * The close entry point is called when an interface is de-activated
4251 * by the OS. The hardware is still under the driver's control, but
4252 * needs to be disabled. A global MAC reset is issued to stop the
4253 * hardware, and all transmit and receive resources are freed.
4254 **/
__igb_close(struct net_device * netdev,bool suspending)4255 static int __igb_close(struct net_device *netdev, bool suspending)
4256 {
4257 struct igb_adapter *adapter = netdev_priv(netdev);
4258 struct pci_dev *pdev = adapter->pdev;
4259
4260 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
4261
4262 if (!suspending)
4263 pm_runtime_get_sync(&pdev->dev);
4264
4265 igb_down(adapter);
4266 igb_free_irq(adapter);
4267
4268 igb_free_all_tx_resources(adapter);
4269 igb_free_all_rx_resources(adapter);
4270
4271 if (!suspending)
4272 pm_runtime_put_sync(&pdev->dev);
4273 return 0;
4274 }
4275
igb_close(struct net_device * netdev)4276 int igb_close(struct net_device *netdev)
4277 {
4278 if (netif_device_present(netdev) || netdev->dismantle)
4279 return __igb_close(netdev, false);
4280 return 0;
4281 }
4282
4283 /**
4284 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
4285 * @tx_ring: tx descriptor ring (for a specific queue) to setup
4286 *
4287 * Return 0 on success, negative on failure
4288 **/
igb_setup_tx_resources(struct igb_ring * tx_ring)4289 int igb_setup_tx_resources(struct igb_ring *tx_ring)
4290 {
4291 struct device *dev = tx_ring->dev;
4292 int size;
4293
4294 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
4295
4296 tx_ring->tx_buffer_info = vmalloc(size);
4297 if (!tx_ring->tx_buffer_info)
4298 goto err;
4299
4300 /* round up to nearest 4K */
4301 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
4302 tx_ring->size = ALIGN(tx_ring->size, 4096);
4303
4304 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
4305 &tx_ring->dma, GFP_KERNEL);
4306 if (!tx_ring->desc)
4307 goto err;
4308
4309 tx_ring->next_to_use = 0;
4310 tx_ring->next_to_clean = 0;
4311
4312 return 0;
4313
4314 err:
4315 vfree(tx_ring->tx_buffer_info);
4316 tx_ring->tx_buffer_info = NULL;
4317 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
4318 return -ENOMEM;
4319 }
4320
4321 /**
4322 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
4323 * (Descriptors) for all queues
4324 * @adapter: board private structure
4325 *
4326 * Return 0 on success, negative on failure
4327 **/
igb_setup_all_tx_resources(struct igb_adapter * adapter)4328 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
4329 {
4330 struct pci_dev *pdev = adapter->pdev;
4331 int i, err = 0;
4332
4333 for (i = 0; i < adapter->num_tx_queues; i++) {
4334 err = igb_setup_tx_resources(adapter->tx_ring[i]);
4335 if (err) {
4336 dev_err(&pdev->dev,
4337 "Allocation for Tx Queue %u failed\n", i);
4338 for (i--; i >= 0; i--)
4339 igb_free_tx_resources(adapter->tx_ring[i]);
4340 break;
4341 }
4342 }
4343
4344 return err;
4345 }
4346
4347 /**
4348 * igb_setup_tctl - configure the transmit control registers
4349 * @adapter: Board private structure
4350 **/
igb_setup_tctl(struct igb_adapter * adapter)4351 void igb_setup_tctl(struct igb_adapter *adapter)
4352 {
4353 struct e1000_hw *hw = &adapter->hw;
4354 u32 tctl;
4355
4356 /* disable queue 0 which is enabled by default on 82575 and 82576 */
4357 wr32(E1000_TXDCTL(0), 0);
4358
4359 /* Program the Transmit Control Register */
4360 tctl = rd32(E1000_TCTL);
4361 tctl &= ~E1000_TCTL_CT;
4362 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
4363 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
4364
4365 igb_config_collision_dist(hw);
4366
4367 /* Enable transmits */
4368 tctl |= E1000_TCTL_EN;
4369
4370 wr32(E1000_TCTL, tctl);
4371 }
4372
4373 /**
4374 * igb_configure_tx_ring - Configure transmit ring after Reset
4375 * @adapter: board private structure
4376 * @ring: tx ring to configure
4377 *
4378 * Configure a transmit ring after a reset.
4379 **/
igb_configure_tx_ring(struct igb_adapter * adapter,struct igb_ring * ring)4380 void igb_configure_tx_ring(struct igb_adapter *adapter,
4381 struct igb_ring *ring)
4382 {
4383 struct e1000_hw *hw = &adapter->hw;
4384 u32 txdctl = 0;
4385 u64 tdba = ring->dma;
4386 int reg_idx = ring->reg_idx;
4387
4388 wr32(E1000_TDLEN(reg_idx),
4389 ring->count * sizeof(union e1000_adv_tx_desc));
4390 wr32(E1000_TDBAL(reg_idx),
4391 tdba & 0x00000000ffffffffULL);
4392 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
4393
4394 ring->tail = adapter->io_addr + E1000_TDT(reg_idx);
4395 wr32(E1000_TDH(reg_idx), 0);
4396 writel(0, ring->tail);
4397
4398 txdctl |= IGB_TX_PTHRESH;
4399 txdctl |= IGB_TX_HTHRESH << 8;
4400 txdctl |= IGB_TX_WTHRESH << 16;
4401
4402 /* reinitialize tx_buffer_info */
4403 memset(ring->tx_buffer_info, 0,
4404 sizeof(struct igb_tx_buffer) * ring->count);
4405
4406 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
4407 wr32(E1000_TXDCTL(reg_idx), txdctl);
4408 }
4409
4410 /**
4411 * igb_configure_tx - Configure transmit Unit after Reset
4412 * @adapter: board private structure
4413 *
4414 * Configure the Tx unit of the MAC after a reset.
4415 **/
igb_configure_tx(struct igb_adapter * adapter)4416 static void igb_configure_tx(struct igb_adapter *adapter)
4417 {
4418 struct e1000_hw *hw = &adapter->hw;
4419 int i;
4420
4421 /* disable the queues */
4422 for (i = 0; i < adapter->num_tx_queues; i++)
4423 wr32(E1000_TXDCTL(adapter->tx_ring[i]->reg_idx), 0);
4424
4425 wrfl();
4426 usleep_range(10000, 20000);
4427
4428 for (i = 0; i < adapter->num_tx_queues; i++)
4429 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
4430 }
4431
4432 /**
4433 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
4434 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
4435 *
4436 * Returns 0 on success, negative on failure
4437 **/
igb_setup_rx_resources(struct igb_ring * rx_ring)4438 int igb_setup_rx_resources(struct igb_ring *rx_ring)
4439 {
4440 struct igb_adapter *adapter = netdev_priv(rx_ring->netdev);
4441 struct device *dev = rx_ring->dev;
4442 int size, res;
4443
4444 /* XDP RX-queue info */
4445 if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
4446 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
4447 res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
4448 rx_ring->queue_index, 0);
4449 if (res < 0) {
4450 dev_err(dev, "Failed to register xdp_rxq index %u\n",
4451 rx_ring->queue_index);
4452 return res;
4453 }
4454
4455 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
4456
4457 rx_ring->rx_buffer_info = vmalloc(size);
4458 if (!rx_ring->rx_buffer_info)
4459 goto err;
4460
4461 /* Round up to nearest 4K */
4462 rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc);
4463 rx_ring->size = ALIGN(rx_ring->size, 4096);
4464
4465 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
4466 &rx_ring->dma, GFP_KERNEL);
4467 if (!rx_ring->desc)
4468 goto err;
4469
4470 rx_ring->next_to_alloc = 0;
4471 rx_ring->next_to_clean = 0;
4472 rx_ring->next_to_use = 0;
4473
4474 rx_ring->xdp_prog = adapter->xdp_prog;
4475
4476 return 0;
4477
4478 err:
4479 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
4480 vfree(rx_ring->rx_buffer_info);
4481 rx_ring->rx_buffer_info = NULL;
4482 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
4483 return -ENOMEM;
4484 }
4485
4486 /**
4487 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
4488 * (Descriptors) for all queues
4489 * @adapter: board private structure
4490 *
4491 * Return 0 on success, negative on failure
4492 **/
igb_setup_all_rx_resources(struct igb_adapter * adapter)4493 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
4494 {
4495 struct pci_dev *pdev = adapter->pdev;
4496 int i, err = 0;
4497
4498 for (i = 0; i < adapter->num_rx_queues; i++) {
4499 err = igb_setup_rx_resources(adapter->rx_ring[i]);
4500 if (err) {
4501 dev_err(&pdev->dev,
4502 "Allocation for Rx Queue %u failed\n", i);
4503 for (i--; i >= 0; i--)
4504 igb_free_rx_resources(adapter->rx_ring[i]);
4505 break;
4506 }
4507 }
4508
4509 return err;
4510 }
4511
4512 /**
4513 * igb_setup_mrqc - configure the multiple receive queue control registers
4514 * @adapter: Board private structure
4515 **/
igb_setup_mrqc(struct igb_adapter * adapter)4516 static void igb_setup_mrqc(struct igb_adapter *adapter)
4517 {
4518 struct e1000_hw *hw = &adapter->hw;
4519 u32 mrqc, rxcsum;
4520 u32 j, num_rx_queues;
4521 u32 rss_key[10];
4522
4523 netdev_rss_key_fill(rss_key, sizeof(rss_key));
4524 for (j = 0; j < 10; j++)
4525 wr32(E1000_RSSRK(j), rss_key[j]);
4526
4527 num_rx_queues = adapter->rss_queues;
4528
4529 switch (hw->mac.type) {
4530 case e1000_82576:
4531 /* 82576 supports 2 RSS queues for SR-IOV */
4532 if (adapter->vfs_allocated_count)
4533 num_rx_queues = 2;
4534 break;
4535 default:
4536 break;
4537 }
4538
4539 if (adapter->rss_indir_tbl_init != num_rx_queues) {
4540 for (j = 0; j < IGB_RETA_SIZE; j++)
4541 adapter->rss_indir_tbl[j] =
4542 (j * num_rx_queues) / IGB_RETA_SIZE;
4543 adapter->rss_indir_tbl_init = num_rx_queues;
4544 }
4545 igb_write_rss_indir_tbl(adapter);
4546
4547 /* Disable raw packet checksumming so that RSS hash is placed in
4548 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
4549 * offloads as they are enabled by default
4550 */
4551 rxcsum = rd32(E1000_RXCSUM);
4552 rxcsum |= E1000_RXCSUM_PCSD;
4553
4554 if (adapter->hw.mac.type >= e1000_82576)
4555 /* Enable Receive Checksum Offload for SCTP */
4556 rxcsum |= E1000_RXCSUM_CRCOFL;
4557
4558 /* Don't need to set TUOFL or IPOFL, they default to 1 */
4559 wr32(E1000_RXCSUM, rxcsum);
4560
4561 /* Generate RSS hash based on packet types, TCP/UDP
4562 * port numbers and/or IPv4/v6 src and dst addresses
4563 */
4564 mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
4565 E1000_MRQC_RSS_FIELD_IPV4_TCP |
4566 E1000_MRQC_RSS_FIELD_IPV6 |
4567 E1000_MRQC_RSS_FIELD_IPV6_TCP |
4568 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
4569
4570 if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP)
4571 mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP;
4572 if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP)
4573 mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP;
4574
4575 /* If VMDq is enabled then we set the appropriate mode for that, else
4576 * we default to RSS so that an RSS hash is calculated per packet even
4577 * if we are only using one queue
4578 */
4579 if (adapter->vfs_allocated_count) {
4580 if (hw->mac.type > e1000_82575) {
4581 /* Set the default pool for the PF's first queue */
4582 u32 vtctl = rd32(E1000_VT_CTL);
4583
4584 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
4585 E1000_VT_CTL_DISABLE_DEF_POOL);
4586 vtctl |= adapter->vfs_allocated_count <<
4587 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
4588 wr32(E1000_VT_CTL, vtctl);
4589 }
4590 if (adapter->rss_queues > 1)
4591 mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_MQ;
4592 else
4593 mrqc |= E1000_MRQC_ENABLE_VMDQ;
4594 } else {
4595 mrqc |= E1000_MRQC_ENABLE_RSS_MQ;
4596 }
4597 igb_vmm_control(adapter);
4598
4599 wr32(E1000_MRQC, mrqc);
4600 }
4601
4602 /**
4603 * igb_setup_rctl - configure the receive control registers
4604 * @adapter: Board private structure
4605 **/
igb_setup_rctl(struct igb_adapter * adapter)4606 void igb_setup_rctl(struct igb_adapter *adapter)
4607 {
4608 struct e1000_hw *hw = &adapter->hw;
4609 u32 rctl;
4610
4611 rctl = rd32(E1000_RCTL);
4612
4613 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4614 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
4615
4616 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
4617 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4618
4619 /* enable stripping of CRC. It's unlikely this will break BMC
4620 * redirection as it did with e1000. Newer features require
4621 * that the HW strips the CRC.
4622 */
4623 rctl |= E1000_RCTL_SECRC;
4624
4625 /* disable store bad packets and clear size bits. */
4626 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
4627
4628 /* enable LPE to allow for reception of jumbo frames */
4629 rctl |= E1000_RCTL_LPE;
4630
4631 /* disable queue 0 to prevent tail write w/o re-config */
4632 wr32(E1000_RXDCTL(0), 0);
4633
4634 /* Attention!!! For SR-IOV PF driver operations you must enable
4635 * queue drop for all VF and PF queues to prevent head of line blocking
4636 * if an un-trusted VF does not provide descriptors to hardware.
4637 */
4638 if (adapter->vfs_allocated_count) {
4639 /* set all queue drop enable bits */
4640 wr32(E1000_QDE, ALL_QUEUES);
4641 }
4642
4643 /* This is useful for sniffing bad packets. */
4644 if (adapter->netdev->features & NETIF_F_RXALL) {
4645 /* UPE and MPE will be handled by normal PROMISC logic
4646 * in e1000e_set_rx_mode
4647 */
4648 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
4649 E1000_RCTL_BAM | /* RX All Bcast Pkts */
4650 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
4651
4652 rctl &= ~(E1000_RCTL_DPF | /* Allow filtered pause */
4653 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
4654 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
4655 * and that breaks VLANs.
4656 */
4657 }
4658
4659 wr32(E1000_RCTL, rctl);
4660 }
4661
igb_set_vf_rlpml(struct igb_adapter * adapter,int size,int vfn)4662 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
4663 int vfn)
4664 {
4665 struct e1000_hw *hw = &adapter->hw;
4666 u32 vmolr;
4667
4668 if (size > MAX_JUMBO_FRAME_SIZE)
4669 size = MAX_JUMBO_FRAME_SIZE;
4670
4671 vmolr = rd32(E1000_VMOLR(vfn));
4672 vmolr &= ~E1000_VMOLR_RLPML_MASK;
4673 vmolr |= size | E1000_VMOLR_LPE;
4674 wr32(E1000_VMOLR(vfn), vmolr);
4675
4676 return 0;
4677 }
4678
igb_set_vf_vlan_strip(struct igb_adapter * adapter,int vfn,bool enable)4679 static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter,
4680 int vfn, bool enable)
4681 {
4682 struct e1000_hw *hw = &adapter->hw;
4683 u32 val, reg;
4684
4685 if (hw->mac.type < e1000_82576)
4686 return;
4687
4688 if (hw->mac.type == e1000_i350)
4689 reg = E1000_DVMOLR(vfn);
4690 else
4691 reg = E1000_VMOLR(vfn);
4692
4693 val = rd32(reg);
4694 if (enable)
4695 val |= E1000_VMOLR_STRVLAN;
4696 else
4697 val &= ~(E1000_VMOLR_STRVLAN);
4698 wr32(reg, val);
4699 }
4700
igb_set_vmolr(struct igb_adapter * adapter,int vfn,bool aupe)4701 static inline void igb_set_vmolr(struct igb_adapter *adapter,
4702 int vfn, bool aupe)
4703 {
4704 struct e1000_hw *hw = &adapter->hw;
4705 u32 vmolr;
4706
4707 /* This register exists only on 82576 and newer so if we are older then
4708 * we should exit and do nothing
4709 */
4710 if (hw->mac.type < e1000_82576)
4711 return;
4712
4713 vmolr = rd32(E1000_VMOLR(vfn));
4714 if (aupe)
4715 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
4716 else
4717 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
4718
4719 /* clear all bits that might not be set */
4720 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
4721
4722 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
4723 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
4724 /* for VMDq only allow the VFs and pool 0 to accept broadcast and
4725 * multicast packets
4726 */
4727 if (vfn <= adapter->vfs_allocated_count)
4728 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
4729
4730 wr32(E1000_VMOLR(vfn), vmolr);
4731 }
4732
4733 /**
4734 * igb_setup_srrctl - configure the split and replication receive control
4735 * registers
4736 * @adapter: Board private structure
4737 * @ring: receive ring to be configured
4738 **/
igb_setup_srrctl(struct igb_adapter * adapter,struct igb_ring * ring)4739 void igb_setup_srrctl(struct igb_adapter *adapter, struct igb_ring *ring)
4740 {
4741 struct e1000_hw *hw = &adapter->hw;
4742 int reg_idx = ring->reg_idx;
4743 u32 srrctl = 0;
4744
4745 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
4746 if (ring_uses_large_buffer(ring))
4747 srrctl |= IGB_RXBUFFER_3072 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4748 else
4749 srrctl |= IGB_RXBUFFER_2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4750 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
4751 if (hw->mac.type >= e1000_82580)
4752 srrctl |= E1000_SRRCTL_TIMESTAMP;
4753 /* Only set Drop Enable if VFs allocated, or we are supporting multiple
4754 * queues and rx flow control is disabled
4755 */
4756 if (adapter->vfs_allocated_count ||
4757 (!(hw->fc.current_mode & e1000_fc_rx_pause) &&
4758 adapter->num_rx_queues > 1))
4759 srrctl |= E1000_SRRCTL_DROP_EN;
4760
4761 wr32(E1000_SRRCTL(reg_idx), srrctl);
4762 }
4763
4764 /**
4765 * igb_configure_rx_ring - Configure a receive ring after Reset
4766 * @adapter: board private structure
4767 * @ring: receive ring to be configured
4768 *
4769 * Configure the Rx unit of the MAC after a reset.
4770 **/
igb_configure_rx_ring(struct igb_adapter * adapter,struct igb_ring * ring)4771 void igb_configure_rx_ring(struct igb_adapter *adapter,
4772 struct igb_ring *ring)
4773 {
4774 struct e1000_hw *hw = &adapter->hw;
4775 union e1000_adv_rx_desc *rx_desc;
4776 u64 rdba = ring->dma;
4777 int reg_idx = ring->reg_idx;
4778 u32 rxdctl = 0;
4779
4780 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
4781 WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
4782 MEM_TYPE_PAGE_SHARED, NULL));
4783
4784 /* disable the queue */
4785 wr32(E1000_RXDCTL(reg_idx), 0);
4786
4787 /* Set DMA base address registers */
4788 wr32(E1000_RDBAL(reg_idx),
4789 rdba & 0x00000000ffffffffULL);
4790 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
4791 wr32(E1000_RDLEN(reg_idx),
4792 ring->count * sizeof(union e1000_adv_rx_desc));
4793
4794 /* initialize head and tail */
4795 ring->tail = adapter->io_addr + E1000_RDT(reg_idx);
4796 wr32(E1000_RDH(reg_idx), 0);
4797 writel(0, ring->tail);
4798
4799 /* set descriptor configuration */
4800 igb_setup_srrctl(adapter, ring);
4801
4802 /* set filtering for VMDQ pools */
4803 igb_set_vmolr(adapter, reg_idx & 0x7, true);
4804
4805 rxdctl |= IGB_RX_PTHRESH;
4806 rxdctl |= IGB_RX_HTHRESH << 8;
4807 rxdctl |= IGB_RX_WTHRESH << 16;
4808
4809 /* initialize rx_buffer_info */
4810 memset(ring->rx_buffer_info, 0,
4811 sizeof(struct igb_rx_buffer) * ring->count);
4812
4813 /* initialize Rx descriptor 0 */
4814 rx_desc = IGB_RX_DESC(ring, 0);
4815 rx_desc->wb.upper.length = 0;
4816
4817 /* enable receive descriptor fetching */
4818 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
4819 wr32(E1000_RXDCTL(reg_idx), rxdctl);
4820 }
4821
igb_set_rx_buffer_len(struct igb_adapter * adapter,struct igb_ring * rx_ring)4822 static void igb_set_rx_buffer_len(struct igb_adapter *adapter,
4823 struct igb_ring *rx_ring)
4824 {
4825 #if (PAGE_SIZE < 8192)
4826 struct e1000_hw *hw = &adapter->hw;
4827 #endif
4828
4829 /* set build_skb and buffer size flags */
4830 clear_ring_build_skb_enabled(rx_ring);
4831 clear_ring_uses_large_buffer(rx_ring);
4832
4833 if (adapter->flags & IGB_FLAG_RX_LEGACY)
4834 return;
4835
4836 set_ring_build_skb_enabled(rx_ring);
4837
4838 #if (PAGE_SIZE < 8192)
4839 if (adapter->max_frame_size > IGB_MAX_FRAME_BUILD_SKB ||
4840 IGB_2K_TOO_SMALL_WITH_PADDING ||
4841 rd32(E1000_RCTL) & E1000_RCTL_SBP)
4842 set_ring_uses_large_buffer(rx_ring);
4843 #endif
4844 }
4845
4846 /**
4847 * igb_configure_rx - Configure receive Unit after Reset
4848 * @adapter: board private structure
4849 *
4850 * Configure the Rx unit of the MAC after a reset.
4851 **/
igb_configure_rx(struct igb_adapter * adapter)4852 static void igb_configure_rx(struct igb_adapter *adapter)
4853 {
4854 int i;
4855
4856 /* set the correct pool for the PF default MAC address in entry 0 */
4857 igb_set_default_mac_filter(adapter);
4858
4859 /* Setup the HW Rx Head and Tail Descriptor Pointers and
4860 * the Base and Length of the Rx Descriptor Ring
4861 */
4862 for (i = 0; i < adapter->num_rx_queues; i++) {
4863 struct igb_ring *rx_ring = adapter->rx_ring[i];
4864
4865 igb_set_rx_buffer_len(adapter, rx_ring);
4866 igb_configure_rx_ring(adapter, rx_ring);
4867 }
4868 }
4869
4870 /**
4871 * igb_free_tx_resources - Free Tx Resources per Queue
4872 * @tx_ring: Tx descriptor ring for a specific queue
4873 *
4874 * Free all transmit software resources
4875 **/
igb_free_tx_resources(struct igb_ring * tx_ring)4876 void igb_free_tx_resources(struct igb_ring *tx_ring)
4877 {
4878 igb_clean_tx_ring(tx_ring);
4879
4880 vfree(tx_ring->tx_buffer_info);
4881 tx_ring->tx_buffer_info = NULL;
4882
4883 /* if not set, then don't free */
4884 if (!tx_ring->desc)
4885 return;
4886
4887 dma_free_coherent(tx_ring->dev, tx_ring->size,
4888 tx_ring->desc, tx_ring->dma);
4889
4890 tx_ring->desc = NULL;
4891 }
4892
4893 /**
4894 * igb_free_all_tx_resources - Free Tx Resources for All Queues
4895 * @adapter: board private structure
4896 *
4897 * Free all transmit software resources
4898 **/
igb_free_all_tx_resources(struct igb_adapter * adapter)4899 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
4900 {
4901 int i;
4902
4903 for (i = 0; i < adapter->num_tx_queues; i++)
4904 if (adapter->tx_ring[i])
4905 igb_free_tx_resources(adapter->tx_ring[i]);
4906 }
4907
4908 /**
4909 * igb_clean_tx_ring - Free Tx Buffers
4910 * @tx_ring: ring to be cleaned
4911 **/
igb_clean_tx_ring(struct igb_ring * tx_ring)4912 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
4913 {
4914 u16 i = tx_ring->next_to_clean;
4915 struct igb_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
4916
4917 while (i != tx_ring->next_to_use) {
4918 union e1000_adv_tx_desc *eop_desc, *tx_desc;
4919
4920 /* Free all the Tx ring sk_buffs or xdp frames */
4921 if (tx_buffer->type == IGB_TYPE_SKB)
4922 dev_kfree_skb_any(tx_buffer->skb);
4923 else
4924 xdp_return_frame(tx_buffer->xdpf);
4925
4926 /* unmap skb header data */
4927 dma_unmap_single(tx_ring->dev,
4928 dma_unmap_addr(tx_buffer, dma),
4929 dma_unmap_len(tx_buffer, len),
4930 DMA_TO_DEVICE);
4931
4932 /* check for eop_desc to determine the end of the packet */
4933 eop_desc = tx_buffer->next_to_watch;
4934 tx_desc = IGB_TX_DESC(tx_ring, i);
4935
4936 /* unmap remaining buffers */
4937 while (tx_desc != eop_desc) {
4938 tx_buffer++;
4939 tx_desc++;
4940 i++;
4941 if (unlikely(i == tx_ring->count)) {
4942 i = 0;
4943 tx_buffer = tx_ring->tx_buffer_info;
4944 tx_desc = IGB_TX_DESC(tx_ring, 0);
4945 }
4946
4947 /* unmap any remaining paged data */
4948 if (dma_unmap_len(tx_buffer, len))
4949 dma_unmap_page(tx_ring->dev,
4950 dma_unmap_addr(tx_buffer, dma),
4951 dma_unmap_len(tx_buffer, len),
4952 DMA_TO_DEVICE);
4953 }
4954
4955 tx_buffer->next_to_watch = NULL;
4956
4957 /* move us one more past the eop_desc for start of next pkt */
4958 tx_buffer++;
4959 i++;
4960 if (unlikely(i == tx_ring->count)) {
4961 i = 0;
4962 tx_buffer = tx_ring->tx_buffer_info;
4963 }
4964 }
4965
4966 /* reset BQL for queue */
4967 netdev_tx_reset_queue(txring_txq(tx_ring));
4968
4969 /* reset next_to_use and next_to_clean */
4970 tx_ring->next_to_use = 0;
4971 tx_ring->next_to_clean = 0;
4972 }
4973
4974 /**
4975 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
4976 * @adapter: board private structure
4977 **/
igb_clean_all_tx_rings(struct igb_adapter * adapter)4978 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
4979 {
4980 int i;
4981
4982 for (i = 0; i < adapter->num_tx_queues; i++)
4983 if (adapter->tx_ring[i])
4984 igb_clean_tx_ring(adapter->tx_ring[i]);
4985 }
4986
4987 /**
4988 * igb_free_rx_resources - Free Rx Resources
4989 * @rx_ring: ring to clean the resources from
4990 *
4991 * Free all receive software resources
4992 **/
igb_free_rx_resources(struct igb_ring * rx_ring)4993 void igb_free_rx_resources(struct igb_ring *rx_ring)
4994 {
4995 igb_clean_rx_ring(rx_ring);
4996
4997 rx_ring->xdp_prog = NULL;
4998 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
4999 vfree(rx_ring->rx_buffer_info);
5000 rx_ring->rx_buffer_info = NULL;
5001
5002 /* if not set, then don't free */
5003 if (!rx_ring->desc)
5004 return;
5005
5006 dma_free_coherent(rx_ring->dev, rx_ring->size,
5007 rx_ring->desc, rx_ring->dma);
5008
5009 rx_ring->desc = NULL;
5010 }
5011
5012 /**
5013 * igb_free_all_rx_resources - Free Rx Resources for All Queues
5014 * @adapter: board private structure
5015 *
5016 * Free all receive software resources
5017 **/
igb_free_all_rx_resources(struct igb_adapter * adapter)5018 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
5019 {
5020 int i;
5021
5022 for (i = 0; i < adapter->num_rx_queues; i++)
5023 if (adapter->rx_ring[i])
5024 igb_free_rx_resources(adapter->rx_ring[i]);
5025 }
5026
5027 /**
5028 * igb_clean_rx_ring - Free Rx Buffers per Queue
5029 * @rx_ring: ring to free buffers from
5030 **/
igb_clean_rx_ring(struct igb_ring * rx_ring)5031 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
5032 {
5033 u16 i = rx_ring->next_to_clean;
5034
5035 dev_kfree_skb(rx_ring->skb);
5036 rx_ring->skb = NULL;
5037
5038 /* Free all the Rx ring sk_buffs */
5039 while (i != rx_ring->next_to_alloc) {
5040 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
5041
5042 /* Invalidate cache lines that may have been written to by
5043 * device so that we avoid corrupting memory.
5044 */
5045 dma_sync_single_range_for_cpu(rx_ring->dev,
5046 buffer_info->dma,
5047 buffer_info->page_offset,
5048 igb_rx_bufsz(rx_ring),
5049 DMA_FROM_DEVICE);
5050
5051 /* free resources associated with mapping */
5052 dma_unmap_page_attrs(rx_ring->dev,
5053 buffer_info->dma,
5054 igb_rx_pg_size(rx_ring),
5055 DMA_FROM_DEVICE,
5056 IGB_RX_DMA_ATTR);
5057 __page_frag_cache_drain(buffer_info->page,
5058 buffer_info->pagecnt_bias);
5059
5060 i++;
5061 if (i == rx_ring->count)
5062 i = 0;
5063 }
5064
5065 rx_ring->next_to_alloc = 0;
5066 rx_ring->next_to_clean = 0;
5067 rx_ring->next_to_use = 0;
5068 }
5069
5070 /**
5071 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
5072 * @adapter: board private structure
5073 **/
igb_clean_all_rx_rings(struct igb_adapter * adapter)5074 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
5075 {
5076 int i;
5077
5078 for (i = 0; i < adapter->num_rx_queues; i++)
5079 if (adapter->rx_ring[i])
5080 igb_clean_rx_ring(adapter->rx_ring[i]);
5081 }
5082
5083 /**
5084 * igb_set_mac - Change the Ethernet Address of the NIC
5085 * @netdev: network interface device structure
5086 * @p: pointer to an address structure
5087 *
5088 * Returns 0 on success, negative on failure
5089 **/
igb_set_mac(struct net_device * netdev,void * p)5090 static int igb_set_mac(struct net_device *netdev, void *p)
5091 {
5092 struct igb_adapter *adapter = netdev_priv(netdev);
5093 struct e1000_hw *hw = &adapter->hw;
5094 struct sockaddr *addr = p;
5095
5096 if (!is_valid_ether_addr(addr->sa_data))
5097 return -EADDRNOTAVAIL;
5098
5099 eth_hw_addr_set(netdev, addr->sa_data);
5100 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
5101
5102 /* set the correct pool for the new PF MAC address in entry 0 */
5103 igb_set_default_mac_filter(adapter);
5104
5105 return 0;
5106 }
5107
5108 /**
5109 * igb_write_mc_addr_list - write multicast addresses to MTA
5110 * @netdev: network interface device structure
5111 *
5112 * Writes multicast address list to the MTA hash table.
5113 * Returns: -ENOMEM on failure
5114 * 0 on no addresses written
5115 * X on writing X addresses to MTA
5116 **/
igb_write_mc_addr_list(struct net_device * netdev)5117 static int igb_write_mc_addr_list(struct net_device *netdev)
5118 {
5119 struct igb_adapter *adapter = netdev_priv(netdev);
5120 struct e1000_hw *hw = &adapter->hw;
5121 struct netdev_hw_addr *ha;
5122 u8 *mta_list;
5123 int i;
5124
5125 if (netdev_mc_empty(netdev)) {
5126 /* nothing to program, so clear mc list */
5127 igb_update_mc_addr_list(hw, NULL, 0);
5128 igb_restore_vf_multicasts(adapter);
5129 return 0;
5130 }
5131
5132 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
5133 if (!mta_list)
5134 return -ENOMEM;
5135
5136 /* The shared function expects a packed array of only addresses. */
5137 i = 0;
5138 netdev_for_each_mc_addr(ha, netdev)
5139 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
5140
5141 igb_update_mc_addr_list(hw, mta_list, i);
5142 kfree(mta_list);
5143
5144 return netdev_mc_count(netdev);
5145 }
5146
igb_vlan_promisc_enable(struct igb_adapter * adapter)5147 static int igb_vlan_promisc_enable(struct igb_adapter *adapter)
5148 {
5149 struct e1000_hw *hw = &adapter->hw;
5150 u32 i, pf_id;
5151
5152 switch (hw->mac.type) {
5153 case e1000_i210:
5154 case e1000_i211:
5155 case e1000_i350:
5156 /* VLAN filtering needed for VLAN prio filter */
5157 if (adapter->netdev->features & NETIF_F_NTUPLE)
5158 break;
5159 fallthrough;
5160 case e1000_82576:
5161 case e1000_82580:
5162 case e1000_i354:
5163 /* VLAN filtering needed for pool filtering */
5164 if (adapter->vfs_allocated_count)
5165 break;
5166 fallthrough;
5167 default:
5168 return 1;
5169 }
5170
5171 /* We are already in VLAN promisc, nothing to do */
5172 if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
5173 return 0;
5174
5175 if (!adapter->vfs_allocated_count)
5176 goto set_vfta;
5177
5178 /* Add PF to all active pools */
5179 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
5180
5181 for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
5182 u32 vlvf = rd32(E1000_VLVF(i));
5183
5184 vlvf |= BIT(pf_id);
5185 wr32(E1000_VLVF(i), vlvf);
5186 }
5187
5188 set_vfta:
5189 /* Set all bits in the VLAN filter table array */
5190 for (i = E1000_VLAN_FILTER_TBL_SIZE; i--;)
5191 hw->mac.ops.write_vfta(hw, i, ~0U);
5192
5193 /* Set flag so we don't redo unnecessary work */
5194 adapter->flags |= IGB_FLAG_VLAN_PROMISC;
5195
5196 return 0;
5197 }
5198
5199 #define VFTA_BLOCK_SIZE 8
igb_scrub_vfta(struct igb_adapter * adapter,u32 vfta_offset)5200 static void igb_scrub_vfta(struct igb_adapter *adapter, u32 vfta_offset)
5201 {
5202 struct e1000_hw *hw = &adapter->hw;
5203 u32 vfta[VFTA_BLOCK_SIZE] = { 0 };
5204 u32 vid_start = vfta_offset * 32;
5205 u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32);
5206 u32 i, vid, word, bits, pf_id;
5207
5208 /* guarantee that we don't scrub out management VLAN */
5209 vid = adapter->mng_vlan_id;
5210 if (vid >= vid_start && vid < vid_end)
5211 vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
5212
5213 if (!adapter->vfs_allocated_count)
5214 goto set_vfta;
5215
5216 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
5217
5218 for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
5219 u32 vlvf = rd32(E1000_VLVF(i));
5220
5221 /* pull VLAN ID from VLVF */
5222 vid = vlvf & VLAN_VID_MASK;
5223
5224 /* only concern ourselves with a certain range */
5225 if (vid < vid_start || vid >= vid_end)
5226 continue;
5227
5228 if (vlvf & E1000_VLVF_VLANID_ENABLE) {
5229 /* record VLAN ID in VFTA */
5230 vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
5231
5232 /* if PF is part of this then continue */
5233 if (test_bit(vid, adapter->active_vlans))
5234 continue;
5235 }
5236
5237 /* remove PF from the pool */
5238 bits = ~BIT(pf_id);
5239 bits &= rd32(E1000_VLVF(i));
5240 wr32(E1000_VLVF(i), bits);
5241 }
5242
5243 set_vfta:
5244 /* extract values from active_vlans and write back to VFTA */
5245 for (i = VFTA_BLOCK_SIZE; i--;) {
5246 vid = (vfta_offset + i) * 32;
5247 word = vid / BITS_PER_LONG;
5248 bits = vid % BITS_PER_LONG;
5249
5250 vfta[i] |= adapter->active_vlans[word] >> bits;
5251
5252 hw->mac.ops.write_vfta(hw, vfta_offset + i, vfta[i]);
5253 }
5254 }
5255
igb_vlan_promisc_disable(struct igb_adapter * adapter)5256 static void igb_vlan_promisc_disable(struct igb_adapter *adapter)
5257 {
5258 u32 i;
5259
5260 /* We are not in VLAN promisc, nothing to do */
5261 if (!(adapter->flags & IGB_FLAG_VLAN_PROMISC))
5262 return;
5263
5264 /* Set flag so we don't redo unnecessary work */
5265 adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
5266
5267 for (i = 0; i < E1000_VLAN_FILTER_TBL_SIZE; i += VFTA_BLOCK_SIZE)
5268 igb_scrub_vfta(adapter, i);
5269 }
5270
5271 /**
5272 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
5273 * @netdev: network interface device structure
5274 *
5275 * The set_rx_mode entry point is called whenever the unicast or multicast
5276 * address lists or the network interface flags are updated. This routine is
5277 * responsible for configuring the hardware for proper unicast, multicast,
5278 * promiscuous mode, and all-multi behavior.
5279 **/
igb_set_rx_mode(struct net_device * netdev)5280 static void igb_set_rx_mode(struct net_device *netdev)
5281 {
5282 struct igb_adapter *adapter = netdev_priv(netdev);
5283 struct e1000_hw *hw = &adapter->hw;
5284 unsigned int vfn = adapter->vfs_allocated_count;
5285 u32 rctl = 0, vmolr = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
5286 int count;
5287
5288 /* Check for Promiscuous and All Multicast modes */
5289 if (netdev->flags & IFF_PROMISC) {
5290 rctl |= E1000_RCTL_UPE | E1000_RCTL_MPE;
5291 vmolr |= E1000_VMOLR_MPME;
5292
5293 /* enable use of UTA filter to force packets to default pool */
5294 if (hw->mac.type == e1000_82576)
5295 vmolr |= E1000_VMOLR_ROPE;
5296 } else {
5297 if (netdev->flags & IFF_ALLMULTI) {
5298 rctl |= E1000_RCTL_MPE;
5299 vmolr |= E1000_VMOLR_MPME;
5300 } else {
5301 /* Write addresses to the MTA, if the attempt fails
5302 * then we should just turn on promiscuous mode so
5303 * that we can at least receive multicast traffic
5304 */
5305 count = igb_write_mc_addr_list(netdev);
5306 if (count < 0) {
5307 rctl |= E1000_RCTL_MPE;
5308 vmolr |= E1000_VMOLR_MPME;
5309 } else if (count) {
5310 vmolr |= E1000_VMOLR_ROMPE;
5311 }
5312 }
5313 }
5314
5315 /* Write addresses to available RAR registers, if there is not
5316 * sufficient space to store all the addresses then enable
5317 * unicast promiscuous mode
5318 */
5319 if (__dev_uc_sync(netdev, igb_uc_sync, igb_uc_unsync)) {
5320 rctl |= E1000_RCTL_UPE;
5321 vmolr |= E1000_VMOLR_ROPE;
5322 }
5323
5324 /* enable VLAN filtering by default */
5325 rctl |= E1000_RCTL_VFE;
5326
5327 /* disable VLAN filtering for modes that require it */
5328 if ((netdev->flags & IFF_PROMISC) ||
5329 (netdev->features & NETIF_F_RXALL)) {
5330 /* if we fail to set all rules then just clear VFE */
5331 if (igb_vlan_promisc_enable(adapter))
5332 rctl &= ~E1000_RCTL_VFE;
5333 } else {
5334 igb_vlan_promisc_disable(adapter);
5335 }
5336
5337 /* update state of unicast, multicast, and VLAN filtering modes */
5338 rctl |= rd32(E1000_RCTL) & ~(E1000_RCTL_UPE | E1000_RCTL_MPE |
5339 E1000_RCTL_VFE);
5340 wr32(E1000_RCTL, rctl);
5341
5342 #if (PAGE_SIZE < 8192)
5343 if (!adapter->vfs_allocated_count) {
5344 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
5345 rlpml = IGB_MAX_FRAME_BUILD_SKB;
5346 }
5347 #endif
5348 wr32(E1000_RLPML, rlpml);
5349
5350 /* In order to support SR-IOV and eventually VMDq it is necessary to set
5351 * the VMOLR to enable the appropriate modes. Without this workaround
5352 * we will have issues with VLAN tag stripping not being done for frames
5353 * that are only arriving because we are the default pool
5354 */
5355 if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
5356 return;
5357
5358 /* set UTA to appropriate mode */
5359 igb_set_uta(adapter, !!(vmolr & E1000_VMOLR_ROPE));
5360
5361 vmolr |= rd32(E1000_VMOLR(vfn)) &
5362 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
5363
5364 /* enable Rx jumbo frames, restrict as needed to support build_skb */
5365 vmolr &= ~E1000_VMOLR_RLPML_MASK;
5366 #if (PAGE_SIZE < 8192)
5367 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
5368 vmolr |= IGB_MAX_FRAME_BUILD_SKB;
5369 else
5370 #endif
5371 vmolr |= MAX_JUMBO_FRAME_SIZE;
5372 vmolr |= E1000_VMOLR_LPE;
5373
5374 wr32(E1000_VMOLR(vfn), vmolr);
5375
5376 igb_restore_vf_multicasts(adapter);
5377 }
5378
igb_check_wvbr(struct igb_adapter * adapter)5379 static void igb_check_wvbr(struct igb_adapter *adapter)
5380 {
5381 struct e1000_hw *hw = &adapter->hw;
5382 u32 wvbr = 0;
5383
5384 switch (hw->mac.type) {
5385 case e1000_82576:
5386 case e1000_i350:
5387 wvbr = rd32(E1000_WVBR);
5388 if (!wvbr)
5389 return;
5390 break;
5391 default:
5392 break;
5393 }
5394
5395 adapter->wvbr |= wvbr;
5396 }
5397
5398 #define IGB_STAGGERED_QUEUE_OFFSET 8
5399
igb_spoof_check(struct igb_adapter * adapter)5400 static void igb_spoof_check(struct igb_adapter *adapter)
5401 {
5402 int j;
5403
5404 if (!adapter->wvbr)
5405 return;
5406
5407 for (j = 0; j < adapter->vfs_allocated_count; j++) {
5408 if (adapter->wvbr & BIT(j) ||
5409 adapter->wvbr & BIT(j + IGB_STAGGERED_QUEUE_OFFSET)) {
5410 dev_warn(&adapter->pdev->dev,
5411 "Spoof event(s) detected on VF %d\n", j);
5412 adapter->wvbr &=
5413 ~(BIT(j) |
5414 BIT(j + IGB_STAGGERED_QUEUE_OFFSET));
5415 }
5416 }
5417 }
5418
5419 /* Need to wait a few seconds after link up to get diagnostic information from
5420 * the phy
5421 */
igb_update_phy_info(struct timer_list * t)5422 static void igb_update_phy_info(struct timer_list *t)
5423 {
5424 struct igb_adapter *adapter = from_timer(adapter, t, phy_info_timer);
5425 igb_get_phy_info(&adapter->hw);
5426 }
5427
5428 /**
5429 * igb_has_link - check shared code for link and determine up/down
5430 * @adapter: pointer to driver private info
5431 **/
igb_has_link(struct igb_adapter * adapter)5432 bool igb_has_link(struct igb_adapter *adapter)
5433 {
5434 struct e1000_hw *hw = &adapter->hw;
5435 bool link_active = false;
5436
5437 /* get_link_status is set on LSC (link status) interrupt or
5438 * rx sequence error interrupt. get_link_status will stay
5439 * false until the e1000_check_for_link establishes link
5440 * for copper adapters ONLY
5441 */
5442 switch (hw->phy.media_type) {
5443 case e1000_media_type_copper:
5444 if (!hw->mac.get_link_status)
5445 return true;
5446 fallthrough;
5447 case e1000_media_type_internal_serdes:
5448 hw->mac.ops.check_for_link(hw);
5449 link_active = !hw->mac.get_link_status;
5450 break;
5451 default:
5452 case e1000_media_type_unknown:
5453 break;
5454 }
5455
5456 if (((hw->mac.type == e1000_i210) ||
5457 (hw->mac.type == e1000_i211)) &&
5458 (hw->phy.id == I210_I_PHY_ID)) {
5459 if (!netif_carrier_ok(adapter->netdev)) {
5460 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
5461 } else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) {
5462 adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE;
5463 adapter->link_check_timeout = jiffies;
5464 }
5465 }
5466
5467 return link_active;
5468 }
5469
igb_thermal_sensor_event(struct e1000_hw * hw,u32 event)5470 static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
5471 {
5472 bool ret = false;
5473 u32 ctrl_ext, thstat;
5474
5475 /* check for thermal sensor event on i350 copper only */
5476 if (hw->mac.type == e1000_i350) {
5477 thstat = rd32(E1000_THSTAT);
5478 ctrl_ext = rd32(E1000_CTRL_EXT);
5479
5480 if ((hw->phy.media_type == e1000_media_type_copper) &&
5481 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII))
5482 ret = !!(thstat & event);
5483 }
5484
5485 return ret;
5486 }
5487
5488 /**
5489 * igb_check_lvmmc - check for malformed packets received
5490 * and indicated in LVMMC register
5491 * @adapter: pointer to adapter
5492 **/
igb_check_lvmmc(struct igb_adapter * adapter)5493 static void igb_check_lvmmc(struct igb_adapter *adapter)
5494 {
5495 struct e1000_hw *hw = &adapter->hw;
5496 u32 lvmmc;
5497
5498 lvmmc = rd32(E1000_LVMMC);
5499 if (lvmmc) {
5500 if (unlikely(net_ratelimit())) {
5501 netdev_warn(adapter->netdev,
5502 "malformed Tx packet detected and dropped, LVMMC:0x%08x\n",
5503 lvmmc);
5504 }
5505 }
5506 }
5507
5508 /**
5509 * igb_watchdog - Timer Call-back
5510 * @t: pointer to timer_list containing our private info pointer
5511 **/
igb_watchdog(struct timer_list * t)5512 static void igb_watchdog(struct timer_list *t)
5513 {
5514 struct igb_adapter *adapter = from_timer(adapter, t, watchdog_timer);
5515 /* Do the rest outside of interrupt context */
5516 schedule_work(&adapter->watchdog_task);
5517 }
5518
igb_watchdog_task(struct work_struct * work)5519 static void igb_watchdog_task(struct work_struct *work)
5520 {
5521 struct igb_adapter *adapter = container_of(work,
5522 struct igb_adapter,
5523 watchdog_task);
5524 struct e1000_hw *hw = &adapter->hw;
5525 struct e1000_phy_info *phy = &hw->phy;
5526 struct net_device *netdev = adapter->netdev;
5527 u32 link;
5528 int i;
5529 u32 connsw;
5530 u16 phy_data, retry_count = 20;
5531
5532 link = igb_has_link(adapter);
5533
5534 if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) {
5535 if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
5536 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
5537 else
5538 link = false;
5539 }
5540
5541 /* Force link down if we have fiber to swap to */
5542 if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
5543 if (hw->phy.media_type == e1000_media_type_copper) {
5544 connsw = rd32(E1000_CONNSW);
5545 if (!(connsw & E1000_CONNSW_AUTOSENSE_EN))
5546 link = 0;
5547 }
5548 }
5549 if (link) {
5550 /* Perform a reset if the media type changed. */
5551 if (hw->dev_spec._82575.media_changed) {
5552 hw->dev_spec._82575.media_changed = false;
5553 adapter->flags |= IGB_FLAG_MEDIA_RESET;
5554 igb_reset(adapter);
5555 }
5556 /* Cancel scheduled suspend requests. */
5557 pm_runtime_resume(netdev->dev.parent);
5558
5559 if (!netif_carrier_ok(netdev)) {
5560 u32 ctrl;
5561
5562 hw->mac.ops.get_speed_and_duplex(hw,
5563 &adapter->link_speed,
5564 &adapter->link_duplex);
5565
5566 ctrl = rd32(E1000_CTRL);
5567 /* Links status message must follow this format */
5568 netdev_info(netdev,
5569 "igb: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
5570 netdev->name,
5571 adapter->link_speed,
5572 adapter->link_duplex == FULL_DUPLEX ?
5573 "Full" : "Half",
5574 (ctrl & E1000_CTRL_TFCE) &&
5575 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
5576 (ctrl & E1000_CTRL_RFCE) ? "RX" :
5577 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None");
5578
5579 /* disable EEE if enabled */
5580 if ((adapter->flags & IGB_FLAG_EEE) &&
5581 (adapter->link_duplex == HALF_DUPLEX)) {
5582 dev_info(&adapter->pdev->dev,
5583 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex.\n");
5584 adapter->hw.dev_spec._82575.eee_disable = true;
5585 adapter->flags &= ~IGB_FLAG_EEE;
5586 }
5587
5588 /* check if SmartSpeed worked */
5589 igb_check_downshift(hw);
5590 if (phy->speed_downgraded)
5591 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
5592
5593 /* check for thermal sensor event */
5594 if (igb_thermal_sensor_event(hw,
5595 E1000_THSTAT_LINK_THROTTLE))
5596 netdev_info(netdev, "The network adapter link speed was downshifted because it overheated\n");
5597
5598 /* adjust timeout factor according to speed/duplex */
5599 adapter->tx_timeout_factor = 1;
5600 switch (adapter->link_speed) {
5601 case SPEED_10:
5602 adapter->tx_timeout_factor = 14;
5603 break;
5604 case SPEED_100:
5605 /* maybe add some timeout factor ? */
5606 break;
5607 }
5608
5609 if (adapter->link_speed != SPEED_1000 ||
5610 !hw->phy.ops.read_reg)
5611 goto no_wait;
5612
5613 /* wait for Remote receiver status OK */
5614 retry_read_status:
5615 if (!igb_read_phy_reg(hw, PHY_1000T_STATUS,
5616 &phy_data)) {
5617 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
5618 retry_count) {
5619 msleep(100);
5620 retry_count--;
5621 goto retry_read_status;
5622 } else if (!retry_count) {
5623 dev_err(&adapter->pdev->dev, "exceed max 2 second\n");
5624 }
5625 } else {
5626 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n");
5627 }
5628 no_wait:
5629 netif_carrier_on(netdev);
5630
5631 igb_ping_all_vfs(adapter);
5632 igb_check_vf_rate_limit(adapter);
5633
5634 /* link state has changed, schedule phy info update */
5635 if (!test_bit(__IGB_DOWN, &adapter->state))
5636 mod_timer(&adapter->phy_info_timer,
5637 round_jiffies(jiffies + 2 * HZ));
5638 }
5639 } else {
5640 if (netif_carrier_ok(netdev)) {
5641 adapter->link_speed = 0;
5642 adapter->link_duplex = 0;
5643
5644 /* check for thermal sensor event */
5645 if (igb_thermal_sensor_event(hw,
5646 E1000_THSTAT_PWR_DOWN)) {
5647 netdev_err(netdev, "The network adapter was stopped because it overheated\n");
5648 }
5649
5650 /* Links status message must follow this format */
5651 netdev_info(netdev, "igb: %s NIC Link is Down\n",
5652 netdev->name);
5653 netif_carrier_off(netdev);
5654
5655 igb_ping_all_vfs(adapter);
5656
5657 /* link state has changed, schedule phy info update */
5658 if (!test_bit(__IGB_DOWN, &adapter->state))
5659 mod_timer(&adapter->phy_info_timer,
5660 round_jiffies(jiffies + 2 * HZ));
5661
5662 /* link is down, time to check for alternate media */
5663 if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
5664 igb_check_swap_media(adapter);
5665 if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
5666 schedule_work(&adapter->reset_task);
5667 /* return immediately */
5668 return;
5669 }
5670 }
5671 pm_schedule_suspend(netdev->dev.parent,
5672 MSEC_PER_SEC * 5);
5673
5674 /* also check for alternate media here */
5675 } else if (!netif_carrier_ok(netdev) &&
5676 (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
5677 igb_check_swap_media(adapter);
5678 if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
5679 schedule_work(&adapter->reset_task);
5680 /* return immediately */
5681 return;
5682 }
5683 }
5684 }
5685
5686 spin_lock(&adapter->stats64_lock);
5687 igb_update_stats(adapter);
5688 spin_unlock(&adapter->stats64_lock);
5689
5690 for (i = 0; i < adapter->num_tx_queues; i++) {
5691 struct igb_ring *tx_ring = adapter->tx_ring[i];
5692 if (!netif_carrier_ok(netdev)) {
5693 /* We've lost link, so the controller stops DMA,
5694 * but we've got queued Tx work that's never going
5695 * to get done, so reset controller to flush Tx.
5696 * (Do the reset outside of interrupt context).
5697 */
5698 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
5699 adapter->tx_timeout_count++;
5700 schedule_work(&adapter->reset_task);
5701 /* return immediately since reset is imminent */
5702 return;
5703 }
5704 }
5705
5706 /* Force detection of hung controller every watchdog period */
5707 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5708 }
5709
5710 /* Cause software interrupt to ensure Rx ring is cleaned */
5711 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
5712 u32 eics = 0;
5713
5714 for (i = 0; i < adapter->num_q_vectors; i++)
5715 eics |= adapter->q_vector[i]->eims_value;
5716 wr32(E1000_EICS, eics);
5717 } else {
5718 wr32(E1000_ICS, E1000_ICS_RXDMT0);
5719 }
5720
5721 igb_spoof_check(adapter);
5722 igb_ptp_rx_hang(adapter);
5723 igb_ptp_tx_hang(adapter);
5724
5725 /* Check LVMMC register on i350/i354 only */
5726 if ((adapter->hw.mac.type == e1000_i350) ||
5727 (adapter->hw.mac.type == e1000_i354))
5728 igb_check_lvmmc(adapter);
5729
5730 /* Reset the timer */
5731 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5732 if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)
5733 mod_timer(&adapter->watchdog_timer,
5734 round_jiffies(jiffies + HZ));
5735 else
5736 mod_timer(&adapter->watchdog_timer,
5737 round_jiffies(jiffies + 2 * HZ));
5738 }
5739 }
5740
5741 enum latency_range {
5742 lowest_latency = 0,
5743 low_latency = 1,
5744 bulk_latency = 2,
5745 latency_invalid = 255
5746 };
5747
5748 /**
5749 * igb_update_ring_itr - update the dynamic ITR value based on packet size
5750 * @q_vector: pointer to q_vector
5751 *
5752 * Stores a new ITR value based on strictly on packet size. This
5753 * algorithm is less sophisticated than that used in igb_update_itr,
5754 * due to the difficulty of synchronizing statistics across multiple
5755 * receive rings. The divisors and thresholds used by this function
5756 * were determined based on theoretical maximum wire speed and testing
5757 * data, in order to minimize response time while increasing bulk
5758 * throughput.
5759 * This functionality is controlled by ethtool's coalescing settings.
5760 * NOTE: This function is called only when operating in a multiqueue
5761 * receive environment.
5762 **/
igb_update_ring_itr(struct igb_q_vector * q_vector)5763 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
5764 {
5765 int new_val = q_vector->itr_val;
5766 int avg_wire_size = 0;
5767 struct igb_adapter *adapter = q_vector->adapter;
5768 unsigned int packets;
5769
5770 /* For non-gigabit speeds, just fix the interrupt rate at 4000
5771 * ints/sec - ITR timer value of 120 ticks.
5772 */
5773 if (adapter->link_speed != SPEED_1000) {
5774 new_val = IGB_4K_ITR;
5775 goto set_itr_val;
5776 }
5777
5778 packets = q_vector->rx.total_packets;
5779 if (packets)
5780 avg_wire_size = q_vector->rx.total_bytes / packets;
5781
5782 packets = q_vector->tx.total_packets;
5783 if (packets)
5784 avg_wire_size = max_t(u32, avg_wire_size,
5785 q_vector->tx.total_bytes / packets);
5786
5787 /* if avg_wire_size isn't set no work was done */
5788 if (!avg_wire_size)
5789 goto clear_counts;
5790
5791 /* Add 24 bytes to size to account for CRC, preamble, and gap */
5792 avg_wire_size += 24;
5793
5794 /* Don't starve jumbo frames */
5795 avg_wire_size = min(avg_wire_size, 3000);
5796
5797 /* Give a little boost to mid-size frames */
5798 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
5799 new_val = avg_wire_size / 3;
5800 else
5801 new_val = avg_wire_size / 2;
5802
5803 /* conservative mode (itr 3) eliminates the lowest_latency setting */
5804 if (new_val < IGB_20K_ITR &&
5805 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
5806 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
5807 new_val = IGB_20K_ITR;
5808
5809 set_itr_val:
5810 if (new_val != q_vector->itr_val) {
5811 q_vector->itr_val = new_val;
5812 q_vector->set_itr = 1;
5813 }
5814 clear_counts:
5815 q_vector->rx.total_bytes = 0;
5816 q_vector->rx.total_packets = 0;
5817 q_vector->tx.total_bytes = 0;
5818 q_vector->tx.total_packets = 0;
5819 }
5820
5821 /**
5822 * igb_update_itr - update the dynamic ITR value based on statistics
5823 * @q_vector: pointer to q_vector
5824 * @ring_container: ring info to update the itr for
5825 *
5826 * Stores a new ITR value based on packets and byte
5827 * counts during the last interrupt. The advantage of per interrupt
5828 * computation is faster updates and more accurate ITR for the current
5829 * traffic pattern. Constants in this function were computed
5830 * based on theoretical maximum wire speed and thresholds were set based
5831 * on testing data as well as attempting to minimize response time
5832 * while increasing bulk throughput.
5833 * This functionality is controlled by ethtool's coalescing settings.
5834 * NOTE: These calculations are only valid when operating in a single-
5835 * queue environment.
5836 **/
igb_update_itr(struct igb_q_vector * q_vector,struct igb_ring_container * ring_container)5837 static void igb_update_itr(struct igb_q_vector *q_vector,
5838 struct igb_ring_container *ring_container)
5839 {
5840 unsigned int packets = ring_container->total_packets;
5841 unsigned int bytes = ring_container->total_bytes;
5842 u8 itrval = ring_container->itr;
5843
5844 /* no packets, exit with status unchanged */
5845 if (packets == 0)
5846 return;
5847
5848 switch (itrval) {
5849 case lowest_latency:
5850 /* handle TSO and jumbo frames */
5851 if (bytes/packets > 8000)
5852 itrval = bulk_latency;
5853 else if ((packets < 5) && (bytes > 512))
5854 itrval = low_latency;
5855 break;
5856 case low_latency: /* 50 usec aka 20000 ints/s */
5857 if (bytes > 10000) {
5858 /* this if handles the TSO accounting */
5859 if (bytes/packets > 8000)
5860 itrval = bulk_latency;
5861 else if ((packets < 10) || ((bytes/packets) > 1200))
5862 itrval = bulk_latency;
5863 else if ((packets > 35))
5864 itrval = lowest_latency;
5865 } else if (bytes/packets > 2000) {
5866 itrval = bulk_latency;
5867 } else if (packets <= 2 && bytes < 512) {
5868 itrval = lowest_latency;
5869 }
5870 break;
5871 case bulk_latency: /* 250 usec aka 4000 ints/s */
5872 if (bytes > 25000) {
5873 if (packets > 35)
5874 itrval = low_latency;
5875 } else if (bytes < 1500) {
5876 itrval = low_latency;
5877 }
5878 break;
5879 }
5880
5881 /* clear work counters since we have the values we need */
5882 ring_container->total_bytes = 0;
5883 ring_container->total_packets = 0;
5884
5885 /* write updated itr to ring container */
5886 ring_container->itr = itrval;
5887 }
5888
igb_set_itr(struct igb_q_vector * q_vector)5889 static void igb_set_itr(struct igb_q_vector *q_vector)
5890 {
5891 struct igb_adapter *adapter = q_vector->adapter;
5892 u32 new_itr = q_vector->itr_val;
5893 u8 current_itr = 0;
5894
5895 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
5896 if (adapter->link_speed != SPEED_1000) {
5897 current_itr = 0;
5898 new_itr = IGB_4K_ITR;
5899 goto set_itr_now;
5900 }
5901
5902 igb_update_itr(q_vector, &q_vector->tx);
5903 igb_update_itr(q_vector, &q_vector->rx);
5904
5905 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
5906
5907 /* conservative mode (itr 3) eliminates the lowest_latency setting */
5908 if (current_itr == lowest_latency &&
5909 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
5910 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
5911 current_itr = low_latency;
5912
5913 switch (current_itr) {
5914 /* counts and packets in update_itr are dependent on these numbers */
5915 case lowest_latency:
5916 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
5917 break;
5918 case low_latency:
5919 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
5920 break;
5921 case bulk_latency:
5922 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
5923 break;
5924 default:
5925 break;
5926 }
5927
5928 set_itr_now:
5929 if (new_itr != q_vector->itr_val) {
5930 /* this attempts to bias the interrupt rate towards Bulk
5931 * by adding intermediate steps when interrupt rate is
5932 * increasing
5933 */
5934 new_itr = new_itr > q_vector->itr_val ?
5935 max((new_itr * q_vector->itr_val) /
5936 (new_itr + (q_vector->itr_val >> 2)),
5937 new_itr) : new_itr;
5938 /* Don't write the value here; it resets the adapter's
5939 * internal timer, and causes us to delay far longer than
5940 * we should between interrupts. Instead, we write the ITR
5941 * value at the beginning of the next interrupt so the timing
5942 * ends up being correct.
5943 */
5944 q_vector->itr_val = new_itr;
5945 q_vector->set_itr = 1;
5946 }
5947 }
5948
igb_tx_ctxtdesc(struct igb_ring * tx_ring,struct igb_tx_buffer * first,u32 vlan_macip_lens,u32 type_tucmd,u32 mss_l4len_idx)5949 static void igb_tx_ctxtdesc(struct igb_ring *tx_ring,
5950 struct igb_tx_buffer *first,
5951 u32 vlan_macip_lens, u32 type_tucmd,
5952 u32 mss_l4len_idx)
5953 {
5954 struct e1000_adv_tx_context_desc *context_desc;
5955 u16 i = tx_ring->next_to_use;
5956 struct timespec64 ts;
5957
5958 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
5959
5960 i++;
5961 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
5962
5963 /* set bits to identify this as an advanced context descriptor */
5964 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
5965
5966 /* For 82575, context index must be unique per ring. */
5967 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
5968 mss_l4len_idx |= tx_ring->reg_idx << 4;
5969
5970 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
5971 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
5972 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
5973
5974 /* We assume there is always a valid tx time available. Invalid times
5975 * should have been handled by the upper layers.
5976 */
5977 if (tx_ring->launchtime_enable) {
5978 ts = ktime_to_timespec64(first->skb->tstamp);
5979 skb_txtime_consumed(first->skb);
5980 context_desc->seqnum_seed = cpu_to_le32(ts.tv_nsec / 32);
5981 } else {
5982 context_desc->seqnum_seed = 0;
5983 }
5984 }
5985
igb_tso(struct igb_ring * tx_ring,struct igb_tx_buffer * first,u8 * hdr_len)5986 static int igb_tso(struct igb_ring *tx_ring,
5987 struct igb_tx_buffer *first,
5988 u8 *hdr_len)
5989 {
5990 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
5991 struct sk_buff *skb = first->skb;
5992 union {
5993 struct iphdr *v4;
5994 struct ipv6hdr *v6;
5995 unsigned char *hdr;
5996 } ip;
5997 union {
5998 struct tcphdr *tcp;
5999 struct udphdr *udp;
6000 unsigned char *hdr;
6001 } l4;
6002 u32 paylen, l4_offset;
6003 int err;
6004
6005 if (skb->ip_summed != CHECKSUM_PARTIAL)
6006 return 0;
6007
6008 if (!skb_is_gso(skb))
6009 return 0;
6010
6011 err = skb_cow_head(skb, 0);
6012 if (err < 0)
6013 return err;
6014
6015 ip.hdr = skb_network_header(skb);
6016 l4.hdr = skb_checksum_start(skb);
6017
6018 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
6019 type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ?
6020 E1000_ADVTXD_TUCMD_L4T_UDP : E1000_ADVTXD_TUCMD_L4T_TCP;
6021
6022 /* initialize outer IP header fields */
6023 if (ip.v4->version == 4) {
6024 unsigned char *csum_start = skb_checksum_start(skb);
6025 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
6026
6027 /* IP header will have to cancel out any data that
6028 * is not a part of the outer IP header
6029 */
6030 ip.v4->check = csum_fold(csum_partial(trans_start,
6031 csum_start - trans_start,
6032 0));
6033 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
6034
6035 ip.v4->tot_len = 0;
6036 first->tx_flags |= IGB_TX_FLAGS_TSO |
6037 IGB_TX_FLAGS_CSUM |
6038 IGB_TX_FLAGS_IPV4;
6039 } else {
6040 ip.v6->payload_len = 0;
6041 first->tx_flags |= IGB_TX_FLAGS_TSO |
6042 IGB_TX_FLAGS_CSUM;
6043 }
6044
6045 /* determine offset of inner transport header */
6046 l4_offset = l4.hdr - skb->data;
6047
6048 /* remove payload length from inner checksum */
6049 paylen = skb->len - l4_offset;
6050 if (type_tucmd & E1000_ADVTXD_TUCMD_L4T_TCP) {
6051 /* compute length of segmentation header */
6052 *hdr_len = (l4.tcp->doff * 4) + l4_offset;
6053 csum_replace_by_diff(&l4.tcp->check,
6054 (__force __wsum)htonl(paylen));
6055 } else {
6056 /* compute length of segmentation header */
6057 *hdr_len = sizeof(*l4.udp) + l4_offset;
6058 csum_replace_by_diff(&l4.udp->check,
6059 (__force __wsum)htonl(paylen));
6060 }
6061
6062 /* update gso size and bytecount with header size */
6063 first->gso_segs = skb_shinfo(skb)->gso_segs;
6064 first->bytecount += (first->gso_segs - 1) * *hdr_len;
6065
6066 /* MSS L4LEN IDX */
6067 mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
6068 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
6069
6070 /* VLAN MACLEN IPLEN */
6071 vlan_macip_lens = l4.hdr - ip.hdr;
6072 vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
6073 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
6074
6075 igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens,
6076 type_tucmd, mss_l4len_idx);
6077
6078 return 1;
6079 }
6080
igb_tx_csum(struct igb_ring * tx_ring,struct igb_tx_buffer * first)6081 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
6082 {
6083 struct sk_buff *skb = first->skb;
6084 u32 vlan_macip_lens = 0;
6085 u32 type_tucmd = 0;
6086
6087 if (skb->ip_summed != CHECKSUM_PARTIAL) {
6088 csum_failed:
6089 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN) &&
6090 !tx_ring->launchtime_enable)
6091 return;
6092 goto no_csum;
6093 }
6094
6095 switch (skb->csum_offset) {
6096 case offsetof(struct tcphdr, check):
6097 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
6098 fallthrough;
6099 case offsetof(struct udphdr, check):
6100 break;
6101 case offsetof(struct sctphdr, checksum):
6102 /* validate that this is actually an SCTP request */
6103 if (skb_csum_is_sctp(skb)) {
6104 type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
6105 break;
6106 }
6107 fallthrough;
6108 default:
6109 skb_checksum_help(skb);
6110 goto csum_failed;
6111 }
6112
6113 /* update TX checksum flag */
6114 first->tx_flags |= IGB_TX_FLAGS_CSUM;
6115 vlan_macip_lens = skb_checksum_start_offset(skb) -
6116 skb_network_offset(skb);
6117 no_csum:
6118 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
6119 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
6120
6121 igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0);
6122 }
6123
6124 #define IGB_SET_FLAG(_input, _flag, _result) \
6125 ((_flag <= _result) ? \
6126 ((u32)(_input & _flag) * (_result / _flag)) : \
6127 ((u32)(_input & _flag) / (_flag / _result)))
6128
igb_tx_cmd_type(struct sk_buff * skb,u32 tx_flags)6129 static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
6130 {
6131 /* set type for advanced descriptor with frame checksum insertion */
6132 u32 cmd_type = E1000_ADVTXD_DTYP_DATA |
6133 E1000_ADVTXD_DCMD_DEXT |
6134 E1000_ADVTXD_DCMD_IFCS;
6135
6136 /* set HW vlan bit if vlan is present */
6137 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN,
6138 (E1000_ADVTXD_DCMD_VLE));
6139
6140 /* set segmentation bits for TSO */
6141 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO,
6142 (E1000_ADVTXD_DCMD_TSE));
6143
6144 /* set timestamp bit if present */
6145 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP,
6146 (E1000_ADVTXD_MAC_TSTAMP));
6147
6148 /* insert frame checksum */
6149 cmd_type ^= IGB_SET_FLAG(skb->no_fcs, 1, E1000_ADVTXD_DCMD_IFCS);
6150
6151 return cmd_type;
6152 }
6153
igb_tx_olinfo_status(struct igb_ring * tx_ring,union e1000_adv_tx_desc * tx_desc,u32 tx_flags,unsigned int paylen)6154 static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
6155 union e1000_adv_tx_desc *tx_desc,
6156 u32 tx_flags, unsigned int paylen)
6157 {
6158 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
6159
6160 /* 82575 requires a unique index per ring */
6161 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
6162 olinfo_status |= tx_ring->reg_idx << 4;
6163
6164 /* insert L4 checksum */
6165 olinfo_status |= IGB_SET_FLAG(tx_flags,
6166 IGB_TX_FLAGS_CSUM,
6167 (E1000_TXD_POPTS_TXSM << 8));
6168
6169 /* insert IPv4 checksum */
6170 olinfo_status |= IGB_SET_FLAG(tx_flags,
6171 IGB_TX_FLAGS_IPV4,
6172 (E1000_TXD_POPTS_IXSM << 8));
6173
6174 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
6175 }
6176
__igb_maybe_stop_tx(struct igb_ring * tx_ring,const u16 size)6177 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
6178 {
6179 struct net_device *netdev = tx_ring->netdev;
6180
6181 netif_stop_subqueue(netdev, tx_ring->queue_index);
6182
6183 /* Herbert's original patch had:
6184 * smp_mb__after_netif_stop_queue();
6185 * but since that doesn't exist yet, just open code it.
6186 */
6187 smp_mb();
6188
6189 /* We need to check again in a case another CPU has just
6190 * made room available.
6191 */
6192 if (igb_desc_unused(tx_ring) < size)
6193 return -EBUSY;
6194
6195 /* A reprieve! */
6196 netif_wake_subqueue(netdev, tx_ring->queue_index);
6197
6198 u64_stats_update_begin(&tx_ring->tx_syncp2);
6199 tx_ring->tx_stats.restart_queue2++;
6200 u64_stats_update_end(&tx_ring->tx_syncp2);
6201
6202 return 0;
6203 }
6204
igb_maybe_stop_tx(struct igb_ring * tx_ring,const u16 size)6205 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
6206 {
6207 if (igb_desc_unused(tx_ring) >= size)
6208 return 0;
6209 return __igb_maybe_stop_tx(tx_ring, size);
6210 }
6211
igb_tx_map(struct igb_ring * tx_ring,struct igb_tx_buffer * first,const u8 hdr_len)6212 static int igb_tx_map(struct igb_ring *tx_ring,
6213 struct igb_tx_buffer *first,
6214 const u8 hdr_len)
6215 {
6216 struct sk_buff *skb = first->skb;
6217 struct igb_tx_buffer *tx_buffer;
6218 union e1000_adv_tx_desc *tx_desc;
6219 skb_frag_t *frag;
6220 dma_addr_t dma;
6221 unsigned int data_len, size;
6222 u32 tx_flags = first->tx_flags;
6223 u32 cmd_type = igb_tx_cmd_type(skb, tx_flags);
6224 u16 i = tx_ring->next_to_use;
6225
6226 tx_desc = IGB_TX_DESC(tx_ring, i);
6227
6228 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
6229
6230 size = skb_headlen(skb);
6231 data_len = skb->data_len;
6232
6233 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
6234
6235 tx_buffer = first;
6236
6237 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
6238 if (dma_mapping_error(tx_ring->dev, dma))
6239 goto dma_error;
6240
6241 /* record length, and DMA address */
6242 dma_unmap_len_set(tx_buffer, len, size);
6243 dma_unmap_addr_set(tx_buffer, dma, dma);
6244
6245 tx_desc->read.buffer_addr = cpu_to_le64(dma);
6246
6247 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
6248 tx_desc->read.cmd_type_len =
6249 cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD);
6250
6251 i++;
6252 tx_desc++;
6253 if (i == tx_ring->count) {
6254 tx_desc = IGB_TX_DESC(tx_ring, 0);
6255 i = 0;
6256 }
6257 tx_desc->read.olinfo_status = 0;
6258
6259 dma += IGB_MAX_DATA_PER_TXD;
6260 size -= IGB_MAX_DATA_PER_TXD;
6261
6262 tx_desc->read.buffer_addr = cpu_to_le64(dma);
6263 }
6264
6265 if (likely(!data_len))
6266 break;
6267
6268 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
6269
6270 i++;
6271 tx_desc++;
6272 if (i == tx_ring->count) {
6273 tx_desc = IGB_TX_DESC(tx_ring, 0);
6274 i = 0;
6275 }
6276 tx_desc->read.olinfo_status = 0;
6277
6278 size = skb_frag_size(frag);
6279 data_len -= size;
6280
6281 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
6282 size, DMA_TO_DEVICE);
6283
6284 tx_buffer = &tx_ring->tx_buffer_info[i];
6285 }
6286
6287 /* write last descriptor with RS and EOP bits */
6288 cmd_type |= size | IGB_TXD_DCMD;
6289 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
6290
6291 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
6292
6293 /* set the timestamp */
6294 first->time_stamp = jiffies;
6295
6296 skb_tx_timestamp(skb);
6297
6298 /* Force memory writes to complete before letting h/w know there
6299 * are new descriptors to fetch. (Only applicable for weak-ordered
6300 * memory model archs, such as IA-64).
6301 *
6302 * We also need this memory barrier to make certain all of the
6303 * status bits have been updated before next_to_watch is written.
6304 */
6305 dma_wmb();
6306
6307 /* set next_to_watch value indicating a packet is present */
6308 first->next_to_watch = tx_desc;
6309
6310 i++;
6311 if (i == tx_ring->count)
6312 i = 0;
6313
6314 tx_ring->next_to_use = i;
6315
6316 /* Make sure there is space in the ring for the next send. */
6317 igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
6318
6319 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
6320 writel(i, tx_ring->tail);
6321 }
6322 return 0;
6323
6324 dma_error:
6325 dev_err(tx_ring->dev, "TX DMA map failed\n");
6326 tx_buffer = &tx_ring->tx_buffer_info[i];
6327
6328 /* clear dma mappings for failed tx_buffer_info map */
6329 while (tx_buffer != first) {
6330 if (dma_unmap_len(tx_buffer, len))
6331 dma_unmap_page(tx_ring->dev,
6332 dma_unmap_addr(tx_buffer, dma),
6333 dma_unmap_len(tx_buffer, len),
6334 DMA_TO_DEVICE);
6335 dma_unmap_len_set(tx_buffer, len, 0);
6336
6337 if (i-- == 0)
6338 i += tx_ring->count;
6339 tx_buffer = &tx_ring->tx_buffer_info[i];
6340 }
6341
6342 if (dma_unmap_len(tx_buffer, len))
6343 dma_unmap_single(tx_ring->dev,
6344 dma_unmap_addr(tx_buffer, dma),
6345 dma_unmap_len(tx_buffer, len),
6346 DMA_TO_DEVICE);
6347 dma_unmap_len_set(tx_buffer, len, 0);
6348
6349 dev_kfree_skb_any(tx_buffer->skb);
6350 tx_buffer->skb = NULL;
6351
6352 tx_ring->next_to_use = i;
6353
6354 return -1;
6355 }
6356
igb_xmit_xdp_ring(struct igb_adapter * adapter,struct igb_ring * tx_ring,struct xdp_frame * xdpf)6357 int igb_xmit_xdp_ring(struct igb_adapter *adapter,
6358 struct igb_ring *tx_ring,
6359 struct xdp_frame *xdpf)
6360 {
6361 struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
6362 u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
6363 u16 count, i, index = tx_ring->next_to_use;
6364 struct igb_tx_buffer *tx_head = &tx_ring->tx_buffer_info[index];
6365 struct igb_tx_buffer *tx_buffer = tx_head;
6366 union e1000_adv_tx_desc *tx_desc = IGB_TX_DESC(tx_ring, index);
6367 u32 len = xdpf->len, cmd_type, olinfo_status;
6368 void *data = xdpf->data;
6369
6370 count = TXD_USE_COUNT(len);
6371 for (i = 0; i < nr_frags; i++)
6372 count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i]));
6373
6374 if (igb_maybe_stop_tx(tx_ring, count + 3))
6375 return IGB_XDP_CONSUMED;
6376
6377 i = 0;
6378 /* record the location of the first descriptor for this packet */
6379 tx_head->bytecount = xdp_get_frame_len(xdpf);
6380 tx_head->type = IGB_TYPE_XDP;
6381 tx_head->gso_segs = 1;
6382 tx_head->xdpf = xdpf;
6383
6384 olinfo_status = tx_head->bytecount << E1000_ADVTXD_PAYLEN_SHIFT;
6385 /* 82575 requires a unique index per ring */
6386 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
6387 olinfo_status |= tx_ring->reg_idx << 4;
6388 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
6389
6390 for (;;) {
6391 dma_addr_t dma;
6392
6393 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
6394 if (dma_mapping_error(tx_ring->dev, dma))
6395 goto unmap;
6396
6397 /* record length, and DMA address */
6398 dma_unmap_len_set(tx_buffer, len, len);
6399 dma_unmap_addr_set(tx_buffer, dma, dma);
6400
6401 /* put descriptor type bits */
6402 cmd_type = E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_DEXT |
6403 E1000_ADVTXD_DCMD_IFCS | len;
6404
6405 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
6406 tx_desc->read.buffer_addr = cpu_to_le64(dma);
6407
6408 tx_buffer->protocol = 0;
6409
6410 if (++index == tx_ring->count)
6411 index = 0;
6412
6413 if (i == nr_frags)
6414 break;
6415
6416 tx_buffer = &tx_ring->tx_buffer_info[index];
6417 tx_desc = IGB_TX_DESC(tx_ring, index);
6418 tx_desc->read.olinfo_status = 0;
6419
6420 data = skb_frag_address(&sinfo->frags[i]);
6421 len = skb_frag_size(&sinfo->frags[i]);
6422 i++;
6423 }
6424 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_TXD_DCMD);
6425
6426 netdev_tx_sent_queue(txring_txq(tx_ring), tx_head->bytecount);
6427 /* set the timestamp */
6428 tx_head->time_stamp = jiffies;
6429
6430 /* Avoid any potential race with xdp_xmit and cleanup */
6431 smp_wmb();
6432
6433 /* set next_to_watch value indicating a packet is present */
6434 tx_head->next_to_watch = tx_desc;
6435 tx_ring->next_to_use = index;
6436
6437 /* Make sure there is space in the ring for the next send. */
6438 igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
6439
6440 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more())
6441 writel(index, tx_ring->tail);
6442
6443 return IGB_XDP_TX;
6444
6445 unmap:
6446 for (;;) {
6447 tx_buffer = &tx_ring->tx_buffer_info[index];
6448 if (dma_unmap_len(tx_buffer, len))
6449 dma_unmap_page(tx_ring->dev,
6450 dma_unmap_addr(tx_buffer, dma),
6451 dma_unmap_len(tx_buffer, len),
6452 DMA_TO_DEVICE);
6453 dma_unmap_len_set(tx_buffer, len, 0);
6454 if (tx_buffer == tx_head)
6455 break;
6456
6457 if (!index)
6458 index += tx_ring->count;
6459 index--;
6460 }
6461
6462 return IGB_XDP_CONSUMED;
6463 }
6464
igb_xmit_frame_ring(struct sk_buff * skb,struct igb_ring * tx_ring)6465 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
6466 struct igb_ring *tx_ring)
6467 {
6468 struct igb_tx_buffer *first;
6469 int tso;
6470 u32 tx_flags = 0;
6471 unsigned short f;
6472 u16 count = TXD_USE_COUNT(skb_headlen(skb));
6473 __be16 protocol = vlan_get_protocol(skb);
6474 u8 hdr_len = 0;
6475
6476 /* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD,
6477 * + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD,
6478 * + 2 desc gap to keep tail from touching head,
6479 * + 1 desc for context descriptor,
6480 * otherwise try next time
6481 */
6482 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
6483 count += TXD_USE_COUNT(skb_frag_size(
6484 &skb_shinfo(skb)->frags[f]));
6485
6486 if (igb_maybe_stop_tx(tx_ring, count + 3)) {
6487 /* this is a hard error */
6488 return NETDEV_TX_BUSY;
6489 }
6490
6491 /* record the location of the first descriptor for this packet */
6492 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
6493 first->type = IGB_TYPE_SKB;
6494 first->skb = skb;
6495 first->bytecount = skb->len;
6496 first->gso_segs = 1;
6497
6498 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
6499 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
6500
6501 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
6502 !test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS,
6503 &adapter->state)) {
6504 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
6505 tx_flags |= IGB_TX_FLAGS_TSTAMP;
6506
6507 adapter->ptp_tx_skb = skb_get(skb);
6508 adapter->ptp_tx_start = jiffies;
6509 if (adapter->hw.mac.type == e1000_82576)
6510 schedule_work(&adapter->ptp_tx_work);
6511 } else {
6512 adapter->tx_hwtstamp_skipped++;
6513 }
6514 }
6515
6516 if (skb_vlan_tag_present(skb)) {
6517 tx_flags |= IGB_TX_FLAGS_VLAN;
6518 tx_flags |= (skb_vlan_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
6519 }
6520
6521 /* record initial flags and protocol */
6522 first->tx_flags = tx_flags;
6523 first->protocol = protocol;
6524
6525 tso = igb_tso(tx_ring, first, &hdr_len);
6526 if (tso < 0)
6527 goto out_drop;
6528 else if (!tso)
6529 igb_tx_csum(tx_ring, first);
6530
6531 if (igb_tx_map(tx_ring, first, hdr_len))
6532 goto cleanup_tx_tstamp;
6533
6534 return NETDEV_TX_OK;
6535
6536 out_drop:
6537 dev_kfree_skb_any(first->skb);
6538 first->skb = NULL;
6539 cleanup_tx_tstamp:
6540 if (unlikely(tx_flags & IGB_TX_FLAGS_TSTAMP)) {
6541 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
6542
6543 dev_kfree_skb_any(adapter->ptp_tx_skb);
6544 adapter->ptp_tx_skb = NULL;
6545 if (adapter->hw.mac.type == e1000_82576)
6546 cancel_work_sync(&adapter->ptp_tx_work);
6547 clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
6548 }
6549
6550 return NETDEV_TX_OK;
6551 }
6552
igb_tx_queue_mapping(struct igb_adapter * adapter,struct sk_buff * skb)6553 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
6554 struct sk_buff *skb)
6555 {
6556 unsigned int r_idx = skb->queue_mapping;
6557
6558 if (r_idx >= adapter->num_tx_queues)
6559 r_idx = r_idx % adapter->num_tx_queues;
6560
6561 return adapter->tx_ring[r_idx];
6562 }
6563
igb_xmit_frame(struct sk_buff * skb,struct net_device * netdev)6564 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
6565 struct net_device *netdev)
6566 {
6567 struct igb_adapter *adapter = netdev_priv(netdev);
6568
6569 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
6570 * in order to meet this minimum size requirement.
6571 */
6572 if (skb_put_padto(skb, 17))
6573 return NETDEV_TX_OK;
6574
6575 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
6576 }
6577
6578 /**
6579 * igb_tx_timeout - Respond to a Tx Hang
6580 * @netdev: network interface device structure
6581 * @txqueue: number of the Tx queue that hung (unused)
6582 **/
igb_tx_timeout(struct net_device * netdev,unsigned int __always_unused txqueue)6583 static void igb_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
6584 {
6585 struct igb_adapter *adapter = netdev_priv(netdev);
6586 struct e1000_hw *hw = &adapter->hw;
6587
6588 /* Do the reset outside of interrupt context */
6589 adapter->tx_timeout_count++;
6590
6591 if (hw->mac.type >= e1000_82580)
6592 hw->dev_spec._82575.global_device_reset = true;
6593
6594 schedule_work(&adapter->reset_task);
6595 wr32(E1000_EICS,
6596 (adapter->eims_enable_mask & ~adapter->eims_other));
6597 }
6598
igb_reset_task(struct work_struct * work)6599 static void igb_reset_task(struct work_struct *work)
6600 {
6601 struct igb_adapter *adapter;
6602 adapter = container_of(work, struct igb_adapter, reset_task);
6603
6604 rtnl_lock();
6605 /* If we're already down or resetting, just bail */
6606 if (test_bit(__IGB_DOWN, &adapter->state) ||
6607 test_bit(__IGB_RESETTING, &adapter->state)) {
6608 rtnl_unlock();
6609 return;
6610 }
6611
6612 igb_dump(adapter);
6613 netdev_err(adapter->netdev, "Reset adapter\n");
6614 igb_reinit_locked(adapter);
6615 rtnl_unlock();
6616 }
6617
6618 /**
6619 * igb_get_stats64 - Get System Network Statistics
6620 * @netdev: network interface device structure
6621 * @stats: rtnl_link_stats64 pointer
6622 **/
igb_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)6623 static void igb_get_stats64(struct net_device *netdev,
6624 struct rtnl_link_stats64 *stats)
6625 {
6626 struct igb_adapter *adapter = netdev_priv(netdev);
6627
6628 spin_lock(&adapter->stats64_lock);
6629 igb_update_stats(adapter);
6630 memcpy(stats, &adapter->stats64, sizeof(*stats));
6631 spin_unlock(&adapter->stats64_lock);
6632 }
6633
6634 /**
6635 * igb_change_mtu - Change the Maximum Transfer Unit
6636 * @netdev: network interface device structure
6637 * @new_mtu: new value for maximum frame size
6638 *
6639 * Returns 0 on success, negative on failure
6640 **/
igb_change_mtu(struct net_device * netdev,int new_mtu)6641 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
6642 {
6643 struct igb_adapter *adapter = netdev_priv(netdev);
6644 int max_frame = new_mtu + IGB_ETH_PKT_HDR_PAD;
6645
6646 if (adapter->xdp_prog) {
6647 int i;
6648
6649 for (i = 0; i < adapter->num_rx_queues; i++) {
6650 struct igb_ring *ring = adapter->rx_ring[i];
6651
6652 if (max_frame > igb_rx_bufsz(ring)) {
6653 netdev_warn(adapter->netdev,
6654 "Requested MTU size is not supported with XDP. Max frame size is %d\n",
6655 max_frame);
6656 return -EINVAL;
6657 }
6658 }
6659 }
6660
6661 /* adjust max frame to be at least the size of a standard frame */
6662 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
6663 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
6664
6665 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
6666 usleep_range(1000, 2000);
6667
6668 /* igb_down has a dependency on max_frame_size */
6669 adapter->max_frame_size = max_frame;
6670
6671 if (netif_running(netdev))
6672 igb_down(adapter);
6673
6674 netdev_dbg(netdev, "changing MTU from %d to %d\n",
6675 netdev->mtu, new_mtu);
6676 netdev->mtu = new_mtu;
6677
6678 if (netif_running(netdev))
6679 igb_up(adapter);
6680 else
6681 igb_reset(adapter);
6682
6683 clear_bit(__IGB_RESETTING, &adapter->state);
6684
6685 return 0;
6686 }
6687
6688 /**
6689 * igb_update_stats - Update the board statistics counters
6690 * @adapter: board private structure
6691 **/
igb_update_stats(struct igb_adapter * adapter)6692 void igb_update_stats(struct igb_adapter *adapter)
6693 {
6694 struct rtnl_link_stats64 *net_stats = &adapter->stats64;
6695 struct e1000_hw *hw = &adapter->hw;
6696 struct pci_dev *pdev = adapter->pdev;
6697 u32 reg, mpc;
6698 int i;
6699 u64 bytes, packets;
6700 unsigned int start;
6701 u64 _bytes, _packets;
6702
6703 /* Prevent stats update while adapter is being reset, or if the pci
6704 * connection is down.
6705 */
6706 if (adapter->link_speed == 0)
6707 return;
6708 if (pci_channel_offline(pdev))
6709 return;
6710
6711 bytes = 0;
6712 packets = 0;
6713
6714 rcu_read_lock();
6715 for (i = 0; i < adapter->num_rx_queues; i++) {
6716 struct igb_ring *ring = adapter->rx_ring[i];
6717 u32 rqdpc = rd32(E1000_RQDPC(i));
6718 if (hw->mac.type >= e1000_i210)
6719 wr32(E1000_RQDPC(i), 0);
6720
6721 if (rqdpc) {
6722 ring->rx_stats.drops += rqdpc;
6723 net_stats->rx_fifo_errors += rqdpc;
6724 }
6725
6726 do {
6727 start = u64_stats_fetch_begin(&ring->rx_syncp);
6728 _bytes = ring->rx_stats.bytes;
6729 _packets = ring->rx_stats.packets;
6730 } while (u64_stats_fetch_retry(&ring->rx_syncp, start));
6731 bytes += _bytes;
6732 packets += _packets;
6733 }
6734
6735 net_stats->rx_bytes = bytes;
6736 net_stats->rx_packets = packets;
6737
6738 bytes = 0;
6739 packets = 0;
6740 for (i = 0; i < adapter->num_tx_queues; i++) {
6741 struct igb_ring *ring = adapter->tx_ring[i];
6742 do {
6743 start = u64_stats_fetch_begin(&ring->tx_syncp);
6744 _bytes = ring->tx_stats.bytes;
6745 _packets = ring->tx_stats.packets;
6746 } while (u64_stats_fetch_retry(&ring->tx_syncp, start));
6747 bytes += _bytes;
6748 packets += _packets;
6749 }
6750 net_stats->tx_bytes = bytes;
6751 net_stats->tx_packets = packets;
6752 rcu_read_unlock();
6753
6754 /* read stats registers */
6755 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
6756 adapter->stats.gprc += rd32(E1000_GPRC);
6757 adapter->stats.gorc += rd32(E1000_GORCL);
6758 rd32(E1000_GORCH); /* clear GORCL */
6759 adapter->stats.bprc += rd32(E1000_BPRC);
6760 adapter->stats.mprc += rd32(E1000_MPRC);
6761 adapter->stats.roc += rd32(E1000_ROC);
6762
6763 adapter->stats.prc64 += rd32(E1000_PRC64);
6764 adapter->stats.prc127 += rd32(E1000_PRC127);
6765 adapter->stats.prc255 += rd32(E1000_PRC255);
6766 adapter->stats.prc511 += rd32(E1000_PRC511);
6767 adapter->stats.prc1023 += rd32(E1000_PRC1023);
6768 adapter->stats.prc1522 += rd32(E1000_PRC1522);
6769 adapter->stats.symerrs += rd32(E1000_SYMERRS);
6770 adapter->stats.sec += rd32(E1000_SEC);
6771
6772 mpc = rd32(E1000_MPC);
6773 adapter->stats.mpc += mpc;
6774 net_stats->rx_fifo_errors += mpc;
6775 adapter->stats.scc += rd32(E1000_SCC);
6776 adapter->stats.ecol += rd32(E1000_ECOL);
6777 adapter->stats.mcc += rd32(E1000_MCC);
6778 adapter->stats.latecol += rd32(E1000_LATECOL);
6779 adapter->stats.dc += rd32(E1000_DC);
6780 adapter->stats.rlec += rd32(E1000_RLEC);
6781 adapter->stats.xonrxc += rd32(E1000_XONRXC);
6782 adapter->stats.xontxc += rd32(E1000_XONTXC);
6783 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
6784 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
6785 adapter->stats.fcruc += rd32(E1000_FCRUC);
6786 adapter->stats.gptc += rd32(E1000_GPTC);
6787 adapter->stats.gotc += rd32(E1000_GOTCL);
6788 rd32(E1000_GOTCH); /* clear GOTCL */
6789 adapter->stats.rnbc += rd32(E1000_RNBC);
6790 adapter->stats.ruc += rd32(E1000_RUC);
6791 adapter->stats.rfc += rd32(E1000_RFC);
6792 adapter->stats.rjc += rd32(E1000_RJC);
6793 adapter->stats.tor += rd32(E1000_TORH);
6794 adapter->stats.tot += rd32(E1000_TOTH);
6795 adapter->stats.tpr += rd32(E1000_TPR);
6796
6797 adapter->stats.ptc64 += rd32(E1000_PTC64);
6798 adapter->stats.ptc127 += rd32(E1000_PTC127);
6799 adapter->stats.ptc255 += rd32(E1000_PTC255);
6800 adapter->stats.ptc511 += rd32(E1000_PTC511);
6801 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
6802 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
6803
6804 adapter->stats.mptc += rd32(E1000_MPTC);
6805 adapter->stats.bptc += rd32(E1000_BPTC);
6806
6807 adapter->stats.tpt += rd32(E1000_TPT);
6808 adapter->stats.colc += rd32(E1000_COLC);
6809
6810 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
6811 /* read internal phy specific stats */
6812 reg = rd32(E1000_CTRL_EXT);
6813 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
6814 adapter->stats.rxerrc += rd32(E1000_RXERRC);
6815
6816 /* this stat has invalid values on i210/i211 */
6817 if ((hw->mac.type != e1000_i210) &&
6818 (hw->mac.type != e1000_i211))
6819 adapter->stats.tncrs += rd32(E1000_TNCRS);
6820 }
6821
6822 adapter->stats.tsctc += rd32(E1000_TSCTC);
6823 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
6824
6825 adapter->stats.iac += rd32(E1000_IAC);
6826 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
6827 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
6828 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
6829 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
6830 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
6831 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
6832 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
6833 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
6834
6835 /* Fill out the OS statistics structure */
6836 net_stats->multicast = adapter->stats.mprc;
6837 net_stats->collisions = adapter->stats.colc;
6838
6839 /* Rx Errors */
6840
6841 /* RLEC on some newer hardware can be incorrect so build
6842 * our own version based on RUC and ROC
6843 */
6844 net_stats->rx_errors = adapter->stats.rxerrc +
6845 adapter->stats.crcerrs + adapter->stats.algnerrc +
6846 adapter->stats.ruc + adapter->stats.roc +
6847 adapter->stats.cexterr;
6848 net_stats->rx_length_errors = adapter->stats.ruc +
6849 adapter->stats.roc;
6850 net_stats->rx_crc_errors = adapter->stats.crcerrs;
6851 net_stats->rx_frame_errors = adapter->stats.algnerrc;
6852 net_stats->rx_missed_errors = adapter->stats.mpc;
6853
6854 /* Tx Errors */
6855 net_stats->tx_errors = adapter->stats.ecol +
6856 adapter->stats.latecol;
6857 net_stats->tx_aborted_errors = adapter->stats.ecol;
6858 net_stats->tx_window_errors = adapter->stats.latecol;
6859 net_stats->tx_carrier_errors = adapter->stats.tncrs;
6860
6861 /* Tx Dropped needs to be maintained elsewhere */
6862
6863 /* Management Stats */
6864 adapter->stats.mgptc += rd32(E1000_MGTPTC);
6865 adapter->stats.mgprc += rd32(E1000_MGTPRC);
6866 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
6867
6868 /* OS2BMC Stats */
6869 reg = rd32(E1000_MANC);
6870 if (reg & E1000_MANC_EN_BMC2OS) {
6871 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
6872 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
6873 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
6874 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
6875 }
6876 }
6877
igb_perout(struct igb_adapter * adapter,int tsintr_tt)6878 static void igb_perout(struct igb_adapter *adapter, int tsintr_tt)
6879 {
6880 int pin = ptp_find_pin(adapter->ptp_clock, PTP_PF_PEROUT, tsintr_tt);
6881 struct e1000_hw *hw = &adapter->hw;
6882 struct timespec64 ts;
6883 u32 tsauxc;
6884
6885 if (pin < 0 || pin >= IGB_N_SDP)
6886 return;
6887
6888 spin_lock(&adapter->tmreg_lock);
6889
6890 if (hw->mac.type == e1000_82580 ||
6891 hw->mac.type == e1000_i354 ||
6892 hw->mac.type == e1000_i350) {
6893 s64 ns = timespec64_to_ns(&adapter->perout[tsintr_tt].period);
6894 u32 systiml, systimh, level_mask, level, rem;
6895 u64 systim, now;
6896
6897 /* read systim registers in sequence */
6898 rd32(E1000_SYSTIMR);
6899 systiml = rd32(E1000_SYSTIML);
6900 systimh = rd32(E1000_SYSTIMH);
6901 systim = (((u64)(systimh & 0xFF)) << 32) | ((u64)systiml);
6902 now = timecounter_cyc2time(&adapter->tc, systim);
6903
6904 if (pin < 2) {
6905 level_mask = (tsintr_tt == 1) ? 0x80000 : 0x40000;
6906 level = (rd32(E1000_CTRL) & level_mask) ? 1 : 0;
6907 } else {
6908 level_mask = (tsintr_tt == 1) ? 0x80 : 0x40;
6909 level = (rd32(E1000_CTRL_EXT) & level_mask) ? 1 : 0;
6910 }
6911
6912 div_u64_rem(now, ns, &rem);
6913 systim = systim + (ns - rem);
6914
6915 /* synchronize pin level with rising/falling edges */
6916 div_u64_rem(now, ns << 1, &rem);
6917 if (rem < ns) {
6918 /* first half of period */
6919 if (level == 0) {
6920 /* output is already low, skip this period */
6921 systim += ns;
6922 pr_notice("igb: periodic output on %s missed falling edge\n",
6923 adapter->sdp_config[pin].name);
6924 }
6925 } else {
6926 /* second half of period */
6927 if (level == 1) {
6928 /* output is already high, skip this period */
6929 systim += ns;
6930 pr_notice("igb: periodic output on %s missed rising edge\n",
6931 adapter->sdp_config[pin].name);
6932 }
6933 }
6934
6935 /* for this chip family tv_sec is the upper part of the binary value,
6936 * so not seconds
6937 */
6938 ts.tv_nsec = (u32)systim;
6939 ts.tv_sec = ((u32)(systim >> 32)) & 0xFF;
6940 } else {
6941 ts = timespec64_add(adapter->perout[tsintr_tt].start,
6942 adapter->perout[tsintr_tt].period);
6943 }
6944
6945 /* u32 conversion of tv_sec is safe until y2106 */
6946 wr32((tsintr_tt == 1) ? E1000_TRGTTIML1 : E1000_TRGTTIML0, ts.tv_nsec);
6947 wr32((tsintr_tt == 1) ? E1000_TRGTTIMH1 : E1000_TRGTTIMH0, (u32)ts.tv_sec);
6948 tsauxc = rd32(E1000_TSAUXC);
6949 tsauxc |= TSAUXC_EN_TT0;
6950 wr32(E1000_TSAUXC, tsauxc);
6951 adapter->perout[tsintr_tt].start = ts;
6952
6953 spin_unlock(&adapter->tmreg_lock);
6954 }
6955
igb_extts(struct igb_adapter * adapter,int tsintr_tt)6956 static void igb_extts(struct igb_adapter *adapter, int tsintr_tt)
6957 {
6958 int pin = ptp_find_pin(adapter->ptp_clock, PTP_PF_EXTTS, tsintr_tt);
6959 int auxstmpl = (tsintr_tt == 1) ? E1000_AUXSTMPL1 : E1000_AUXSTMPL0;
6960 int auxstmph = (tsintr_tt == 1) ? E1000_AUXSTMPH1 : E1000_AUXSTMPH0;
6961 struct e1000_hw *hw = &adapter->hw;
6962 struct ptp_clock_event event;
6963 struct timespec64 ts;
6964 unsigned long flags;
6965
6966 if (pin < 0 || pin >= IGB_N_SDP)
6967 return;
6968
6969 if (hw->mac.type == e1000_82580 ||
6970 hw->mac.type == e1000_i354 ||
6971 hw->mac.type == e1000_i350) {
6972 u64 ns = rd32(auxstmpl);
6973
6974 ns += ((u64)(rd32(auxstmph) & 0xFF)) << 32;
6975 spin_lock_irqsave(&adapter->tmreg_lock, flags);
6976 ns = timecounter_cyc2time(&adapter->tc, ns);
6977 spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
6978 ts = ns_to_timespec64(ns);
6979 } else {
6980 ts.tv_nsec = rd32(auxstmpl);
6981 ts.tv_sec = rd32(auxstmph);
6982 }
6983
6984 event.type = PTP_CLOCK_EXTTS;
6985 event.index = tsintr_tt;
6986 event.timestamp = ts.tv_sec * 1000000000ULL + ts.tv_nsec;
6987 ptp_clock_event(adapter->ptp_clock, &event);
6988 }
6989
igb_tsync_interrupt(struct igb_adapter * adapter)6990 static void igb_tsync_interrupt(struct igb_adapter *adapter)
6991 {
6992 const u32 mask = (TSINTR_SYS_WRAP | E1000_TSICR_TXTS |
6993 TSINTR_TT0 | TSINTR_TT1 |
6994 TSINTR_AUTT0 | TSINTR_AUTT1);
6995 struct e1000_hw *hw = &adapter->hw;
6996 u32 tsicr = rd32(E1000_TSICR);
6997 struct ptp_clock_event event;
6998
6999 if (hw->mac.type == e1000_82580) {
7000 /* 82580 has a hardware bug that requires an explicit
7001 * write to clear the TimeSync interrupt cause.
7002 */
7003 wr32(E1000_TSICR, tsicr & mask);
7004 }
7005
7006 if (tsicr & TSINTR_SYS_WRAP) {
7007 event.type = PTP_CLOCK_PPS;
7008 if (adapter->ptp_caps.pps)
7009 ptp_clock_event(adapter->ptp_clock, &event);
7010 }
7011
7012 if (tsicr & E1000_TSICR_TXTS) {
7013 /* retrieve hardware timestamp */
7014 schedule_work(&adapter->ptp_tx_work);
7015 }
7016
7017 if (tsicr & TSINTR_TT0)
7018 igb_perout(adapter, 0);
7019
7020 if (tsicr & TSINTR_TT1)
7021 igb_perout(adapter, 1);
7022
7023 if (tsicr & TSINTR_AUTT0)
7024 igb_extts(adapter, 0);
7025
7026 if (tsicr & TSINTR_AUTT1)
7027 igb_extts(adapter, 1);
7028 }
7029
igb_msix_other(int irq,void * data)7030 static irqreturn_t igb_msix_other(int irq, void *data)
7031 {
7032 struct igb_adapter *adapter = data;
7033 struct e1000_hw *hw = &adapter->hw;
7034 u32 icr = rd32(E1000_ICR);
7035 /* reading ICR causes bit 31 of EICR to be cleared */
7036
7037 if (icr & E1000_ICR_DRSTA)
7038 schedule_work(&adapter->reset_task);
7039
7040 if (icr & E1000_ICR_DOUTSYNC) {
7041 /* HW is reporting DMA is out of sync */
7042 adapter->stats.doosync++;
7043 /* The DMA Out of Sync is also indication of a spoof event
7044 * in IOV mode. Check the Wrong VM Behavior register to
7045 * see if it is really a spoof event.
7046 */
7047 igb_check_wvbr(adapter);
7048 }
7049
7050 /* Check for a mailbox event */
7051 if (icr & E1000_ICR_VMMB)
7052 igb_msg_task(adapter);
7053
7054 if (icr & E1000_ICR_LSC) {
7055 hw->mac.get_link_status = 1;
7056 /* guard against interrupt when we're going down */
7057 if (!test_bit(__IGB_DOWN, &adapter->state))
7058 mod_timer(&adapter->watchdog_timer, jiffies + 1);
7059 }
7060
7061 if (icr & E1000_ICR_TS)
7062 igb_tsync_interrupt(adapter);
7063
7064 wr32(E1000_EIMS, adapter->eims_other);
7065
7066 return IRQ_HANDLED;
7067 }
7068
igb_write_itr(struct igb_q_vector * q_vector)7069 static void igb_write_itr(struct igb_q_vector *q_vector)
7070 {
7071 struct igb_adapter *adapter = q_vector->adapter;
7072 u32 itr_val = q_vector->itr_val & 0x7FFC;
7073
7074 if (!q_vector->set_itr)
7075 return;
7076
7077 if (!itr_val)
7078 itr_val = 0x4;
7079
7080 if (adapter->hw.mac.type == e1000_82575)
7081 itr_val |= itr_val << 16;
7082 else
7083 itr_val |= E1000_EITR_CNT_IGNR;
7084
7085 writel(itr_val, q_vector->itr_register);
7086 q_vector->set_itr = 0;
7087 }
7088
igb_msix_ring(int irq,void * data)7089 static irqreturn_t igb_msix_ring(int irq, void *data)
7090 {
7091 struct igb_q_vector *q_vector = data;
7092
7093 /* Write the ITR value calculated from the previous interrupt. */
7094 igb_write_itr(q_vector);
7095
7096 napi_schedule(&q_vector->napi);
7097
7098 return IRQ_HANDLED;
7099 }
7100
7101 #ifdef CONFIG_IGB_DCA
igb_update_tx_dca(struct igb_adapter * adapter,struct igb_ring * tx_ring,int cpu)7102 static void igb_update_tx_dca(struct igb_adapter *adapter,
7103 struct igb_ring *tx_ring,
7104 int cpu)
7105 {
7106 struct e1000_hw *hw = &adapter->hw;
7107 u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);
7108
7109 if (hw->mac.type != e1000_82575)
7110 txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT;
7111
7112 /* We can enable relaxed ordering for reads, but not writes when
7113 * DCA is enabled. This is due to a known issue in some chipsets
7114 * which will cause the DCA tag to be cleared.
7115 */
7116 txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN |
7117 E1000_DCA_TXCTRL_DATA_RRO_EN |
7118 E1000_DCA_TXCTRL_DESC_DCA_EN;
7119
7120 wr32(E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl);
7121 }
7122
igb_update_rx_dca(struct igb_adapter * adapter,struct igb_ring * rx_ring,int cpu)7123 static void igb_update_rx_dca(struct igb_adapter *adapter,
7124 struct igb_ring *rx_ring,
7125 int cpu)
7126 {
7127 struct e1000_hw *hw = &adapter->hw;
7128 u32 rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu);
7129
7130 if (hw->mac.type != e1000_82575)
7131 rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT;
7132
7133 /* We can enable relaxed ordering for reads, but not writes when
7134 * DCA is enabled. This is due to a known issue in some chipsets
7135 * which will cause the DCA tag to be cleared.
7136 */
7137 rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN |
7138 E1000_DCA_RXCTRL_DESC_DCA_EN;
7139
7140 wr32(E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl);
7141 }
7142
igb_update_dca(struct igb_q_vector * q_vector)7143 static void igb_update_dca(struct igb_q_vector *q_vector)
7144 {
7145 struct igb_adapter *adapter = q_vector->adapter;
7146 int cpu = get_cpu();
7147
7148 if (q_vector->cpu == cpu)
7149 goto out_no_update;
7150
7151 if (q_vector->tx.ring)
7152 igb_update_tx_dca(adapter, q_vector->tx.ring, cpu);
7153
7154 if (q_vector->rx.ring)
7155 igb_update_rx_dca(adapter, q_vector->rx.ring, cpu);
7156
7157 q_vector->cpu = cpu;
7158 out_no_update:
7159 put_cpu();
7160 }
7161
igb_setup_dca(struct igb_adapter * adapter)7162 static void igb_setup_dca(struct igb_adapter *adapter)
7163 {
7164 struct e1000_hw *hw = &adapter->hw;
7165 int i;
7166
7167 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
7168 return;
7169
7170 /* Always use CB2 mode, difference is masked in the CB driver. */
7171 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
7172
7173 for (i = 0; i < adapter->num_q_vectors; i++) {
7174 adapter->q_vector[i]->cpu = -1;
7175 igb_update_dca(adapter->q_vector[i]);
7176 }
7177 }
7178
__igb_notify_dca(struct device * dev,void * data)7179 static int __igb_notify_dca(struct device *dev, void *data)
7180 {
7181 struct net_device *netdev = dev_get_drvdata(dev);
7182 struct igb_adapter *adapter = netdev_priv(netdev);
7183 struct pci_dev *pdev = adapter->pdev;
7184 struct e1000_hw *hw = &adapter->hw;
7185 unsigned long event = *(unsigned long *)data;
7186
7187 switch (event) {
7188 case DCA_PROVIDER_ADD:
7189 /* if already enabled, don't do it again */
7190 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
7191 break;
7192 if (dca_add_requester(dev) == 0) {
7193 adapter->flags |= IGB_FLAG_DCA_ENABLED;
7194 dev_info(&pdev->dev, "DCA enabled\n");
7195 igb_setup_dca(adapter);
7196 break;
7197 }
7198 fallthrough; /* since DCA is disabled. */
7199 case DCA_PROVIDER_REMOVE:
7200 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
7201 /* without this a class_device is left
7202 * hanging around in the sysfs model
7203 */
7204 dca_remove_requester(dev);
7205 dev_info(&pdev->dev, "DCA disabled\n");
7206 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
7207 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
7208 }
7209 break;
7210 }
7211
7212 return 0;
7213 }
7214
igb_notify_dca(struct notifier_block * nb,unsigned long event,void * p)7215 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
7216 void *p)
7217 {
7218 int ret_val;
7219
7220 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
7221 __igb_notify_dca);
7222
7223 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
7224 }
7225 #endif /* CONFIG_IGB_DCA */
7226
7227 #ifdef CONFIG_PCI_IOV
igb_vf_configure(struct igb_adapter * adapter,int vf)7228 static int igb_vf_configure(struct igb_adapter *adapter, int vf)
7229 {
7230 unsigned char mac_addr[ETH_ALEN];
7231
7232 eth_zero_addr(mac_addr);
7233 igb_set_vf_mac(adapter, vf, mac_addr);
7234
7235 /* By default spoof check is enabled for all VFs */
7236 adapter->vf_data[vf].spoofchk_enabled = true;
7237
7238 /* By default VFs are not trusted */
7239 adapter->vf_data[vf].trusted = false;
7240
7241 return 0;
7242 }
7243
7244 #endif
igb_ping_all_vfs(struct igb_adapter * adapter)7245 static void igb_ping_all_vfs(struct igb_adapter *adapter)
7246 {
7247 struct e1000_hw *hw = &adapter->hw;
7248 u32 ping;
7249 int i;
7250
7251 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
7252 ping = E1000_PF_CONTROL_MSG;
7253 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
7254 ping |= E1000_VT_MSGTYPE_CTS;
7255 igb_write_mbx(hw, &ping, 1, i);
7256 }
7257 }
7258
igb_set_vf_promisc(struct igb_adapter * adapter,u32 * msgbuf,u32 vf)7259 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
7260 {
7261 struct e1000_hw *hw = &adapter->hw;
7262 u32 vmolr = rd32(E1000_VMOLR(vf));
7263 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7264
7265 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7266 IGB_VF_FLAG_MULTI_PROMISC);
7267 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
7268
7269 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
7270 vmolr |= E1000_VMOLR_MPME;
7271 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7272 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
7273 } else {
7274 /* if we have hashes and we are clearing a multicast promisc
7275 * flag we need to write the hashes to the MTA as this step
7276 * was previously skipped
7277 */
7278 if (vf_data->num_vf_mc_hashes > 30) {
7279 vmolr |= E1000_VMOLR_MPME;
7280 } else if (vf_data->num_vf_mc_hashes) {
7281 int j;
7282
7283 vmolr |= E1000_VMOLR_ROMPE;
7284 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
7285 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
7286 }
7287 }
7288
7289 wr32(E1000_VMOLR(vf), vmolr);
7290
7291 /* there are flags left unprocessed, likely not supported */
7292 if (*msgbuf & E1000_VT_MSGINFO_MASK)
7293 return -EINVAL;
7294
7295 return 0;
7296 }
7297
igb_set_vf_multicasts(struct igb_adapter * adapter,u32 * msgbuf,u32 vf)7298 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
7299 u32 *msgbuf, u32 vf)
7300 {
7301 int n = FIELD_GET(E1000_VT_MSGINFO_MASK, msgbuf[0]);
7302 u16 *hash_list = (u16 *)&msgbuf[1];
7303 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7304 int i;
7305
7306 /* salt away the number of multicast addresses assigned
7307 * to this VF for later use to restore when the PF multi cast
7308 * list changes
7309 */
7310 vf_data->num_vf_mc_hashes = n;
7311
7312 /* only up to 30 hash values supported */
7313 if (n > 30)
7314 n = 30;
7315
7316 /* store the hashes for later use */
7317 for (i = 0; i < n; i++)
7318 vf_data->vf_mc_hashes[i] = hash_list[i];
7319
7320 /* Flush and reset the mta with the new values */
7321 igb_set_rx_mode(adapter->netdev);
7322
7323 return 0;
7324 }
7325
igb_restore_vf_multicasts(struct igb_adapter * adapter)7326 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
7327 {
7328 struct e1000_hw *hw = &adapter->hw;
7329 struct vf_data_storage *vf_data;
7330 int i, j;
7331
7332 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7333 u32 vmolr = rd32(E1000_VMOLR(i));
7334
7335 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
7336
7337 vf_data = &adapter->vf_data[i];
7338
7339 if ((vf_data->num_vf_mc_hashes > 30) ||
7340 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
7341 vmolr |= E1000_VMOLR_MPME;
7342 } else if (vf_data->num_vf_mc_hashes) {
7343 vmolr |= E1000_VMOLR_ROMPE;
7344 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
7345 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
7346 }
7347 wr32(E1000_VMOLR(i), vmolr);
7348 }
7349 }
7350
igb_clear_vf_vfta(struct igb_adapter * adapter,u32 vf)7351 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
7352 {
7353 struct e1000_hw *hw = &adapter->hw;
7354 u32 pool_mask, vlvf_mask, i;
7355
7356 /* create mask for VF and other pools */
7357 pool_mask = E1000_VLVF_POOLSEL_MASK;
7358 vlvf_mask = BIT(E1000_VLVF_POOLSEL_SHIFT + vf);
7359
7360 /* drop PF from pool bits */
7361 pool_mask &= ~BIT(E1000_VLVF_POOLSEL_SHIFT +
7362 adapter->vfs_allocated_count);
7363
7364 /* Find the vlan filter for this id */
7365 for (i = E1000_VLVF_ARRAY_SIZE; i--;) {
7366 u32 vlvf = rd32(E1000_VLVF(i));
7367 u32 vfta_mask, vid, vfta;
7368
7369 /* remove the vf from the pool */
7370 if (!(vlvf & vlvf_mask))
7371 continue;
7372
7373 /* clear out bit from VLVF */
7374 vlvf ^= vlvf_mask;
7375
7376 /* if other pools are present, just remove ourselves */
7377 if (vlvf & pool_mask)
7378 goto update_vlvfb;
7379
7380 /* if PF is present, leave VFTA */
7381 if (vlvf & E1000_VLVF_POOLSEL_MASK)
7382 goto update_vlvf;
7383
7384 vid = vlvf & E1000_VLVF_VLANID_MASK;
7385 vfta_mask = BIT(vid % 32);
7386
7387 /* clear bit from VFTA */
7388 vfta = adapter->shadow_vfta[vid / 32];
7389 if (vfta & vfta_mask)
7390 hw->mac.ops.write_vfta(hw, vid / 32, vfta ^ vfta_mask);
7391 update_vlvf:
7392 /* clear pool selection enable */
7393 if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
7394 vlvf &= E1000_VLVF_POOLSEL_MASK;
7395 else
7396 vlvf = 0;
7397 update_vlvfb:
7398 /* clear pool bits */
7399 wr32(E1000_VLVF(i), vlvf);
7400 }
7401 }
7402
igb_find_vlvf_entry(struct e1000_hw * hw,u32 vlan)7403 static int igb_find_vlvf_entry(struct e1000_hw *hw, u32 vlan)
7404 {
7405 u32 vlvf;
7406 int idx;
7407
7408 /* short cut the special case */
7409 if (vlan == 0)
7410 return 0;
7411
7412 /* Search for the VLAN id in the VLVF entries */
7413 for (idx = E1000_VLVF_ARRAY_SIZE; --idx;) {
7414 vlvf = rd32(E1000_VLVF(idx));
7415 if ((vlvf & VLAN_VID_MASK) == vlan)
7416 break;
7417 }
7418
7419 return idx;
7420 }
7421
igb_update_pf_vlvf(struct igb_adapter * adapter,u32 vid)7422 static void igb_update_pf_vlvf(struct igb_adapter *adapter, u32 vid)
7423 {
7424 struct e1000_hw *hw = &adapter->hw;
7425 u32 bits, pf_id;
7426 int idx;
7427
7428 idx = igb_find_vlvf_entry(hw, vid);
7429 if (!idx)
7430 return;
7431
7432 /* See if any other pools are set for this VLAN filter
7433 * entry other than the PF.
7434 */
7435 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
7436 bits = ~BIT(pf_id) & E1000_VLVF_POOLSEL_MASK;
7437 bits &= rd32(E1000_VLVF(idx));
7438
7439 /* Disable the filter so this falls into the default pool. */
7440 if (!bits) {
7441 if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
7442 wr32(E1000_VLVF(idx), BIT(pf_id));
7443 else
7444 wr32(E1000_VLVF(idx), 0);
7445 }
7446 }
7447
igb_set_vf_vlan(struct igb_adapter * adapter,u32 vid,bool add,u32 vf)7448 static s32 igb_set_vf_vlan(struct igb_adapter *adapter, u32 vid,
7449 bool add, u32 vf)
7450 {
7451 int pf_id = adapter->vfs_allocated_count;
7452 struct e1000_hw *hw = &adapter->hw;
7453 int err;
7454
7455 /* If VLAN overlaps with one the PF is currently monitoring make
7456 * sure that we are able to allocate a VLVF entry. This may be
7457 * redundant but it guarantees PF will maintain visibility to
7458 * the VLAN.
7459 */
7460 if (add && test_bit(vid, adapter->active_vlans)) {
7461 err = igb_vfta_set(hw, vid, pf_id, true, false);
7462 if (err)
7463 return err;
7464 }
7465
7466 err = igb_vfta_set(hw, vid, vf, add, false);
7467
7468 if (add && !err)
7469 return err;
7470
7471 /* If we failed to add the VF VLAN or we are removing the VF VLAN
7472 * we may need to drop the PF pool bit in order to allow us to free
7473 * up the VLVF resources.
7474 */
7475 if (test_bit(vid, adapter->active_vlans) ||
7476 (adapter->flags & IGB_FLAG_VLAN_PROMISC))
7477 igb_update_pf_vlvf(adapter, vid);
7478
7479 return err;
7480 }
7481
igb_set_vmvir(struct igb_adapter * adapter,u32 vid,u32 vf)7482 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
7483 {
7484 struct e1000_hw *hw = &adapter->hw;
7485
7486 if (vid)
7487 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
7488 else
7489 wr32(E1000_VMVIR(vf), 0);
7490 }
7491
igb_enable_port_vlan(struct igb_adapter * adapter,int vf,u16 vlan,u8 qos)7492 static int igb_enable_port_vlan(struct igb_adapter *adapter, int vf,
7493 u16 vlan, u8 qos)
7494 {
7495 int err;
7496
7497 err = igb_set_vf_vlan(adapter, vlan, true, vf);
7498 if (err)
7499 return err;
7500
7501 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
7502 igb_set_vmolr(adapter, vf, !vlan);
7503
7504 /* revoke access to previous VLAN */
7505 if (vlan != adapter->vf_data[vf].pf_vlan)
7506 igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
7507 false, vf);
7508
7509 adapter->vf_data[vf].pf_vlan = vlan;
7510 adapter->vf_data[vf].pf_qos = qos;
7511 igb_set_vf_vlan_strip(adapter, vf, true);
7512 dev_info(&adapter->pdev->dev,
7513 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
7514 if (test_bit(__IGB_DOWN, &adapter->state)) {
7515 dev_warn(&adapter->pdev->dev,
7516 "The VF VLAN has been set, but the PF device is not up.\n");
7517 dev_warn(&adapter->pdev->dev,
7518 "Bring the PF device up before attempting to use the VF device.\n");
7519 }
7520
7521 return err;
7522 }
7523
igb_disable_port_vlan(struct igb_adapter * adapter,int vf)7524 static int igb_disable_port_vlan(struct igb_adapter *adapter, int vf)
7525 {
7526 /* Restore tagless access via VLAN 0 */
7527 igb_set_vf_vlan(adapter, 0, true, vf);
7528
7529 igb_set_vmvir(adapter, 0, vf);
7530 igb_set_vmolr(adapter, vf, true);
7531
7532 /* Remove any PF assigned VLAN */
7533 if (adapter->vf_data[vf].pf_vlan)
7534 igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
7535 false, vf);
7536
7537 adapter->vf_data[vf].pf_vlan = 0;
7538 adapter->vf_data[vf].pf_qos = 0;
7539 igb_set_vf_vlan_strip(adapter, vf, false);
7540
7541 return 0;
7542 }
7543
igb_ndo_set_vf_vlan(struct net_device * netdev,int vf,u16 vlan,u8 qos,__be16 vlan_proto)7544 static int igb_ndo_set_vf_vlan(struct net_device *netdev, int vf,
7545 u16 vlan, u8 qos, __be16 vlan_proto)
7546 {
7547 struct igb_adapter *adapter = netdev_priv(netdev);
7548
7549 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
7550 return -EINVAL;
7551
7552 if (vlan_proto != htons(ETH_P_8021Q))
7553 return -EPROTONOSUPPORT;
7554
7555 return (vlan || qos) ? igb_enable_port_vlan(adapter, vf, vlan, qos) :
7556 igb_disable_port_vlan(adapter, vf);
7557 }
7558
igb_set_vf_vlan_msg(struct igb_adapter * adapter,u32 * msgbuf,u32 vf)7559 static int igb_set_vf_vlan_msg(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
7560 {
7561 int add = FIELD_GET(E1000_VT_MSGINFO_MASK, msgbuf[0]);
7562 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
7563 int ret;
7564
7565 if (adapter->vf_data[vf].pf_vlan)
7566 return -1;
7567
7568 /* VLAN 0 is a special case, don't allow it to be removed */
7569 if (!vid && !add)
7570 return 0;
7571
7572 ret = igb_set_vf_vlan(adapter, vid, !!add, vf);
7573 if (!ret)
7574 igb_set_vf_vlan_strip(adapter, vf, !!vid);
7575 return ret;
7576 }
7577
igb_vf_reset(struct igb_adapter * adapter,u32 vf)7578 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
7579 {
7580 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7581
7582 /* clear flags - except flag that indicates PF has set the MAC */
7583 vf_data->flags &= IGB_VF_FLAG_PF_SET_MAC;
7584 vf_data->last_nack = jiffies;
7585
7586 /* reset vlans for device */
7587 igb_clear_vf_vfta(adapter, vf);
7588 igb_set_vf_vlan(adapter, vf_data->pf_vlan, true, vf);
7589 igb_set_vmvir(adapter, vf_data->pf_vlan |
7590 (vf_data->pf_qos << VLAN_PRIO_SHIFT), vf);
7591 igb_set_vmolr(adapter, vf, !vf_data->pf_vlan);
7592 igb_set_vf_vlan_strip(adapter, vf, !!(vf_data->pf_vlan));
7593
7594 /* reset multicast table array for vf */
7595 adapter->vf_data[vf].num_vf_mc_hashes = 0;
7596
7597 /* Flush and reset the mta with the new values */
7598 igb_set_rx_mode(adapter->netdev);
7599 }
7600
igb_vf_reset_event(struct igb_adapter * adapter,u32 vf)7601 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
7602 {
7603 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
7604
7605 /* clear mac address as we were hotplug removed/added */
7606 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
7607 eth_zero_addr(vf_mac);
7608
7609 /* process remaining reset events */
7610 igb_vf_reset(adapter, vf);
7611 }
7612
igb_vf_reset_msg(struct igb_adapter * adapter,u32 vf)7613 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
7614 {
7615 struct e1000_hw *hw = &adapter->hw;
7616 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
7617 u32 reg, msgbuf[3] = {};
7618 u8 *addr = (u8 *)(&msgbuf[1]);
7619
7620 /* process all the same items cleared in a function level reset */
7621 igb_vf_reset(adapter, vf);
7622
7623 /* set vf mac address */
7624 igb_set_vf_mac(adapter, vf, vf_mac);
7625
7626 /* enable transmit and receive for vf */
7627 reg = rd32(E1000_VFTE);
7628 wr32(E1000_VFTE, reg | BIT(vf));
7629 reg = rd32(E1000_VFRE);
7630 wr32(E1000_VFRE, reg | BIT(vf));
7631
7632 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
7633
7634 /* reply to reset with ack and vf mac address */
7635 if (!is_zero_ether_addr(vf_mac)) {
7636 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
7637 memcpy(addr, vf_mac, ETH_ALEN);
7638 } else {
7639 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK;
7640 }
7641 igb_write_mbx(hw, msgbuf, 3, vf);
7642 }
7643
igb_flush_mac_table(struct igb_adapter * adapter)7644 static void igb_flush_mac_table(struct igb_adapter *adapter)
7645 {
7646 struct e1000_hw *hw = &adapter->hw;
7647 int i;
7648
7649 for (i = 0; i < hw->mac.rar_entry_count; i++) {
7650 adapter->mac_table[i].state &= ~IGB_MAC_STATE_IN_USE;
7651 eth_zero_addr(adapter->mac_table[i].addr);
7652 adapter->mac_table[i].queue = 0;
7653 igb_rar_set_index(adapter, i);
7654 }
7655 }
7656
igb_available_rars(struct igb_adapter * adapter,u8 queue)7657 static int igb_available_rars(struct igb_adapter *adapter, u8 queue)
7658 {
7659 struct e1000_hw *hw = &adapter->hw;
7660 /* do not count rar entries reserved for VFs MAC addresses */
7661 int rar_entries = hw->mac.rar_entry_count -
7662 adapter->vfs_allocated_count;
7663 int i, count = 0;
7664
7665 for (i = 0; i < rar_entries; i++) {
7666 /* do not count default entries */
7667 if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT)
7668 continue;
7669
7670 /* do not count "in use" entries for different queues */
7671 if ((adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE) &&
7672 (adapter->mac_table[i].queue != queue))
7673 continue;
7674
7675 count++;
7676 }
7677
7678 return count;
7679 }
7680
7681 /* Set default MAC address for the PF in the first RAR entry */
igb_set_default_mac_filter(struct igb_adapter * adapter)7682 static void igb_set_default_mac_filter(struct igb_adapter *adapter)
7683 {
7684 struct igb_mac_addr *mac_table = &adapter->mac_table[0];
7685
7686 ether_addr_copy(mac_table->addr, adapter->hw.mac.addr);
7687 mac_table->queue = adapter->vfs_allocated_count;
7688 mac_table->state = IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
7689
7690 igb_rar_set_index(adapter, 0);
7691 }
7692
7693 /* If the filter to be added and an already existing filter express
7694 * the same address and address type, it should be possible to only
7695 * override the other configurations, for example the queue to steer
7696 * traffic.
7697 */
igb_mac_entry_can_be_used(const struct igb_mac_addr * entry,const u8 * addr,const u8 flags)7698 static bool igb_mac_entry_can_be_used(const struct igb_mac_addr *entry,
7699 const u8 *addr, const u8 flags)
7700 {
7701 if (!(entry->state & IGB_MAC_STATE_IN_USE))
7702 return true;
7703
7704 if ((entry->state & IGB_MAC_STATE_SRC_ADDR) !=
7705 (flags & IGB_MAC_STATE_SRC_ADDR))
7706 return false;
7707
7708 if (!ether_addr_equal(addr, entry->addr))
7709 return false;
7710
7711 return true;
7712 }
7713
7714 /* Add a MAC filter for 'addr' directing matching traffic to 'queue',
7715 * 'flags' is used to indicate what kind of match is made, match is by
7716 * default for the destination address, if matching by source address
7717 * is desired the flag IGB_MAC_STATE_SRC_ADDR can be used.
7718 */
igb_add_mac_filter_flags(struct igb_adapter * adapter,const u8 * addr,const u8 queue,const u8 flags)7719 static int igb_add_mac_filter_flags(struct igb_adapter *adapter,
7720 const u8 *addr, const u8 queue,
7721 const u8 flags)
7722 {
7723 struct e1000_hw *hw = &adapter->hw;
7724 int rar_entries = hw->mac.rar_entry_count -
7725 adapter->vfs_allocated_count;
7726 int i;
7727
7728 if (is_zero_ether_addr(addr))
7729 return -EINVAL;
7730
7731 /* Search for the first empty entry in the MAC table.
7732 * Do not touch entries at the end of the table reserved for the VF MAC
7733 * addresses.
7734 */
7735 for (i = 0; i < rar_entries; i++) {
7736 if (!igb_mac_entry_can_be_used(&adapter->mac_table[i],
7737 addr, flags))
7738 continue;
7739
7740 ether_addr_copy(adapter->mac_table[i].addr, addr);
7741 adapter->mac_table[i].queue = queue;
7742 adapter->mac_table[i].state |= IGB_MAC_STATE_IN_USE | flags;
7743
7744 igb_rar_set_index(adapter, i);
7745 return i;
7746 }
7747
7748 return -ENOSPC;
7749 }
7750
igb_add_mac_filter(struct igb_adapter * adapter,const u8 * addr,const u8 queue)7751 static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr,
7752 const u8 queue)
7753 {
7754 return igb_add_mac_filter_flags(adapter, addr, queue, 0);
7755 }
7756
7757 /* Remove a MAC filter for 'addr' directing matching traffic to
7758 * 'queue', 'flags' is used to indicate what kind of match need to be
7759 * removed, match is by default for the destination address, if
7760 * matching by source address is to be removed the flag
7761 * IGB_MAC_STATE_SRC_ADDR can be used.
7762 */
igb_del_mac_filter_flags(struct igb_adapter * adapter,const u8 * addr,const u8 queue,const u8 flags)7763 static int igb_del_mac_filter_flags(struct igb_adapter *adapter,
7764 const u8 *addr, const u8 queue,
7765 const u8 flags)
7766 {
7767 struct e1000_hw *hw = &adapter->hw;
7768 int rar_entries = hw->mac.rar_entry_count -
7769 adapter->vfs_allocated_count;
7770 int i;
7771
7772 if (is_zero_ether_addr(addr))
7773 return -EINVAL;
7774
7775 /* Search for matching entry in the MAC table based on given address
7776 * and queue. Do not touch entries at the end of the table reserved
7777 * for the VF MAC addresses.
7778 */
7779 for (i = 0; i < rar_entries; i++) {
7780 if (!(adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE))
7781 continue;
7782 if ((adapter->mac_table[i].state & flags) != flags)
7783 continue;
7784 if (adapter->mac_table[i].queue != queue)
7785 continue;
7786 if (!ether_addr_equal(adapter->mac_table[i].addr, addr))
7787 continue;
7788
7789 /* When a filter for the default address is "deleted",
7790 * we return it to its initial configuration
7791 */
7792 if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) {
7793 adapter->mac_table[i].state =
7794 IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
7795 adapter->mac_table[i].queue =
7796 adapter->vfs_allocated_count;
7797 } else {
7798 adapter->mac_table[i].state = 0;
7799 adapter->mac_table[i].queue = 0;
7800 eth_zero_addr(adapter->mac_table[i].addr);
7801 }
7802
7803 igb_rar_set_index(adapter, i);
7804 return 0;
7805 }
7806
7807 return -ENOENT;
7808 }
7809
igb_del_mac_filter(struct igb_adapter * adapter,const u8 * addr,const u8 queue)7810 static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr,
7811 const u8 queue)
7812 {
7813 return igb_del_mac_filter_flags(adapter, addr, queue, 0);
7814 }
7815
igb_add_mac_steering_filter(struct igb_adapter * adapter,const u8 * addr,u8 queue,u8 flags)7816 int igb_add_mac_steering_filter(struct igb_adapter *adapter,
7817 const u8 *addr, u8 queue, u8 flags)
7818 {
7819 struct e1000_hw *hw = &adapter->hw;
7820
7821 /* In theory, this should be supported on 82575 as well, but
7822 * that part wasn't easily accessible during development.
7823 */
7824 if (hw->mac.type != e1000_i210)
7825 return -EOPNOTSUPP;
7826
7827 return igb_add_mac_filter_flags(adapter, addr, queue,
7828 IGB_MAC_STATE_QUEUE_STEERING | flags);
7829 }
7830
igb_del_mac_steering_filter(struct igb_adapter * adapter,const u8 * addr,u8 queue,u8 flags)7831 int igb_del_mac_steering_filter(struct igb_adapter *adapter,
7832 const u8 *addr, u8 queue, u8 flags)
7833 {
7834 return igb_del_mac_filter_flags(adapter, addr, queue,
7835 IGB_MAC_STATE_QUEUE_STEERING | flags);
7836 }
7837
igb_uc_sync(struct net_device * netdev,const unsigned char * addr)7838 static int igb_uc_sync(struct net_device *netdev, const unsigned char *addr)
7839 {
7840 struct igb_adapter *adapter = netdev_priv(netdev);
7841 int ret;
7842
7843 ret = igb_add_mac_filter(adapter, addr, adapter->vfs_allocated_count);
7844
7845 return min_t(int, ret, 0);
7846 }
7847
igb_uc_unsync(struct net_device * netdev,const unsigned char * addr)7848 static int igb_uc_unsync(struct net_device *netdev, const unsigned char *addr)
7849 {
7850 struct igb_adapter *adapter = netdev_priv(netdev);
7851
7852 igb_del_mac_filter(adapter, addr, adapter->vfs_allocated_count);
7853
7854 return 0;
7855 }
7856
igb_set_vf_mac_filter(struct igb_adapter * adapter,const int vf,const u32 info,const u8 * addr)7857 static int igb_set_vf_mac_filter(struct igb_adapter *adapter, const int vf,
7858 const u32 info, const u8 *addr)
7859 {
7860 struct pci_dev *pdev = adapter->pdev;
7861 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7862 struct list_head *pos;
7863 struct vf_mac_filter *entry = NULL;
7864 int ret = 0;
7865
7866 if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
7867 !vf_data->trusted) {
7868 dev_warn(&pdev->dev,
7869 "VF %d requested MAC filter but is administratively denied\n",
7870 vf);
7871 return -EINVAL;
7872 }
7873 if (!is_valid_ether_addr(addr)) {
7874 dev_warn(&pdev->dev,
7875 "VF %d attempted to set invalid MAC filter\n",
7876 vf);
7877 return -EINVAL;
7878 }
7879
7880 switch (info) {
7881 case E1000_VF_MAC_FILTER_CLR:
7882 /* remove all unicast MAC filters related to the current VF */
7883 list_for_each(pos, &adapter->vf_macs.l) {
7884 entry = list_entry(pos, struct vf_mac_filter, l);
7885 if (entry->vf == vf) {
7886 entry->vf = -1;
7887 entry->free = true;
7888 igb_del_mac_filter(adapter, entry->vf_mac, vf);
7889 }
7890 }
7891 break;
7892 case E1000_VF_MAC_FILTER_ADD:
7893 /* try to find empty slot in the list */
7894 list_for_each(pos, &adapter->vf_macs.l) {
7895 entry = list_entry(pos, struct vf_mac_filter, l);
7896 if (entry->free)
7897 break;
7898 }
7899
7900 if (entry && entry->free) {
7901 entry->free = false;
7902 entry->vf = vf;
7903 ether_addr_copy(entry->vf_mac, addr);
7904
7905 ret = igb_add_mac_filter(adapter, addr, vf);
7906 ret = min_t(int, ret, 0);
7907 } else {
7908 ret = -ENOSPC;
7909 }
7910
7911 if (ret == -ENOSPC)
7912 dev_warn(&pdev->dev,
7913 "VF %d has requested MAC filter but there is no space for it\n",
7914 vf);
7915 break;
7916 default:
7917 ret = -EINVAL;
7918 break;
7919 }
7920
7921 return ret;
7922 }
7923
igb_set_vf_mac_addr(struct igb_adapter * adapter,u32 * msg,int vf)7924 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
7925 {
7926 struct pci_dev *pdev = adapter->pdev;
7927 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7928 u32 info = msg[0] & E1000_VT_MSGINFO_MASK;
7929
7930 /* The VF MAC Address is stored in a packed array of bytes
7931 * starting at the second 32 bit word of the msg array
7932 */
7933 unsigned char *addr = (unsigned char *)&msg[1];
7934 int ret = 0;
7935
7936 if (!info) {
7937 if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
7938 !vf_data->trusted) {
7939 dev_warn(&pdev->dev,
7940 "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n",
7941 vf);
7942 return -EINVAL;
7943 }
7944
7945 if (!is_valid_ether_addr(addr)) {
7946 dev_warn(&pdev->dev,
7947 "VF %d attempted to set invalid MAC\n",
7948 vf);
7949 return -EINVAL;
7950 }
7951
7952 ret = igb_set_vf_mac(adapter, vf, addr);
7953 } else {
7954 ret = igb_set_vf_mac_filter(adapter, vf, info, addr);
7955 }
7956
7957 return ret;
7958 }
7959
igb_rcv_ack_from_vf(struct igb_adapter * adapter,u32 vf)7960 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
7961 {
7962 struct e1000_hw *hw = &adapter->hw;
7963 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7964 u32 msg = E1000_VT_MSGTYPE_NACK;
7965
7966 /* if device isn't clear to send it shouldn't be reading either */
7967 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
7968 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
7969 igb_write_mbx(hw, &msg, 1, vf);
7970 vf_data->last_nack = jiffies;
7971 }
7972 }
7973
igb_rcv_msg_from_vf(struct igb_adapter * adapter,u32 vf)7974 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
7975 {
7976 struct pci_dev *pdev = adapter->pdev;
7977 u32 msgbuf[E1000_VFMAILBOX_SIZE];
7978 struct e1000_hw *hw = &adapter->hw;
7979 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7980 s32 retval;
7981
7982 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf, false);
7983
7984 if (retval) {
7985 /* if receive failed revoke VF CTS stats and restart init */
7986 dev_err(&pdev->dev, "Error receiving message from VF\n");
7987 vf_data->flags &= ~IGB_VF_FLAG_CTS;
7988 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
7989 goto unlock;
7990 goto out;
7991 }
7992
7993 /* this is a message we already processed, do nothing */
7994 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
7995 goto unlock;
7996
7997 /* until the vf completes a reset it should not be
7998 * allowed to start any configuration.
7999 */
8000 if (msgbuf[0] == E1000_VF_RESET) {
8001 /* unlocks mailbox */
8002 igb_vf_reset_msg(adapter, vf);
8003 return;
8004 }
8005
8006 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
8007 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
8008 goto unlock;
8009 retval = -1;
8010 goto out;
8011 }
8012
8013 switch ((msgbuf[0] & 0xFFFF)) {
8014 case E1000_VF_SET_MAC_ADDR:
8015 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
8016 break;
8017 case E1000_VF_SET_PROMISC:
8018 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
8019 break;
8020 case E1000_VF_SET_MULTICAST:
8021 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
8022 break;
8023 case E1000_VF_SET_LPE:
8024 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
8025 break;
8026 case E1000_VF_SET_VLAN:
8027 retval = -1;
8028 if (vf_data->pf_vlan)
8029 dev_warn(&pdev->dev,
8030 "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n",
8031 vf);
8032 else
8033 retval = igb_set_vf_vlan_msg(adapter, msgbuf, vf);
8034 break;
8035 default:
8036 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
8037 retval = -1;
8038 break;
8039 }
8040
8041 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
8042 out:
8043 /* notify the VF of the results of what it sent us */
8044 if (retval)
8045 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
8046 else
8047 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
8048
8049 /* unlocks mailbox */
8050 igb_write_mbx(hw, msgbuf, 1, vf);
8051 return;
8052
8053 unlock:
8054 igb_unlock_mbx(hw, vf);
8055 }
8056
igb_msg_task(struct igb_adapter * adapter)8057 static void igb_msg_task(struct igb_adapter *adapter)
8058 {
8059 struct e1000_hw *hw = &adapter->hw;
8060 unsigned long flags;
8061 u32 vf;
8062
8063 spin_lock_irqsave(&adapter->vfs_lock, flags);
8064 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
8065 /* process any reset requests */
8066 if (!igb_check_for_rst(hw, vf))
8067 igb_vf_reset_event(adapter, vf);
8068
8069 /* process any messages pending */
8070 if (!igb_check_for_msg(hw, vf))
8071 igb_rcv_msg_from_vf(adapter, vf);
8072
8073 /* process any acks */
8074 if (!igb_check_for_ack(hw, vf))
8075 igb_rcv_ack_from_vf(adapter, vf);
8076 }
8077 spin_unlock_irqrestore(&adapter->vfs_lock, flags);
8078 }
8079
8080 /**
8081 * igb_set_uta - Set unicast filter table address
8082 * @adapter: board private structure
8083 * @set: boolean indicating if we are setting or clearing bits
8084 *
8085 * The unicast table address is a register array of 32-bit registers.
8086 * The table is meant to be used in a way similar to how the MTA is used
8087 * however due to certain limitations in the hardware it is necessary to
8088 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
8089 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
8090 **/
igb_set_uta(struct igb_adapter * adapter,bool set)8091 static void igb_set_uta(struct igb_adapter *adapter, bool set)
8092 {
8093 struct e1000_hw *hw = &adapter->hw;
8094 u32 uta = set ? ~0 : 0;
8095 int i;
8096
8097 /* we only need to do this if VMDq is enabled */
8098 if (!adapter->vfs_allocated_count)
8099 return;
8100
8101 for (i = hw->mac.uta_reg_count; i--;)
8102 array_wr32(E1000_UTA, i, uta);
8103 }
8104
8105 /**
8106 * igb_intr_msi - Interrupt Handler
8107 * @irq: interrupt number
8108 * @data: pointer to a network interface device structure
8109 **/
igb_intr_msi(int irq,void * data)8110 static irqreturn_t igb_intr_msi(int irq, void *data)
8111 {
8112 struct igb_adapter *adapter = data;
8113 struct igb_q_vector *q_vector = adapter->q_vector[0];
8114 struct e1000_hw *hw = &adapter->hw;
8115 /* read ICR disables interrupts using IAM */
8116 u32 icr = rd32(E1000_ICR);
8117
8118 igb_write_itr(q_vector);
8119
8120 if (icr & E1000_ICR_DRSTA)
8121 schedule_work(&adapter->reset_task);
8122
8123 if (icr & E1000_ICR_DOUTSYNC) {
8124 /* HW is reporting DMA is out of sync */
8125 adapter->stats.doosync++;
8126 }
8127
8128 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
8129 hw->mac.get_link_status = 1;
8130 if (!test_bit(__IGB_DOWN, &adapter->state))
8131 mod_timer(&adapter->watchdog_timer, jiffies + 1);
8132 }
8133
8134 if (icr & E1000_ICR_TS)
8135 igb_tsync_interrupt(adapter);
8136
8137 napi_schedule(&q_vector->napi);
8138
8139 return IRQ_HANDLED;
8140 }
8141
8142 /**
8143 * igb_intr - Legacy Interrupt Handler
8144 * @irq: interrupt number
8145 * @data: pointer to a network interface device structure
8146 **/
igb_intr(int irq,void * data)8147 static irqreturn_t igb_intr(int irq, void *data)
8148 {
8149 struct igb_adapter *adapter = data;
8150 struct igb_q_vector *q_vector = adapter->q_vector[0];
8151 struct e1000_hw *hw = &adapter->hw;
8152 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
8153 * need for the IMC write
8154 */
8155 u32 icr = rd32(E1000_ICR);
8156
8157 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
8158 * not set, then the adapter didn't send an interrupt
8159 */
8160 if (!(icr & E1000_ICR_INT_ASSERTED))
8161 return IRQ_NONE;
8162
8163 igb_write_itr(q_vector);
8164
8165 if (icr & E1000_ICR_DRSTA)
8166 schedule_work(&adapter->reset_task);
8167
8168 if (icr & E1000_ICR_DOUTSYNC) {
8169 /* HW is reporting DMA is out of sync */
8170 adapter->stats.doosync++;
8171 }
8172
8173 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
8174 hw->mac.get_link_status = 1;
8175 /* guard against interrupt when we're going down */
8176 if (!test_bit(__IGB_DOWN, &adapter->state))
8177 mod_timer(&adapter->watchdog_timer, jiffies + 1);
8178 }
8179
8180 if (icr & E1000_ICR_TS)
8181 igb_tsync_interrupt(adapter);
8182
8183 napi_schedule(&q_vector->napi);
8184
8185 return IRQ_HANDLED;
8186 }
8187
igb_ring_irq_enable(struct igb_q_vector * q_vector)8188 static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
8189 {
8190 struct igb_adapter *adapter = q_vector->adapter;
8191 struct e1000_hw *hw = &adapter->hw;
8192
8193 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
8194 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
8195 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
8196 igb_set_itr(q_vector);
8197 else
8198 igb_update_ring_itr(q_vector);
8199 }
8200
8201 if (!test_bit(__IGB_DOWN, &adapter->state)) {
8202 if (adapter->flags & IGB_FLAG_HAS_MSIX)
8203 wr32(E1000_EIMS, q_vector->eims_value);
8204 else
8205 igb_irq_enable(adapter);
8206 }
8207 }
8208
8209 /**
8210 * igb_poll - NAPI Rx polling callback
8211 * @napi: napi polling structure
8212 * @budget: count of how many packets we should handle
8213 **/
igb_poll(struct napi_struct * napi,int budget)8214 static int igb_poll(struct napi_struct *napi, int budget)
8215 {
8216 struct igb_q_vector *q_vector = container_of(napi,
8217 struct igb_q_vector,
8218 napi);
8219 bool clean_complete = true;
8220 int work_done = 0;
8221
8222 #ifdef CONFIG_IGB_DCA
8223 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
8224 igb_update_dca(q_vector);
8225 #endif
8226 if (q_vector->tx.ring)
8227 clean_complete = igb_clean_tx_irq(q_vector, budget);
8228
8229 if (q_vector->rx.ring) {
8230 int cleaned = igb_clean_rx_irq(q_vector, budget);
8231
8232 work_done += cleaned;
8233 if (cleaned >= budget)
8234 clean_complete = false;
8235 }
8236
8237 /* If all work not completed, return budget and keep polling */
8238 if (!clean_complete)
8239 return budget;
8240
8241 /* Exit the polling mode, but don't re-enable interrupts if stack might
8242 * poll us due to busy-polling
8243 */
8244 if (likely(napi_complete_done(napi, work_done)))
8245 igb_ring_irq_enable(q_vector);
8246
8247 return work_done;
8248 }
8249
8250 /**
8251 * igb_clean_tx_irq - Reclaim resources after transmit completes
8252 * @q_vector: pointer to q_vector containing needed info
8253 * @napi_budget: Used to determine if we are in netpoll
8254 *
8255 * returns true if ring is completely cleaned
8256 **/
igb_clean_tx_irq(struct igb_q_vector * q_vector,int napi_budget)8257 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector, int napi_budget)
8258 {
8259 struct igb_adapter *adapter = q_vector->adapter;
8260 struct igb_ring *tx_ring = q_vector->tx.ring;
8261 struct igb_tx_buffer *tx_buffer;
8262 union e1000_adv_tx_desc *tx_desc;
8263 unsigned int total_bytes = 0, total_packets = 0;
8264 unsigned int budget = q_vector->tx.work_limit;
8265 unsigned int i = tx_ring->next_to_clean;
8266
8267 if (test_bit(__IGB_DOWN, &adapter->state))
8268 return true;
8269
8270 tx_buffer = &tx_ring->tx_buffer_info[i];
8271 tx_desc = IGB_TX_DESC(tx_ring, i);
8272 i -= tx_ring->count;
8273
8274 do {
8275 union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
8276
8277 /* if next_to_watch is not set then there is no work pending */
8278 if (!eop_desc)
8279 break;
8280
8281 /* prevent any other reads prior to eop_desc */
8282 smp_rmb();
8283
8284 /* if DD is not set pending work has not been completed */
8285 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
8286 break;
8287
8288 /* clear next_to_watch to prevent false hangs */
8289 tx_buffer->next_to_watch = NULL;
8290
8291 /* update the statistics for this packet */
8292 total_bytes += tx_buffer->bytecount;
8293 total_packets += tx_buffer->gso_segs;
8294
8295 /* free the skb */
8296 if (tx_buffer->type == IGB_TYPE_SKB)
8297 napi_consume_skb(tx_buffer->skb, napi_budget);
8298 else
8299 xdp_return_frame(tx_buffer->xdpf);
8300
8301 /* unmap skb header data */
8302 dma_unmap_single(tx_ring->dev,
8303 dma_unmap_addr(tx_buffer, dma),
8304 dma_unmap_len(tx_buffer, len),
8305 DMA_TO_DEVICE);
8306
8307 /* clear tx_buffer data */
8308 dma_unmap_len_set(tx_buffer, len, 0);
8309
8310 /* clear last DMA location and unmap remaining buffers */
8311 while (tx_desc != eop_desc) {
8312 tx_buffer++;
8313 tx_desc++;
8314 i++;
8315 if (unlikely(!i)) {
8316 i -= tx_ring->count;
8317 tx_buffer = tx_ring->tx_buffer_info;
8318 tx_desc = IGB_TX_DESC(tx_ring, 0);
8319 }
8320
8321 /* unmap any remaining paged data */
8322 if (dma_unmap_len(tx_buffer, len)) {
8323 dma_unmap_page(tx_ring->dev,
8324 dma_unmap_addr(tx_buffer, dma),
8325 dma_unmap_len(tx_buffer, len),
8326 DMA_TO_DEVICE);
8327 dma_unmap_len_set(tx_buffer, len, 0);
8328 }
8329 }
8330
8331 /* move us one more past the eop_desc for start of next pkt */
8332 tx_buffer++;
8333 tx_desc++;
8334 i++;
8335 if (unlikely(!i)) {
8336 i -= tx_ring->count;
8337 tx_buffer = tx_ring->tx_buffer_info;
8338 tx_desc = IGB_TX_DESC(tx_ring, 0);
8339 }
8340
8341 /* issue prefetch for next Tx descriptor */
8342 prefetch(tx_desc);
8343
8344 /* update budget accounting */
8345 budget--;
8346 } while (likely(budget));
8347
8348 netdev_tx_completed_queue(txring_txq(tx_ring),
8349 total_packets, total_bytes);
8350 i += tx_ring->count;
8351 tx_ring->next_to_clean = i;
8352 u64_stats_update_begin(&tx_ring->tx_syncp);
8353 tx_ring->tx_stats.bytes += total_bytes;
8354 tx_ring->tx_stats.packets += total_packets;
8355 u64_stats_update_end(&tx_ring->tx_syncp);
8356 q_vector->tx.total_bytes += total_bytes;
8357 q_vector->tx.total_packets += total_packets;
8358
8359 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
8360 struct e1000_hw *hw = &adapter->hw;
8361
8362 /* Detect a transmit hang in hardware, this serializes the
8363 * check with the clearing of time_stamp and movement of i
8364 */
8365 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
8366 if (tx_buffer->next_to_watch &&
8367 time_after(jiffies, tx_buffer->time_stamp +
8368 (adapter->tx_timeout_factor * HZ)) &&
8369 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
8370
8371 /* detected Tx unit hang */
8372 dev_err(tx_ring->dev,
8373 "Detected Tx Unit Hang\n"
8374 " Tx Queue <%d>\n"
8375 " TDH <%x>\n"
8376 " TDT <%x>\n"
8377 " next_to_use <%x>\n"
8378 " next_to_clean <%x>\n"
8379 "buffer_info[next_to_clean]\n"
8380 " time_stamp <%lx>\n"
8381 " next_to_watch <%p>\n"
8382 " jiffies <%lx>\n"
8383 " desc.status <%x>\n",
8384 tx_ring->queue_index,
8385 rd32(E1000_TDH(tx_ring->reg_idx)),
8386 readl(tx_ring->tail),
8387 tx_ring->next_to_use,
8388 tx_ring->next_to_clean,
8389 tx_buffer->time_stamp,
8390 tx_buffer->next_to_watch,
8391 jiffies,
8392 tx_buffer->next_to_watch->wb.status);
8393 netif_stop_subqueue(tx_ring->netdev,
8394 tx_ring->queue_index);
8395
8396 /* we are about to reset, no point in enabling stuff */
8397 return true;
8398 }
8399 }
8400
8401 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
8402 if (unlikely(total_packets &&
8403 netif_carrier_ok(tx_ring->netdev) &&
8404 igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
8405 /* Make sure that anybody stopping the queue after this
8406 * sees the new next_to_clean.
8407 */
8408 smp_mb();
8409 if (__netif_subqueue_stopped(tx_ring->netdev,
8410 tx_ring->queue_index) &&
8411 !(test_bit(__IGB_DOWN, &adapter->state))) {
8412 netif_wake_subqueue(tx_ring->netdev,
8413 tx_ring->queue_index);
8414
8415 u64_stats_update_begin(&tx_ring->tx_syncp);
8416 tx_ring->tx_stats.restart_queue++;
8417 u64_stats_update_end(&tx_ring->tx_syncp);
8418 }
8419 }
8420
8421 return !!budget;
8422 }
8423
8424 /**
8425 * igb_reuse_rx_page - page flip buffer and store it back on the ring
8426 * @rx_ring: rx descriptor ring to store buffers on
8427 * @old_buff: donor buffer to have page reused
8428 *
8429 * Synchronizes page for reuse by the adapter
8430 **/
igb_reuse_rx_page(struct igb_ring * rx_ring,struct igb_rx_buffer * old_buff)8431 static void igb_reuse_rx_page(struct igb_ring *rx_ring,
8432 struct igb_rx_buffer *old_buff)
8433 {
8434 struct igb_rx_buffer *new_buff;
8435 u16 nta = rx_ring->next_to_alloc;
8436
8437 new_buff = &rx_ring->rx_buffer_info[nta];
8438
8439 /* update, and store next to alloc */
8440 nta++;
8441 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
8442
8443 /* Transfer page from old buffer to new buffer.
8444 * Move each member individually to avoid possible store
8445 * forwarding stalls.
8446 */
8447 new_buff->dma = old_buff->dma;
8448 new_buff->page = old_buff->page;
8449 new_buff->page_offset = old_buff->page_offset;
8450 new_buff->pagecnt_bias = old_buff->pagecnt_bias;
8451 }
8452
igb_can_reuse_rx_page(struct igb_rx_buffer * rx_buffer,int rx_buf_pgcnt)8453 static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
8454 int rx_buf_pgcnt)
8455 {
8456 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
8457 struct page *page = rx_buffer->page;
8458
8459 /* avoid re-using remote and pfmemalloc pages */
8460 if (!dev_page_is_reusable(page))
8461 return false;
8462
8463 #if (PAGE_SIZE < 8192)
8464 /* if we are only owner of page we can reuse it */
8465 if (unlikely((rx_buf_pgcnt - pagecnt_bias) > 1))
8466 return false;
8467 #else
8468 #define IGB_LAST_OFFSET \
8469 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048)
8470
8471 if (rx_buffer->page_offset > IGB_LAST_OFFSET)
8472 return false;
8473 #endif
8474
8475 /* If we have drained the page fragment pool we need to update
8476 * the pagecnt_bias and page count so that we fully restock the
8477 * number of references the driver holds.
8478 */
8479 if (unlikely(pagecnt_bias == 1)) {
8480 page_ref_add(page, USHRT_MAX - 1);
8481 rx_buffer->pagecnt_bias = USHRT_MAX;
8482 }
8483
8484 return true;
8485 }
8486
8487 /**
8488 * igb_add_rx_frag - Add contents of Rx buffer to sk_buff
8489 * @rx_ring: rx descriptor ring to transact packets on
8490 * @rx_buffer: buffer containing page to add
8491 * @skb: sk_buff to place the data into
8492 * @size: size of buffer to be added
8493 *
8494 * This function will add the data contained in rx_buffer->page to the skb.
8495 **/
igb_add_rx_frag(struct igb_ring * rx_ring,struct igb_rx_buffer * rx_buffer,struct sk_buff * skb,unsigned int size)8496 static void igb_add_rx_frag(struct igb_ring *rx_ring,
8497 struct igb_rx_buffer *rx_buffer,
8498 struct sk_buff *skb,
8499 unsigned int size)
8500 {
8501 #if (PAGE_SIZE < 8192)
8502 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8503 #else
8504 unsigned int truesize = ring_uses_build_skb(rx_ring) ?
8505 SKB_DATA_ALIGN(IGB_SKB_PAD + size) :
8506 SKB_DATA_ALIGN(size);
8507 #endif
8508 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
8509 rx_buffer->page_offset, size, truesize);
8510 #if (PAGE_SIZE < 8192)
8511 rx_buffer->page_offset ^= truesize;
8512 #else
8513 rx_buffer->page_offset += truesize;
8514 #endif
8515 }
8516
igb_construct_skb(struct igb_ring * rx_ring,struct igb_rx_buffer * rx_buffer,struct xdp_buff * xdp,ktime_t timestamp)8517 static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring,
8518 struct igb_rx_buffer *rx_buffer,
8519 struct xdp_buff *xdp,
8520 ktime_t timestamp)
8521 {
8522 #if (PAGE_SIZE < 8192)
8523 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8524 #else
8525 unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end -
8526 xdp->data_hard_start);
8527 #endif
8528 unsigned int size = xdp->data_end - xdp->data;
8529 unsigned int headlen;
8530 struct sk_buff *skb;
8531
8532 /* prefetch first cache line of first page */
8533 net_prefetch(xdp->data);
8534
8535 /* allocate a skb to store the frags */
8536 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);
8537 if (unlikely(!skb))
8538 return NULL;
8539
8540 if (timestamp)
8541 skb_hwtstamps(skb)->hwtstamp = timestamp;
8542
8543 /* Determine available headroom for copy */
8544 headlen = size;
8545 if (headlen > IGB_RX_HDR_LEN)
8546 headlen = eth_get_headlen(skb->dev, xdp->data, IGB_RX_HDR_LEN);
8547
8548 /* align pull length to size of long to optimize memcpy performance */
8549 memcpy(__skb_put(skb, headlen), xdp->data, ALIGN(headlen, sizeof(long)));
8550
8551 /* update all of the pointers */
8552 size -= headlen;
8553 if (size) {
8554 skb_add_rx_frag(skb, 0, rx_buffer->page,
8555 (xdp->data + headlen) - page_address(rx_buffer->page),
8556 size, truesize);
8557 #if (PAGE_SIZE < 8192)
8558 rx_buffer->page_offset ^= truesize;
8559 #else
8560 rx_buffer->page_offset += truesize;
8561 #endif
8562 } else {
8563 rx_buffer->pagecnt_bias++;
8564 }
8565
8566 return skb;
8567 }
8568
igb_build_skb(struct igb_ring * rx_ring,struct igb_rx_buffer * rx_buffer,struct xdp_buff * xdp,ktime_t timestamp)8569 static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring,
8570 struct igb_rx_buffer *rx_buffer,
8571 struct xdp_buff *xdp,
8572 ktime_t timestamp)
8573 {
8574 #if (PAGE_SIZE < 8192)
8575 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8576 #else
8577 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
8578 SKB_DATA_ALIGN(xdp->data_end -
8579 xdp->data_hard_start);
8580 #endif
8581 unsigned int metasize = xdp->data - xdp->data_meta;
8582 struct sk_buff *skb;
8583
8584 /* prefetch first cache line of first page */
8585 net_prefetch(xdp->data_meta);
8586
8587 /* build an skb around the page buffer */
8588 skb = napi_build_skb(xdp->data_hard_start, truesize);
8589 if (unlikely(!skb))
8590 return NULL;
8591
8592 /* update pointers within the skb to store the data */
8593 skb_reserve(skb, xdp->data - xdp->data_hard_start);
8594 __skb_put(skb, xdp->data_end - xdp->data);
8595
8596 if (metasize)
8597 skb_metadata_set(skb, metasize);
8598
8599 if (timestamp)
8600 skb_hwtstamps(skb)->hwtstamp = timestamp;
8601
8602 /* update buffer offset */
8603 #if (PAGE_SIZE < 8192)
8604 rx_buffer->page_offset ^= truesize;
8605 #else
8606 rx_buffer->page_offset += truesize;
8607 #endif
8608
8609 return skb;
8610 }
8611
igb_run_xdp(struct igb_adapter * adapter,struct igb_ring * rx_ring,struct xdp_buff * xdp)8612 static struct sk_buff *igb_run_xdp(struct igb_adapter *adapter,
8613 struct igb_ring *rx_ring,
8614 struct xdp_buff *xdp)
8615 {
8616 int err, result = IGB_XDP_PASS;
8617 struct bpf_prog *xdp_prog;
8618 u32 act;
8619
8620 xdp_prog = READ_ONCE(rx_ring->xdp_prog);
8621
8622 if (!xdp_prog)
8623 goto xdp_out;
8624
8625 prefetchw(xdp->data_hard_start); /* xdp_frame write */
8626
8627 act = bpf_prog_run_xdp(xdp_prog, xdp);
8628 switch (act) {
8629 case XDP_PASS:
8630 break;
8631 case XDP_TX:
8632 result = igb_xdp_xmit_back(adapter, xdp);
8633 if (result == IGB_XDP_CONSUMED)
8634 goto out_failure;
8635 break;
8636 case XDP_REDIRECT:
8637 err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog);
8638 if (err)
8639 goto out_failure;
8640 result = IGB_XDP_REDIR;
8641 break;
8642 default:
8643 bpf_warn_invalid_xdp_action(adapter->netdev, xdp_prog, act);
8644 fallthrough;
8645 case XDP_ABORTED:
8646 out_failure:
8647 trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
8648 fallthrough;
8649 case XDP_DROP:
8650 result = IGB_XDP_CONSUMED;
8651 break;
8652 }
8653 xdp_out:
8654 return ERR_PTR(-result);
8655 }
8656
igb_rx_frame_truesize(struct igb_ring * rx_ring,unsigned int size)8657 static unsigned int igb_rx_frame_truesize(struct igb_ring *rx_ring,
8658 unsigned int size)
8659 {
8660 unsigned int truesize;
8661
8662 #if (PAGE_SIZE < 8192)
8663 truesize = igb_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */
8664 #else
8665 truesize = ring_uses_build_skb(rx_ring) ?
8666 SKB_DATA_ALIGN(IGB_SKB_PAD + size) +
8667 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
8668 SKB_DATA_ALIGN(size);
8669 #endif
8670 return truesize;
8671 }
8672
igb_rx_buffer_flip(struct igb_ring * rx_ring,struct igb_rx_buffer * rx_buffer,unsigned int size)8673 static void igb_rx_buffer_flip(struct igb_ring *rx_ring,
8674 struct igb_rx_buffer *rx_buffer,
8675 unsigned int size)
8676 {
8677 unsigned int truesize = igb_rx_frame_truesize(rx_ring, size);
8678 #if (PAGE_SIZE < 8192)
8679 rx_buffer->page_offset ^= truesize;
8680 #else
8681 rx_buffer->page_offset += truesize;
8682 #endif
8683 }
8684
igb_rx_checksum(struct igb_ring * ring,union e1000_adv_rx_desc * rx_desc,struct sk_buff * skb)8685 static inline void igb_rx_checksum(struct igb_ring *ring,
8686 union e1000_adv_rx_desc *rx_desc,
8687 struct sk_buff *skb)
8688 {
8689 skb_checksum_none_assert(skb);
8690
8691 /* Ignore Checksum bit is set */
8692 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
8693 return;
8694
8695 /* Rx checksum disabled via ethtool */
8696 if (!(ring->netdev->features & NETIF_F_RXCSUM))
8697 return;
8698
8699 /* TCP/UDP checksum error bit is set */
8700 if (igb_test_staterr(rx_desc,
8701 E1000_RXDEXT_STATERR_TCPE |
8702 E1000_RXDEXT_STATERR_IPE)) {
8703 /* work around errata with sctp packets where the TCPE aka
8704 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
8705 * packets, (aka let the stack check the crc32c)
8706 */
8707 if (!((skb->len == 60) &&
8708 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
8709 u64_stats_update_begin(&ring->rx_syncp);
8710 ring->rx_stats.csum_err++;
8711 u64_stats_update_end(&ring->rx_syncp);
8712 }
8713 /* let the stack verify checksum errors */
8714 return;
8715 }
8716 /* It must be a TCP or UDP packet with a valid checksum */
8717 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
8718 E1000_RXD_STAT_UDPCS))
8719 skb->ip_summed = CHECKSUM_UNNECESSARY;
8720
8721 dev_dbg(ring->dev, "cksum success: bits %08X\n",
8722 le32_to_cpu(rx_desc->wb.upper.status_error));
8723 }
8724
igb_rx_hash(struct igb_ring * ring,union e1000_adv_rx_desc * rx_desc,struct sk_buff * skb)8725 static inline void igb_rx_hash(struct igb_ring *ring,
8726 union e1000_adv_rx_desc *rx_desc,
8727 struct sk_buff *skb)
8728 {
8729 if (ring->netdev->features & NETIF_F_RXHASH)
8730 skb_set_hash(skb,
8731 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
8732 PKT_HASH_TYPE_L3);
8733 }
8734
8735 /**
8736 * igb_is_non_eop - process handling of non-EOP buffers
8737 * @rx_ring: Rx ring being processed
8738 * @rx_desc: Rx descriptor for current buffer
8739 *
8740 * This function updates next to clean. If the buffer is an EOP buffer
8741 * this function exits returning false, otherwise it will place the
8742 * sk_buff in the next buffer to be chained and return true indicating
8743 * that this is in fact a non-EOP buffer.
8744 **/
igb_is_non_eop(struct igb_ring * rx_ring,union e1000_adv_rx_desc * rx_desc)8745 static bool igb_is_non_eop(struct igb_ring *rx_ring,
8746 union e1000_adv_rx_desc *rx_desc)
8747 {
8748 u32 ntc = rx_ring->next_to_clean + 1;
8749
8750 /* fetch, update, and store next to clean */
8751 ntc = (ntc < rx_ring->count) ? ntc : 0;
8752 rx_ring->next_to_clean = ntc;
8753
8754 prefetch(IGB_RX_DESC(rx_ring, ntc));
8755
8756 if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)))
8757 return false;
8758
8759 return true;
8760 }
8761
8762 /**
8763 * igb_cleanup_headers - Correct corrupted or empty headers
8764 * @rx_ring: rx descriptor ring packet is being transacted on
8765 * @rx_desc: pointer to the EOP Rx descriptor
8766 * @skb: pointer to current skb being fixed
8767 *
8768 * Address the case where we are pulling data in on pages only
8769 * and as such no data is present in the skb header.
8770 *
8771 * In addition if skb is not at least 60 bytes we need to pad it so that
8772 * it is large enough to qualify as a valid Ethernet frame.
8773 *
8774 * Returns true if an error was encountered and skb was freed.
8775 **/
igb_cleanup_headers(struct igb_ring * rx_ring,union e1000_adv_rx_desc * rx_desc,struct sk_buff * skb)8776 static bool igb_cleanup_headers(struct igb_ring *rx_ring,
8777 union e1000_adv_rx_desc *rx_desc,
8778 struct sk_buff *skb)
8779 {
8780 /* XDP packets use error pointer so abort at this point */
8781 if (IS_ERR(skb))
8782 return true;
8783
8784 if (unlikely((igb_test_staterr(rx_desc,
8785 E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) {
8786 struct net_device *netdev = rx_ring->netdev;
8787 if (!(netdev->features & NETIF_F_RXALL)) {
8788 dev_kfree_skb_any(skb);
8789 return true;
8790 }
8791 }
8792
8793 /* if eth_skb_pad returns an error the skb was freed */
8794 if (eth_skb_pad(skb))
8795 return true;
8796
8797 return false;
8798 }
8799
8800 /**
8801 * igb_process_skb_fields - Populate skb header fields from Rx descriptor
8802 * @rx_ring: rx descriptor ring packet is being transacted on
8803 * @rx_desc: pointer to the EOP Rx descriptor
8804 * @skb: pointer to current skb being populated
8805 *
8806 * This function checks the ring, descriptor, and packet information in
8807 * order to populate the hash, checksum, VLAN, timestamp, protocol, and
8808 * other fields within the skb.
8809 **/
igb_process_skb_fields(struct igb_ring * rx_ring,union e1000_adv_rx_desc * rx_desc,struct sk_buff * skb)8810 static void igb_process_skb_fields(struct igb_ring *rx_ring,
8811 union e1000_adv_rx_desc *rx_desc,
8812 struct sk_buff *skb)
8813 {
8814 struct net_device *dev = rx_ring->netdev;
8815
8816 igb_rx_hash(rx_ring, rx_desc, skb);
8817
8818 igb_rx_checksum(rx_ring, rx_desc, skb);
8819
8820 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TS) &&
8821 !igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))
8822 igb_ptp_rx_rgtstamp(rx_ring->q_vector, skb);
8823
8824 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
8825 igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
8826 u16 vid;
8827
8828 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
8829 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
8830 vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan);
8831 else
8832 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
8833
8834 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
8835 }
8836
8837 skb_record_rx_queue(skb, rx_ring->queue_index);
8838
8839 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
8840 }
8841
igb_rx_offset(struct igb_ring * rx_ring)8842 static unsigned int igb_rx_offset(struct igb_ring *rx_ring)
8843 {
8844 return ring_uses_build_skb(rx_ring) ? IGB_SKB_PAD : 0;
8845 }
8846
igb_get_rx_buffer(struct igb_ring * rx_ring,const unsigned int size,int * rx_buf_pgcnt)8847 static struct igb_rx_buffer *igb_get_rx_buffer(struct igb_ring *rx_ring,
8848 const unsigned int size, int *rx_buf_pgcnt)
8849 {
8850 struct igb_rx_buffer *rx_buffer;
8851
8852 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
8853 *rx_buf_pgcnt =
8854 #if (PAGE_SIZE < 8192)
8855 page_count(rx_buffer->page);
8856 #else
8857 0;
8858 #endif
8859 prefetchw(rx_buffer->page);
8860
8861 /* we are reusing so sync this buffer for CPU use */
8862 dma_sync_single_range_for_cpu(rx_ring->dev,
8863 rx_buffer->dma,
8864 rx_buffer->page_offset,
8865 size,
8866 DMA_FROM_DEVICE);
8867
8868 rx_buffer->pagecnt_bias--;
8869
8870 return rx_buffer;
8871 }
8872
igb_put_rx_buffer(struct igb_ring * rx_ring,struct igb_rx_buffer * rx_buffer,int rx_buf_pgcnt)8873 static void igb_put_rx_buffer(struct igb_ring *rx_ring,
8874 struct igb_rx_buffer *rx_buffer, int rx_buf_pgcnt)
8875 {
8876 if (igb_can_reuse_rx_page(rx_buffer, rx_buf_pgcnt)) {
8877 /* hand second half of page back to the ring */
8878 igb_reuse_rx_page(rx_ring, rx_buffer);
8879 } else {
8880 /* We are not reusing the buffer so unmap it and free
8881 * any references we are holding to it
8882 */
8883 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
8884 igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
8885 IGB_RX_DMA_ATTR);
8886 __page_frag_cache_drain(rx_buffer->page,
8887 rx_buffer->pagecnt_bias);
8888 }
8889
8890 /* clear contents of rx_buffer */
8891 rx_buffer->page = NULL;
8892 }
8893
igb_clean_rx_irq(struct igb_q_vector * q_vector,const int budget)8894 static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
8895 {
8896 unsigned int total_bytes = 0, total_packets = 0;
8897 struct igb_adapter *adapter = q_vector->adapter;
8898 struct igb_ring *rx_ring = q_vector->rx.ring;
8899 u16 cleaned_count = igb_desc_unused(rx_ring);
8900 struct sk_buff *skb = rx_ring->skb;
8901 int cpu = smp_processor_id();
8902 unsigned int xdp_xmit = 0;
8903 struct netdev_queue *nq;
8904 struct xdp_buff xdp;
8905 u32 frame_sz = 0;
8906 int rx_buf_pgcnt;
8907
8908 /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */
8909 #if (PAGE_SIZE < 8192)
8910 frame_sz = igb_rx_frame_truesize(rx_ring, 0);
8911 #endif
8912 xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq);
8913
8914 while (likely(total_packets < budget)) {
8915 union e1000_adv_rx_desc *rx_desc;
8916 struct igb_rx_buffer *rx_buffer;
8917 ktime_t timestamp = 0;
8918 int pkt_offset = 0;
8919 unsigned int size;
8920 void *pktbuf;
8921
8922 /* return some buffers to hardware, one at a time is too slow */
8923 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
8924 igb_alloc_rx_buffers(rx_ring, cleaned_count);
8925 cleaned_count = 0;
8926 }
8927
8928 rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);
8929 size = le16_to_cpu(rx_desc->wb.upper.length);
8930 if (!size)
8931 break;
8932
8933 /* This memory barrier is needed to keep us from reading
8934 * any other fields out of the rx_desc until we know the
8935 * descriptor has been written back
8936 */
8937 dma_rmb();
8938
8939 rx_buffer = igb_get_rx_buffer(rx_ring, size, &rx_buf_pgcnt);
8940 pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset;
8941
8942 /* pull rx packet timestamp if available and valid */
8943 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
8944 int ts_hdr_len;
8945
8946 ts_hdr_len = igb_ptp_rx_pktstamp(rx_ring->q_vector,
8947 pktbuf, ×tamp);
8948
8949 pkt_offset += ts_hdr_len;
8950 size -= ts_hdr_len;
8951 }
8952
8953 /* retrieve a buffer from the ring */
8954 if (!skb) {
8955 unsigned char *hard_start = pktbuf - igb_rx_offset(rx_ring);
8956 unsigned int offset = pkt_offset + igb_rx_offset(rx_ring);
8957
8958 xdp_prepare_buff(&xdp, hard_start, offset, size, true);
8959 xdp_buff_clear_frags_flag(&xdp);
8960 #if (PAGE_SIZE > 4096)
8961 /* At larger PAGE_SIZE, frame_sz depend on len size */
8962 xdp.frame_sz = igb_rx_frame_truesize(rx_ring, size);
8963 #endif
8964 skb = igb_run_xdp(adapter, rx_ring, &xdp);
8965 }
8966
8967 if (IS_ERR(skb)) {
8968 unsigned int xdp_res = -PTR_ERR(skb);
8969
8970 if (xdp_res & (IGB_XDP_TX | IGB_XDP_REDIR)) {
8971 xdp_xmit |= xdp_res;
8972 igb_rx_buffer_flip(rx_ring, rx_buffer, size);
8973 } else {
8974 rx_buffer->pagecnt_bias++;
8975 }
8976 total_packets++;
8977 total_bytes += size;
8978 } else if (skb)
8979 igb_add_rx_frag(rx_ring, rx_buffer, skb, size);
8980 else if (ring_uses_build_skb(rx_ring))
8981 skb = igb_build_skb(rx_ring, rx_buffer, &xdp,
8982 timestamp);
8983 else
8984 skb = igb_construct_skb(rx_ring, rx_buffer,
8985 &xdp, timestamp);
8986
8987 /* exit if we failed to retrieve a buffer */
8988 if (!skb) {
8989 rx_ring->rx_stats.alloc_failed++;
8990 rx_buffer->pagecnt_bias++;
8991 break;
8992 }
8993
8994 igb_put_rx_buffer(rx_ring, rx_buffer, rx_buf_pgcnt);
8995 cleaned_count++;
8996
8997 /* fetch next buffer in frame if non-eop */
8998 if (igb_is_non_eop(rx_ring, rx_desc))
8999 continue;
9000
9001 /* verify the packet layout is correct */
9002 if (igb_cleanup_headers(rx_ring, rx_desc, skb)) {
9003 skb = NULL;
9004 continue;
9005 }
9006
9007 /* probably a little skewed due to removing CRC */
9008 total_bytes += skb->len;
9009
9010 /* populate checksum, timestamp, VLAN, and protocol */
9011 igb_process_skb_fields(rx_ring, rx_desc, skb);
9012
9013 napi_gro_receive(&q_vector->napi, skb);
9014
9015 /* reset skb pointer */
9016 skb = NULL;
9017
9018 /* update budget accounting */
9019 total_packets++;
9020 }
9021
9022 /* place incomplete frames back on ring for completion */
9023 rx_ring->skb = skb;
9024
9025 if (xdp_xmit & IGB_XDP_REDIR)
9026 xdp_do_flush();
9027
9028 if (xdp_xmit & IGB_XDP_TX) {
9029 struct igb_ring *tx_ring = igb_xdp_tx_queue_mapping(adapter);
9030
9031 nq = txring_txq(tx_ring);
9032 __netif_tx_lock(nq, cpu);
9033 igb_xdp_ring_update_tail(tx_ring);
9034 __netif_tx_unlock(nq);
9035 }
9036
9037 u64_stats_update_begin(&rx_ring->rx_syncp);
9038 rx_ring->rx_stats.packets += total_packets;
9039 rx_ring->rx_stats.bytes += total_bytes;
9040 u64_stats_update_end(&rx_ring->rx_syncp);
9041 q_vector->rx.total_packets += total_packets;
9042 q_vector->rx.total_bytes += total_bytes;
9043
9044 if (cleaned_count)
9045 igb_alloc_rx_buffers(rx_ring, cleaned_count);
9046
9047 return total_packets;
9048 }
9049
igb_alloc_mapped_page(struct igb_ring * rx_ring,struct igb_rx_buffer * bi)9050 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
9051 struct igb_rx_buffer *bi)
9052 {
9053 struct page *page = bi->page;
9054 dma_addr_t dma;
9055
9056 /* since we are recycling buffers we should seldom need to alloc */
9057 if (likely(page))
9058 return true;
9059
9060 /* alloc new page for storage */
9061 page = dev_alloc_pages(igb_rx_pg_order(rx_ring));
9062 if (unlikely(!page)) {
9063 rx_ring->rx_stats.alloc_failed++;
9064 return false;
9065 }
9066
9067 /* map page for use */
9068 dma = dma_map_page_attrs(rx_ring->dev, page, 0,
9069 igb_rx_pg_size(rx_ring),
9070 DMA_FROM_DEVICE,
9071 IGB_RX_DMA_ATTR);
9072
9073 /* if mapping failed free memory back to system since
9074 * there isn't much point in holding memory we can't use
9075 */
9076 if (dma_mapping_error(rx_ring->dev, dma)) {
9077 __free_pages(page, igb_rx_pg_order(rx_ring));
9078
9079 rx_ring->rx_stats.alloc_failed++;
9080 return false;
9081 }
9082
9083 bi->dma = dma;
9084 bi->page = page;
9085 bi->page_offset = igb_rx_offset(rx_ring);
9086 page_ref_add(page, USHRT_MAX - 1);
9087 bi->pagecnt_bias = USHRT_MAX;
9088
9089 return true;
9090 }
9091
9092 /**
9093 * igb_alloc_rx_buffers - Replace used receive buffers
9094 * @rx_ring: rx descriptor ring to allocate new receive buffers
9095 * @cleaned_count: count of buffers to allocate
9096 **/
igb_alloc_rx_buffers(struct igb_ring * rx_ring,u16 cleaned_count)9097 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9098 {
9099 union e1000_adv_rx_desc *rx_desc;
9100 struct igb_rx_buffer *bi;
9101 u16 i = rx_ring->next_to_use;
9102 u16 bufsz;
9103
9104 /* nothing to do */
9105 if (!cleaned_count)
9106 return;
9107
9108 rx_desc = IGB_RX_DESC(rx_ring, i);
9109 bi = &rx_ring->rx_buffer_info[i];
9110 i -= rx_ring->count;
9111
9112 bufsz = igb_rx_bufsz(rx_ring);
9113
9114 do {
9115 if (!igb_alloc_mapped_page(rx_ring, bi))
9116 break;
9117
9118 /* sync the buffer for use by the device */
9119 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
9120 bi->page_offset, bufsz,
9121 DMA_FROM_DEVICE);
9122
9123 /* Refresh the desc even if buffer_addrs didn't change
9124 * because each write-back erases this info.
9125 */
9126 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
9127
9128 rx_desc++;
9129 bi++;
9130 i++;
9131 if (unlikely(!i)) {
9132 rx_desc = IGB_RX_DESC(rx_ring, 0);
9133 bi = rx_ring->rx_buffer_info;
9134 i -= rx_ring->count;
9135 }
9136
9137 /* clear the length for the next_to_use descriptor */
9138 rx_desc->wb.upper.length = 0;
9139
9140 cleaned_count--;
9141 } while (cleaned_count);
9142
9143 i += rx_ring->count;
9144
9145 if (rx_ring->next_to_use != i) {
9146 /* record the next descriptor to use */
9147 rx_ring->next_to_use = i;
9148
9149 /* update next to alloc since we have filled the ring */
9150 rx_ring->next_to_alloc = i;
9151
9152 /* Force memory writes to complete before letting h/w
9153 * know there are new descriptors to fetch. (Only
9154 * applicable for weak-ordered memory model archs,
9155 * such as IA-64).
9156 */
9157 dma_wmb();
9158 writel(i, rx_ring->tail);
9159 }
9160 }
9161
9162 /**
9163 * igb_mii_ioctl -
9164 * @netdev: pointer to netdev struct
9165 * @ifr: interface structure
9166 * @cmd: ioctl command to execute
9167 **/
igb_mii_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)9168 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
9169 {
9170 struct igb_adapter *adapter = netdev_priv(netdev);
9171 struct mii_ioctl_data *data = if_mii(ifr);
9172
9173 if (adapter->hw.phy.media_type != e1000_media_type_copper)
9174 return -EOPNOTSUPP;
9175
9176 switch (cmd) {
9177 case SIOCGMIIPHY:
9178 data->phy_id = adapter->hw.phy.addr;
9179 break;
9180 case SIOCGMIIREG:
9181 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
9182 &data->val_out))
9183 return -EIO;
9184 break;
9185 case SIOCSMIIREG:
9186 default:
9187 return -EOPNOTSUPP;
9188 }
9189 return 0;
9190 }
9191
9192 /**
9193 * igb_ioctl -
9194 * @netdev: pointer to netdev struct
9195 * @ifr: interface structure
9196 * @cmd: ioctl command to execute
9197 **/
igb_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)9198 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
9199 {
9200 switch (cmd) {
9201 case SIOCGMIIPHY:
9202 case SIOCGMIIREG:
9203 case SIOCSMIIREG:
9204 return igb_mii_ioctl(netdev, ifr, cmd);
9205 case SIOCGHWTSTAMP:
9206 return igb_ptp_get_ts_config(netdev, ifr);
9207 case SIOCSHWTSTAMP:
9208 return igb_ptp_set_ts_config(netdev, ifr);
9209 default:
9210 return -EOPNOTSUPP;
9211 }
9212 }
9213
igb_read_pci_cfg(struct e1000_hw * hw,u32 reg,u16 * value)9214 void igb_read_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
9215 {
9216 struct igb_adapter *adapter = hw->back;
9217
9218 pci_read_config_word(adapter->pdev, reg, value);
9219 }
9220
igb_write_pci_cfg(struct e1000_hw * hw,u32 reg,u16 * value)9221 void igb_write_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
9222 {
9223 struct igb_adapter *adapter = hw->back;
9224
9225 pci_write_config_word(adapter->pdev, reg, *value);
9226 }
9227
igb_read_pcie_cap_reg(struct e1000_hw * hw,u32 reg,u16 * value)9228 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
9229 {
9230 struct igb_adapter *adapter = hw->back;
9231
9232 if (pcie_capability_read_word(adapter->pdev, reg, value))
9233 return -E1000_ERR_CONFIG;
9234
9235 return 0;
9236 }
9237
igb_write_pcie_cap_reg(struct e1000_hw * hw,u32 reg,u16 * value)9238 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
9239 {
9240 struct igb_adapter *adapter = hw->back;
9241
9242 if (pcie_capability_write_word(adapter->pdev, reg, *value))
9243 return -E1000_ERR_CONFIG;
9244
9245 return 0;
9246 }
9247
igb_vlan_mode(struct net_device * netdev,netdev_features_t features)9248 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
9249 {
9250 struct igb_adapter *adapter = netdev_priv(netdev);
9251 struct e1000_hw *hw = &adapter->hw;
9252 u32 ctrl, rctl;
9253 bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
9254
9255 if (enable) {
9256 /* enable VLAN tag insert/strip */
9257 ctrl = rd32(E1000_CTRL);
9258 ctrl |= E1000_CTRL_VME;
9259 wr32(E1000_CTRL, ctrl);
9260
9261 /* Disable CFI check */
9262 rctl = rd32(E1000_RCTL);
9263 rctl &= ~E1000_RCTL_CFIEN;
9264 wr32(E1000_RCTL, rctl);
9265 } else {
9266 /* disable VLAN tag insert/strip */
9267 ctrl = rd32(E1000_CTRL);
9268 ctrl &= ~E1000_CTRL_VME;
9269 wr32(E1000_CTRL, ctrl);
9270 }
9271
9272 igb_set_vf_vlan_strip(adapter, adapter->vfs_allocated_count, enable);
9273 }
9274
igb_vlan_rx_add_vid(struct net_device * netdev,__be16 proto,u16 vid)9275 static int igb_vlan_rx_add_vid(struct net_device *netdev,
9276 __be16 proto, u16 vid)
9277 {
9278 struct igb_adapter *adapter = netdev_priv(netdev);
9279 struct e1000_hw *hw = &adapter->hw;
9280 int pf_id = adapter->vfs_allocated_count;
9281
9282 /* add the filter since PF can receive vlans w/o entry in vlvf */
9283 if (!vid || !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
9284 igb_vfta_set(hw, vid, pf_id, true, !!vid);
9285
9286 set_bit(vid, adapter->active_vlans);
9287
9288 return 0;
9289 }
9290
igb_vlan_rx_kill_vid(struct net_device * netdev,__be16 proto,u16 vid)9291 static int igb_vlan_rx_kill_vid(struct net_device *netdev,
9292 __be16 proto, u16 vid)
9293 {
9294 struct igb_adapter *adapter = netdev_priv(netdev);
9295 int pf_id = adapter->vfs_allocated_count;
9296 struct e1000_hw *hw = &adapter->hw;
9297
9298 /* remove VID from filter table */
9299 if (vid && !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
9300 igb_vfta_set(hw, vid, pf_id, false, true);
9301
9302 clear_bit(vid, adapter->active_vlans);
9303
9304 return 0;
9305 }
9306
igb_restore_vlan(struct igb_adapter * adapter)9307 static void igb_restore_vlan(struct igb_adapter *adapter)
9308 {
9309 u16 vid = 1;
9310
9311 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
9312 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0);
9313
9314 for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID)
9315 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
9316 }
9317
igb_set_spd_dplx(struct igb_adapter * adapter,u32 spd,u8 dplx)9318 int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9319 {
9320 struct pci_dev *pdev = adapter->pdev;
9321 struct e1000_mac_info *mac = &adapter->hw.mac;
9322
9323 mac->autoneg = 0;
9324
9325 /* Make sure dplx is at most 1 bit and lsb of speed is not set
9326 * for the switch() below to work
9327 */
9328 if ((spd & 1) || (dplx & ~1))
9329 goto err_inval;
9330
9331 /* Fiber NIC's only allow 1000 gbps Full duplex
9332 * and 100Mbps Full duplex for 100baseFx sfp
9333 */
9334 if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
9335 switch (spd + dplx) {
9336 case SPEED_10 + DUPLEX_HALF:
9337 case SPEED_10 + DUPLEX_FULL:
9338 case SPEED_100 + DUPLEX_HALF:
9339 goto err_inval;
9340 default:
9341 break;
9342 }
9343 }
9344
9345 switch (spd + dplx) {
9346 case SPEED_10 + DUPLEX_HALF:
9347 mac->forced_speed_duplex = ADVERTISE_10_HALF;
9348 break;
9349 case SPEED_10 + DUPLEX_FULL:
9350 mac->forced_speed_duplex = ADVERTISE_10_FULL;
9351 break;
9352 case SPEED_100 + DUPLEX_HALF:
9353 mac->forced_speed_duplex = ADVERTISE_100_HALF;
9354 break;
9355 case SPEED_100 + DUPLEX_FULL:
9356 mac->forced_speed_duplex = ADVERTISE_100_FULL;
9357 break;
9358 case SPEED_1000 + DUPLEX_FULL:
9359 mac->autoneg = 1;
9360 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
9361 break;
9362 case SPEED_1000 + DUPLEX_HALF: /* not supported */
9363 default:
9364 goto err_inval;
9365 }
9366
9367 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
9368 adapter->hw.phy.mdix = AUTO_ALL_MODES;
9369
9370 return 0;
9371
9372 err_inval:
9373 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9374 return -EINVAL;
9375 }
9376
__igb_shutdown(struct pci_dev * pdev,bool * enable_wake,bool runtime)9377 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
9378 bool runtime)
9379 {
9380 struct net_device *netdev = pci_get_drvdata(pdev);
9381 struct igb_adapter *adapter = netdev_priv(netdev);
9382 struct e1000_hw *hw = &adapter->hw;
9383 u32 ctrl, rctl, status;
9384 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
9385 bool wake;
9386
9387 rtnl_lock();
9388 netif_device_detach(netdev);
9389
9390 if (netif_running(netdev))
9391 __igb_close(netdev, true);
9392
9393 igb_ptp_suspend(adapter);
9394
9395 igb_clear_interrupt_scheme(adapter);
9396 rtnl_unlock();
9397
9398 status = rd32(E1000_STATUS);
9399 if (status & E1000_STATUS_LU)
9400 wufc &= ~E1000_WUFC_LNKC;
9401
9402 if (wufc) {
9403 igb_setup_rctl(adapter);
9404 igb_set_rx_mode(netdev);
9405
9406 /* turn on all-multi mode if wake on multicast is enabled */
9407 if (wufc & E1000_WUFC_MC) {
9408 rctl = rd32(E1000_RCTL);
9409 rctl |= E1000_RCTL_MPE;
9410 wr32(E1000_RCTL, rctl);
9411 }
9412
9413 ctrl = rd32(E1000_CTRL);
9414 ctrl |= E1000_CTRL_ADVD3WUC;
9415 wr32(E1000_CTRL, ctrl);
9416
9417 /* Allow time for pending master requests to run */
9418 igb_disable_pcie_master(hw);
9419
9420 wr32(E1000_WUC, E1000_WUC_PME_EN);
9421 wr32(E1000_WUFC, wufc);
9422 } else {
9423 wr32(E1000_WUC, 0);
9424 wr32(E1000_WUFC, 0);
9425 }
9426
9427 wake = wufc || adapter->en_mng_pt;
9428 if (!wake)
9429 igb_power_down_link(adapter);
9430 else
9431 igb_power_up_link(adapter);
9432
9433 if (enable_wake)
9434 *enable_wake = wake;
9435
9436 /* Release control of h/w to f/w. If f/w is AMT enabled, this
9437 * would have already happened in close and is redundant.
9438 */
9439 igb_release_hw_control(adapter);
9440
9441 pci_disable_device(pdev);
9442
9443 return 0;
9444 }
9445
igb_deliver_wake_packet(struct net_device * netdev)9446 static void igb_deliver_wake_packet(struct net_device *netdev)
9447 {
9448 struct igb_adapter *adapter = netdev_priv(netdev);
9449 struct e1000_hw *hw = &adapter->hw;
9450 struct sk_buff *skb;
9451 u32 wupl;
9452
9453 wupl = rd32(E1000_WUPL) & E1000_WUPL_MASK;
9454
9455 /* WUPM stores only the first 128 bytes of the wake packet.
9456 * Read the packet only if we have the whole thing.
9457 */
9458 if ((wupl == 0) || (wupl > E1000_WUPM_BYTES))
9459 return;
9460
9461 skb = netdev_alloc_skb_ip_align(netdev, E1000_WUPM_BYTES);
9462 if (!skb)
9463 return;
9464
9465 skb_put(skb, wupl);
9466
9467 /* Ensure reads are 32-bit aligned */
9468 wupl = roundup(wupl, 4);
9469
9470 memcpy_fromio(skb->data, hw->hw_addr + E1000_WUPM_REG(0), wupl);
9471
9472 skb->protocol = eth_type_trans(skb, netdev);
9473 netif_rx(skb);
9474 }
9475
igb_suspend(struct device * dev)9476 static int __maybe_unused igb_suspend(struct device *dev)
9477 {
9478 return __igb_shutdown(to_pci_dev(dev), NULL, 0);
9479 }
9480
__igb_resume(struct device * dev,bool rpm)9481 static int __maybe_unused __igb_resume(struct device *dev, bool rpm)
9482 {
9483 struct pci_dev *pdev = to_pci_dev(dev);
9484 struct net_device *netdev = pci_get_drvdata(pdev);
9485 struct igb_adapter *adapter = netdev_priv(netdev);
9486 struct e1000_hw *hw = &adapter->hw;
9487 u32 err, val;
9488
9489 pci_set_power_state(pdev, PCI_D0);
9490 pci_restore_state(pdev);
9491 pci_save_state(pdev);
9492
9493 if (!pci_device_is_present(pdev))
9494 return -ENODEV;
9495 err = pci_enable_device_mem(pdev);
9496 if (err) {
9497 dev_err(&pdev->dev,
9498 "igb: Cannot enable PCI device from suspend\n");
9499 return err;
9500 }
9501 pci_set_master(pdev);
9502
9503 pci_enable_wake(pdev, PCI_D3hot, 0);
9504 pci_enable_wake(pdev, PCI_D3cold, 0);
9505
9506 if (igb_init_interrupt_scheme(adapter, true)) {
9507 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
9508 return -ENOMEM;
9509 }
9510
9511 igb_reset(adapter);
9512
9513 /* let the f/w know that the h/w is now under the control of the
9514 * driver.
9515 */
9516 igb_get_hw_control(adapter);
9517
9518 val = rd32(E1000_WUS);
9519 if (val & WAKE_PKT_WUS)
9520 igb_deliver_wake_packet(netdev);
9521
9522 wr32(E1000_WUS, ~0);
9523
9524 if (!rpm)
9525 rtnl_lock();
9526 if (!err && netif_running(netdev))
9527 err = __igb_open(netdev, true);
9528
9529 if (!err)
9530 netif_device_attach(netdev);
9531 if (!rpm)
9532 rtnl_unlock();
9533
9534 return err;
9535 }
9536
igb_resume(struct device * dev)9537 static int __maybe_unused igb_resume(struct device *dev)
9538 {
9539 return __igb_resume(dev, false);
9540 }
9541
igb_runtime_idle(struct device * dev)9542 static int __maybe_unused igb_runtime_idle(struct device *dev)
9543 {
9544 struct net_device *netdev = dev_get_drvdata(dev);
9545 struct igb_adapter *adapter = netdev_priv(netdev);
9546
9547 if (!igb_has_link(adapter))
9548 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
9549
9550 return -EBUSY;
9551 }
9552
igb_runtime_suspend(struct device * dev)9553 static int __maybe_unused igb_runtime_suspend(struct device *dev)
9554 {
9555 return __igb_shutdown(to_pci_dev(dev), NULL, 1);
9556 }
9557
igb_runtime_resume(struct device * dev)9558 static int __maybe_unused igb_runtime_resume(struct device *dev)
9559 {
9560 return __igb_resume(dev, true);
9561 }
9562
igb_shutdown(struct pci_dev * pdev)9563 static void igb_shutdown(struct pci_dev *pdev)
9564 {
9565 bool wake;
9566
9567 __igb_shutdown(pdev, &wake, 0);
9568
9569 if (system_state == SYSTEM_POWER_OFF) {
9570 pci_wake_from_d3(pdev, wake);
9571 pci_set_power_state(pdev, PCI_D3hot);
9572 }
9573 }
9574
igb_pci_sriov_configure(struct pci_dev * dev,int num_vfs)9575 static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
9576 {
9577 #ifdef CONFIG_PCI_IOV
9578 int err;
9579
9580 if (num_vfs == 0) {
9581 return igb_disable_sriov(dev, true);
9582 } else {
9583 err = igb_enable_sriov(dev, num_vfs, true);
9584 return err ? err : num_vfs;
9585 }
9586 #endif
9587 return 0;
9588 }
9589
9590 /**
9591 * igb_io_error_detected - called when PCI error is detected
9592 * @pdev: Pointer to PCI device
9593 * @state: The current pci connection state
9594 *
9595 * This function is called after a PCI bus error affecting
9596 * this device has been detected.
9597 **/
igb_io_error_detected(struct pci_dev * pdev,pci_channel_state_t state)9598 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
9599 pci_channel_state_t state)
9600 {
9601 struct net_device *netdev = pci_get_drvdata(pdev);
9602 struct igb_adapter *adapter = netdev_priv(netdev);
9603
9604 if (state == pci_channel_io_normal) {
9605 dev_warn(&pdev->dev, "Non-correctable non-fatal error reported.\n");
9606 return PCI_ERS_RESULT_CAN_RECOVER;
9607 }
9608
9609 netif_device_detach(netdev);
9610
9611 if (state == pci_channel_io_perm_failure)
9612 return PCI_ERS_RESULT_DISCONNECT;
9613
9614 if (netif_running(netdev))
9615 igb_down(adapter);
9616 pci_disable_device(pdev);
9617
9618 /* Request a slot reset. */
9619 return PCI_ERS_RESULT_NEED_RESET;
9620 }
9621
9622 /**
9623 * igb_io_slot_reset - called after the pci bus has been reset.
9624 * @pdev: Pointer to PCI device
9625 *
9626 * Restart the card from scratch, as if from a cold-boot. Implementation
9627 * resembles the first-half of the __igb_resume routine.
9628 **/
igb_io_slot_reset(struct pci_dev * pdev)9629 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
9630 {
9631 struct net_device *netdev = pci_get_drvdata(pdev);
9632 struct igb_adapter *adapter = netdev_priv(netdev);
9633 struct e1000_hw *hw = &adapter->hw;
9634 pci_ers_result_t result;
9635
9636 if (pci_enable_device_mem(pdev)) {
9637 dev_err(&pdev->dev,
9638 "Cannot re-enable PCI device after reset.\n");
9639 result = PCI_ERS_RESULT_DISCONNECT;
9640 } else {
9641 pci_set_master(pdev);
9642 pci_restore_state(pdev);
9643 pci_save_state(pdev);
9644
9645 pci_enable_wake(pdev, PCI_D3hot, 0);
9646 pci_enable_wake(pdev, PCI_D3cold, 0);
9647
9648 /* In case of PCI error, adapter lose its HW address
9649 * so we should re-assign it here.
9650 */
9651 hw->hw_addr = adapter->io_addr;
9652
9653 igb_reset(adapter);
9654 wr32(E1000_WUS, ~0);
9655 result = PCI_ERS_RESULT_RECOVERED;
9656 }
9657
9658 return result;
9659 }
9660
9661 /**
9662 * igb_io_resume - called when traffic can start flowing again.
9663 * @pdev: Pointer to PCI device
9664 *
9665 * This callback is called when the error recovery driver tells us that
9666 * its OK to resume normal operation. Implementation resembles the
9667 * second-half of the __igb_resume routine.
9668 */
igb_io_resume(struct pci_dev * pdev)9669 static void igb_io_resume(struct pci_dev *pdev)
9670 {
9671 struct net_device *netdev = pci_get_drvdata(pdev);
9672 struct igb_adapter *adapter = netdev_priv(netdev);
9673
9674 if (netif_running(netdev)) {
9675 if (!test_bit(__IGB_DOWN, &adapter->state)) {
9676 dev_dbg(&pdev->dev, "Resuming from non-fatal error, do nothing.\n");
9677 return;
9678 }
9679 if (igb_up(adapter)) {
9680 dev_err(&pdev->dev, "igb_up failed after reset\n");
9681 return;
9682 }
9683 }
9684
9685 netif_device_attach(netdev);
9686
9687 /* let the f/w know that the h/w is now under the control of the
9688 * driver.
9689 */
9690 igb_get_hw_control(adapter);
9691 }
9692
9693 /**
9694 * igb_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table
9695 * @adapter: Pointer to adapter structure
9696 * @index: Index of the RAR entry which need to be synced with MAC table
9697 **/
igb_rar_set_index(struct igb_adapter * adapter,u32 index)9698 static void igb_rar_set_index(struct igb_adapter *adapter, u32 index)
9699 {
9700 struct e1000_hw *hw = &adapter->hw;
9701 u32 rar_low, rar_high;
9702 u8 *addr = adapter->mac_table[index].addr;
9703
9704 /* HW expects these to be in network order when they are plugged
9705 * into the registers which are little endian. In order to guarantee
9706 * that ordering we need to do an leXX_to_cpup here in order to be
9707 * ready for the byteswap that occurs with writel
9708 */
9709 rar_low = le32_to_cpup((__le32 *)(addr));
9710 rar_high = le16_to_cpup((__le16 *)(addr + 4));
9711
9712 /* Indicate to hardware the Address is Valid. */
9713 if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE) {
9714 if (is_valid_ether_addr(addr))
9715 rar_high |= E1000_RAH_AV;
9716
9717 if (adapter->mac_table[index].state & IGB_MAC_STATE_SRC_ADDR)
9718 rar_high |= E1000_RAH_ASEL_SRC_ADDR;
9719
9720 switch (hw->mac.type) {
9721 case e1000_82575:
9722 case e1000_i210:
9723 if (adapter->mac_table[index].state &
9724 IGB_MAC_STATE_QUEUE_STEERING)
9725 rar_high |= E1000_RAH_QSEL_ENABLE;
9726
9727 rar_high |= E1000_RAH_POOL_1 *
9728 adapter->mac_table[index].queue;
9729 break;
9730 default:
9731 rar_high |= E1000_RAH_POOL_1 <<
9732 adapter->mac_table[index].queue;
9733 break;
9734 }
9735 }
9736
9737 wr32(E1000_RAL(index), rar_low);
9738 wrfl();
9739 wr32(E1000_RAH(index), rar_high);
9740 wrfl();
9741 }
9742
igb_set_vf_mac(struct igb_adapter * adapter,int vf,unsigned char * mac_addr)9743 static int igb_set_vf_mac(struct igb_adapter *adapter,
9744 int vf, unsigned char *mac_addr)
9745 {
9746 struct e1000_hw *hw = &adapter->hw;
9747 /* VF MAC addresses start at end of receive addresses and moves
9748 * towards the first, as a result a collision should not be possible
9749 */
9750 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
9751 unsigned char *vf_mac_addr = adapter->vf_data[vf].vf_mac_addresses;
9752
9753 ether_addr_copy(vf_mac_addr, mac_addr);
9754 ether_addr_copy(adapter->mac_table[rar_entry].addr, mac_addr);
9755 adapter->mac_table[rar_entry].queue = vf;
9756 adapter->mac_table[rar_entry].state |= IGB_MAC_STATE_IN_USE;
9757 igb_rar_set_index(adapter, rar_entry);
9758
9759 return 0;
9760 }
9761
igb_ndo_set_vf_mac(struct net_device * netdev,int vf,u8 * mac)9762 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
9763 {
9764 struct igb_adapter *adapter = netdev_priv(netdev);
9765
9766 if (vf >= adapter->vfs_allocated_count)
9767 return -EINVAL;
9768
9769 /* Setting the VF MAC to 0 reverts the IGB_VF_FLAG_PF_SET_MAC
9770 * flag and allows to overwrite the MAC via VF netdev. This
9771 * is necessary to allow libvirt a way to restore the original
9772 * MAC after unbinding vfio-pci and reloading igbvf after shutting
9773 * down a VM.
9774 */
9775 if (is_zero_ether_addr(mac)) {
9776 adapter->vf_data[vf].flags &= ~IGB_VF_FLAG_PF_SET_MAC;
9777 dev_info(&adapter->pdev->dev,
9778 "remove administratively set MAC on VF %d\n",
9779 vf);
9780 } else if (is_valid_ether_addr(mac)) {
9781 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
9782 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n",
9783 mac, vf);
9784 dev_info(&adapter->pdev->dev,
9785 "Reload the VF driver to make this change effective.");
9786 /* Generate additional warning if PF is down */
9787 if (test_bit(__IGB_DOWN, &adapter->state)) {
9788 dev_warn(&adapter->pdev->dev,
9789 "The VF MAC address has been set, but the PF device is not up.\n");
9790 dev_warn(&adapter->pdev->dev,
9791 "Bring the PF device up before attempting to use the VF device.\n");
9792 }
9793 } else {
9794 return -EINVAL;
9795 }
9796 return igb_set_vf_mac(adapter, vf, mac);
9797 }
9798
igb_link_mbps(int internal_link_speed)9799 static int igb_link_mbps(int internal_link_speed)
9800 {
9801 switch (internal_link_speed) {
9802 case SPEED_100:
9803 return 100;
9804 case SPEED_1000:
9805 return 1000;
9806 default:
9807 return 0;
9808 }
9809 }
9810
igb_set_vf_rate_limit(struct e1000_hw * hw,int vf,int tx_rate,int link_speed)9811 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
9812 int link_speed)
9813 {
9814 int rf_dec, rf_int;
9815 u32 bcnrc_val;
9816
9817 if (tx_rate != 0) {
9818 /* Calculate the rate factor values to set */
9819 rf_int = link_speed / tx_rate;
9820 rf_dec = (link_speed - (rf_int * tx_rate));
9821 rf_dec = (rf_dec * BIT(E1000_RTTBCNRC_RF_INT_SHIFT)) /
9822 tx_rate;
9823
9824 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
9825 bcnrc_val |= ((rf_int << E1000_RTTBCNRC_RF_INT_SHIFT) &
9826 E1000_RTTBCNRC_RF_INT_MASK);
9827 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
9828 } else {
9829 bcnrc_val = 0;
9830 }
9831
9832 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
9833 /* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
9834 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
9835 */
9836 wr32(E1000_RTTBCNRM, 0x14);
9837 wr32(E1000_RTTBCNRC, bcnrc_val);
9838 }
9839
igb_check_vf_rate_limit(struct igb_adapter * adapter)9840 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
9841 {
9842 int actual_link_speed, i;
9843 bool reset_rate = false;
9844
9845 /* VF TX rate limit was not set or not supported */
9846 if ((adapter->vf_rate_link_speed == 0) ||
9847 (adapter->hw.mac.type != e1000_82576))
9848 return;
9849
9850 actual_link_speed = igb_link_mbps(adapter->link_speed);
9851 if (actual_link_speed != adapter->vf_rate_link_speed) {
9852 reset_rate = true;
9853 adapter->vf_rate_link_speed = 0;
9854 dev_info(&adapter->pdev->dev,
9855 "Link speed has been changed. VF Transmit rate is disabled\n");
9856 }
9857
9858 for (i = 0; i < adapter->vfs_allocated_count; i++) {
9859 if (reset_rate)
9860 adapter->vf_data[i].tx_rate = 0;
9861
9862 igb_set_vf_rate_limit(&adapter->hw, i,
9863 adapter->vf_data[i].tx_rate,
9864 actual_link_speed);
9865 }
9866 }
9867
igb_ndo_set_vf_bw(struct net_device * netdev,int vf,int min_tx_rate,int max_tx_rate)9868 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf,
9869 int min_tx_rate, int max_tx_rate)
9870 {
9871 struct igb_adapter *adapter = netdev_priv(netdev);
9872 struct e1000_hw *hw = &adapter->hw;
9873 int actual_link_speed;
9874
9875 if (hw->mac.type != e1000_82576)
9876 return -EOPNOTSUPP;
9877
9878 if (min_tx_rate)
9879 return -EINVAL;
9880
9881 actual_link_speed = igb_link_mbps(adapter->link_speed);
9882 if ((vf >= adapter->vfs_allocated_count) ||
9883 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
9884 (max_tx_rate < 0) ||
9885 (max_tx_rate > actual_link_speed))
9886 return -EINVAL;
9887
9888 adapter->vf_rate_link_speed = actual_link_speed;
9889 adapter->vf_data[vf].tx_rate = (u16)max_tx_rate;
9890 igb_set_vf_rate_limit(hw, vf, max_tx_rate, actual_link_speed);
9891
9892 return 0;
9893 }
9894
igb_ndo_set_vf_spoofchk(struct net_device * netdev,int vf,bool setting)9895 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
9896 bool setting)
9897 {
9898 struct igb_adapter *adapter = netdev_priv(netdev);
9899 struct e1000_hw *hw = &adapter->hw;
9900 u32 reg_val, reg_offset;
9901
9902 if (!adapter->vfs_allocated_count)
9903 return -EOPNOTSUPP;
9904
9905 if (vf >= adapter->vfs_allocated_count)
9906 return -EINVAL;
9907
9908 reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC;
9909 reg_val = rd32(reg_offset);
9910 if (setting)
9911 reg_val |= (BIT(vf) |
9912 BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
9913 else
9914 reg_val &= ~(BIT(vf) |
9915 BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
9916 wr32(reg_offset, reg_val);
9917
9918 adapter->vf_data[vf].spoofchk_enabled = setting;
9919 return 0;
9920 }
9921
igb_ndo_set_vf_trust(struct net_device * netdev,int vf,bool setting)9922 static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, bool setting)
9923 {
9924 struct igb_adapter *adapter = netdev_priv(netdev);
9925
9926 if (vf >= adapter->vfs_allocated_count)
9927 return -EINVAL;
9928 if (adapter->vf_data[vf].trusted == setting)
9929 return 0;
9930
9931 adapter->vf_data[vf].trusted = setting;
9932
9933 dev_info(&adapter->pdev->dev, "VF %u is %strusted\n",
9934 vf, setting ? "" : "not ");
9935 return 0;
9936 }
9937
igb_ndo_get_vf_config(struct net_device * netdev,int vf,struct ifla_vf_info * ivi)9938 static int igb_ndo_get_vf_config(struct net_device *netdev,
9939 int vf, struct ifla_vf_info *ivi)
9940 {
9941 struct igb_adapter *adapter = netdev_priv(netdev);
9942 if (vf >= adapter->vfs_allocated_count)
9943 return -EINVAL;
9944 ivi->vf = vf;
9945 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
9946 ivi->max_tx_rate = adapter->vf_data[vf].tx_rate;
9947 ivi->min_tx_rate = 0;
9948 ivi->vlan = adapter->vf_data[vf].pf_vlan;
9949 ivi->qos = adapter->vf_data[vf].pf_qos;
9950 ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled;
9951 ivi->trusted = adapter->vf_data[vf].trusted;
9952 return 0;
9953 }
9954
igb_vmm_control(struct igb_adapter * adapter)9955 static void igb_vmm_control(struct igb_adapter *adapter)
9956 {
9957 struct e1000_hw *hw = &adapter->hw;
9958 u32 reg;
9959
9960 switch (hw->mac.type) {
9961 case e1000_82575:
9962 case e1000_i210:
9963 case e1000_i211:
9964 case e1000_i354:
9965 default:
9966 /* replication is not supported for 82575 */
9967 return;
9968 case e1000_82576:
9969 /* notify HW that the MAC is adding vlan tags */
9970 reg = rd32(E1000_DTXCTL);
9971 reg |= E1000_DTXCTL_VLAN_ADDED;
9972 wr32(E1000_DTXCTL, reg);
9973 fallthrough;
9974 case e1000_82580:
9975 /* enable replication vlan tag stripping */
9976 reg = rd32(E1000_RPLOLR);
9977 reg |= E1000_RPLOLR_STRVLAN;
9978 wr32(E1000_RPLOLR, reg);
9979 fallthrough;
9980 case e1000_i350:
9981 /* none of the above registers are supported by i350 */
9982 break;
9983 }
9984
9985 if (adapter->vfs_allocated_count) {
9986 igb_vmdq_set_loopback_pf(hw, true);
9987 igb_vmdq_set_replication_pf(hw, true);
9988 igb_vmdq_set_anti_spoofing_pf(hw, true,
9989 adapter->vfs_allocated_count);
9990 } else {
9991 igb_vmdq_set_loopback_pf(hw, false);
9992 igb_vmdq_set_replication_pf(hw, false);
9993 }
9994 }
9995
igb_init_dmac(struct igb_adapter * adapter,u32 pba)9996 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
9997 {
9998 struct e1000_hw *hw = &adapter->hw;
9999 u32 dmac_thr;
10000 u16 hwm;
10001 u32 reg;
10002
10003 if (hw->mac.type > e1000_82580) {
10004 if (adapter->flags & IGB_FLAG_DMAC) {
10005 /* force threshold to 0. */
10006 wr32(E1000_DMCTXTH, 0);
10007
10008 /* DMA Coalescing high water mark needs to be greater
10009 * than the Rx threshold. Set hwm to PBA - max frame
10010 * size in 16B units, capping it at PBA - 6KB.
10011 */
10012 hwm = 64 * (pba - 6);
10013 reg = rd32(E1000_FCRTC);
10014 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
10015 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
10016 & E1000_FCRTC_RTH_COAL_MASK);
10017 wr32(E1000_FCRTC, reg);
10018
10019 /* Set the DMA Coalescing Rx threshold to PBA - 2 * max
10020 * frame size, capping it at PBA - 10KB.
10021 */
10022 dmac_thr = pba - 10;
10023 reg = rd32(E1000_DMACR);
10024 reg &= ~E1000_DMACR_DMACTHR_MASK;
10025 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
10026 & E1000_DMACR_DMACTHR_MASK);
10027
10028 /* transition to L0x or L1 if available..*/
10029 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
10030
10031 /* watchdog timer= +-1000 usec in 32usec intervals */
10032 reg |= (1000 >> 5);
10033
10034 /* Disable BMC-to-OS Watchdog Enable */
10035 if (hw->mac.type != e1000_i354)
10036 reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
10037 wr32(E1000_DMACR, reg);
10038
10039 /* no lower threshold to disable
10040 * coalescing(smart fifb)-UTRESH=0
10041 */
10042 wr32(E1000_DMCRTRH, 0);
10043
10044 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
10045
10046 wr32(E1000_DMCTLX, reg);
10047
10048 /* free space in tx packet buffer to wake from
10049 * DMA coal
10050 */
10051 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
10052 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
10053 }
10054
10055 if (hw->mac.type >= e1000_i210 ||
10056 (adapter->flags & IGB_FLAG_DMAC)) {
10057 reg = rd32(E1000_PCIEMISC);
10058 reg |= E1000_PCIEMISC_LX_DECISION;
10059 wr32(E1000_PCIEMISC, reg);
10060 } /* endif adapter->dmac is not disabled */
10061 } else if (hw->mac.type == e1000_82580) {
10062 u32 reg = rd32(E1000_PCIEMISC);
10063
10064 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
10065 wr32(E1000_DMACR, 0);
10066 }
10067 }
10068
10069 /**
10070 * igb_read_i2c_byte - Reads 8 bit word over I2C
10071 * @hw: pointer to hardware structure
10072 * @byte_offset: byte offset to read
10073 * @dev_addr: device address
10074 * @data: value read
10075 *
10076 * Performs byte read operation over I2C interface at
10077 * a specified device address.
10078 **/
igb_read_i2c_byte(struct e1000_hw * hw,u8 byte_offset,u8 dev_addr,u8 * data)10079 s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
10080 u8 dev_addr, u8 *data)
10081 {
10082 struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
10083 struct i2c_client *this_client = adapter->i2c_client;
10084 s32 status;
10085 u16 swfw_mask = 0;
10086
10087 if (!this_client)
10088 return E1000_ERR_I2C;
10089
10090 swfw_mask = E1000_SWFW_PHY0_SM;
10091
10092 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
10093 return E1000_ERR_SWFW_SYNC;
10094
10095 status = i2c_smbus_read_byte_data(this_client, byte_offset);
10096 hw->mac.ops.release_swfw_sync(hw, swfw_mask);
10097
10098 if (status < 0)
10099 return E1000_ERR_I2C;
10100 else {
10101 *data = status;
10102 return 0;
10103 }
10104 }
10105
10106 /**
10107 * igb_write_i2c_byte - Writes 8 bit word over I2C
10108 * @hw: pointer to hardware structure
10109 * @byte_offset: byte offset to write
10110 * @dev_addr: device address
10111 * @data: value to write
10112 *
10113 * Performs byte write operation over I2C interface at
10114 * a specified device address.
10115 **/
igb_write_i2c_byte(struct e1000_hw * hw,u8 byte_offset,u8 dev_addr,u8 data)10116 s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
10117 u8 dev_addr, u8 data)
10118 {
10119 struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
10120 struct i2c_client *this_client = adapter->i2c_client;
10121 s32 status;
10122 u16 swfw_mask = E1000_SWFW_PHY0_SM;
10123
10124 if (!this_client)
10125 return E1000_ERR_I2C;
10126
10127 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
10128 return E1000_ERR_SWFW_SYNC;
10129 status = i2c_smbus_write_byte_data(this_client, byte_offset, data);
10130 hw->mac.ops.release_swfw_sync(hw, swfw_mask);
10131
10132 if (status)
10133 return E1000_ERR_I2C;
10134 else
10135 return 0;
10136
10137 }
10138
igb_reinit_queues(struct igb_adapter * adapter)10139 int igb_reinit_queues(struct igb_adapter *adapter)
10140 {
10141 struct net_device *netdev = adapter->netdev;
10142 struct pci_dev *pdev = adapter->pdev;
10143 int err = 0;
10144
10145 if (netif_running(netdev))
10146 igb_close(netdev);
10147
10148 igb_reset_interrupt_capability(adapter);
10149
10150 if (igb_init_interrupt_scheme(adapter, true)) {
10151 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
10152 return -ENOMEM;
10153 }
10154
10155 if (netif_running(netdev))
10156 err = igb_open(netdev);
10157
10158 return err;
10159 }
10160
igb_nfc_filter_exit(struct igb_adapter * adapter)10161 static void igb_nfc_filter_exit(struct igb_adapter *adapter)
10162 {
10163 struct igb_nfc_filter *rule;
10164
10165 spin_lock(&adapter->nfc_lock);
10166
10167 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
10168 igb_erase_filter(adapter, rule);
10169
10170 hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node)
10171 igb_erase_filter(adapter, rule);
10172
10173 spin_unlock(&adapter->nfc_lock);
10174 }
10175
igb_nfc_filter_restore(struct igb_adapter * adapter)10176 static void igb_nfc_filter_restore(struct igb_adapter *adapter)
10177 {
10178 struct igb_nfc_filter *rule;
10179
10180 spin_lock(&adapter->nfc_lock);
10181
10182 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
10183 igb_add_filter(adapter, rule);
10184
10185 spin_unlock(&adapter->nfc_lock);
10186 }
10187 /* igb_main.c */
10188