1 /******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2018 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, see <http://www.gnu.org/licenses/>. 17 18 The full GNU General Public License is included in this distribution in 19 the file called "COPYING". 20 21 Contact Information: 22 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 25 *******************************************************************************/ 26 27 /****************************************************************************** 28 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 29 ******************************************************************************/ 30 31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 32 33 #include <linux/types.h> 34 #include <linux/bitops.h> 35 #include <linux/module.h> 36 #include <linux/pci.h> 37 #include <linux/netdevice.h> 38 #include <linux/vmalloc.h> 39 #include <linux/string.h> 40 #include <linux/in.h> 41 #include <linux/ip.h> 42 #include <linux/tcp.h> 43 #include <linux/sctp.h> 44 #include <linux/ipv6.h> 45 #include <linux/slab.h> 46 #include <net/checksum.h> 47 #include <net/ip6_checksum.h> 48 #include <linux/ethtool.h> 49 #include <linux/if.h> 50 #include <linux/if_vlan.h> 51 #include <linux/prefetch.h> 52 #include <net/mpls.h> 53 #include <linux/bpf.h> 54 #include <linux/bpf_trace.h> 55 #include <linux/atomic.h> 56 57 #include "ixgbevf.h" 58 59 const char ixgbevf_driver_name[] = "ixgbevf"; 60 static const char ixgbevf_driver_string[] = 61 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 62 63 #define DRV_VERSION "4.1.0-k" 64 const char ixgbevf_driver_version[] = DRV_VERSION; 65 static char ixgbevf_copyright[] = 66 "Copyright (c) 2009 - 2015 Intel Corporation."; 67 68 static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 69 [board_82599_vf] = &ixgbevf_82599_vf_info, 70 [board_82599_vf_hv] = &ixgbevf_82599_vf_hv_info, 71 [board_X540_vf] = &ixgbevf_X540_vf_info, 72 [board_X540_vf_hv] = &ixgbevf_X540_vf_hv_info, 73 [board_X550_vf] = &ixgbevf_X550_vf_info, 74 [board_X550_vf_hv] = &ixgbevf_X550_vf_hv_info, 75 [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info, 76 [board_X550EM_x_vf_hv] = &ixgbevf_X550EM_x_vf_hv_info, 77 [board_x550em_a_vf] = &ixgbevf_x550em_a_vf_info, 78 }; 79 80 /* ixgbevf_pci_tbl - PCI Device ID Table 81 * 82 * Wildcard entries (PCI_ANY_ID) should come last 83 * Last entry must be all 0s 84 * 85 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 86 * Class, Class Mask, private data (not used) } 87 */ 88 static const struct pci_device_id ixgbevf_pci_tbl[] = { 89 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf }, 90 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF_HV), board_82599_vf_hv }, 91 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf }, 92 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF_HV), board_X540_vf_hv }, 93 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf }, 94 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF_HV), board_X550_vf_hv }, 95 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf }, 96 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF_HV), board_X550EM_x_vf_hv}, 97 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_VF), board_x550em_a_vf }, 98 /* required last entry */ 99 {0, } 100 }; 101 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 102 103 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 104 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); 105 MODULE_LICENSE("GPL"); 106 MODULE_VERSION(DRV_VERSION); 107 108 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 109 static int debug = -1; 110 module_param(debug, int, 0); 111 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 112 113 static struct workqueue_struct *ixgbevf_wq; 114 115 static void ixgbevf_service_event_schedule(struct ixgbevf_adapter *adapter) 116 { 117 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 118 !test_bit(__IXGBEVF_REMOVING, &adapter->state) && 119 !test_and_set_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)) 120 queue_work(ixgbevf_wq, &adapter->service_task); 121 } 122 123 static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter) 124 { 125 BUG_ON(!test_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)); 126 127 /* flush memory to make sure state is correct before next watchdog */ 128 smp_mb__before_atomic(); 129 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 130 } 131 132 /* forward decls */ 133 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); 134 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); 135 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); 136 static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer); 137 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, 138 struct ixgbevf_rx_buffer *old_buff); 139 140 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) 141 { 142 struct ixgbevf_adapter *adapter = hw->back; 143 144 if (!hw->hw_addr) 145 return; 146 hw->hw_addr = NULL; 147 dev_err(&adapter->pdev->dev, "Adapter removed\n"); 148 if (test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 149 ixgbevf_service_event_schedule(adapter); 150 } 151 152 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg) 153 { 154 u32 value; 155 156 /* The following check not only optimizes a bit by not 157 * performing a read on the status register when the 158 * register just read was a status register read that 159 * returned IXGBE_FAILED_READ_REG. It also blocks any 160 * potential recursion. 161 */ 162 if (reg == IXGBE_VFSTATUS) { 163 ixgbevf_remove_adapter(hw); 164 return; 165 } 166 value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS); 167 if (value == IXGBE_FAILED_READ_REG) 168 ixgbevf_remove_adapter(hw); 169 } 170 171 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg) 172 { 173 u8 __iomem *reg_addr = READ_ONCE(hw->hw_addr); 174 u32 value; 175 176 if (IXGBE_REMOVED(reg_addr)) 177 return IXGBE_FAILED_READ_REG; 178 value = readl(reg_addr + reg); 179 if (unlikely(value == IXGBE_FAILED_READ_REG)) 180 ixgbevf_check_remove(hw, reg); 181 return value; 182 } 183 184 /** 185 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 186 * @adapter: pointer to adapter struct 187 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 188 * @queue: queue to map the corresponding interrupt to 189 * @msix_vector: the vector to map to the corresponding queue 190 **/ 191 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 192 u8 queue, u8 msix_vector) 193 { 194 u32 ivar, index; 195 struct ixgbe_hw *hw = &adapter->hw; 196 197 if (direction == -1) { 198 /* other causes */ 199 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 200 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 201 ivar &= ~0xFF; 202 ivar |= msix_vector; 203 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 204 } else { 205 /* Tx or Rx causes */ 206 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 207 index = ((16 * (queue & 1)) + (8 * direction)); 208 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 209 ivar &= ~(0xFF << index); 210 ivar |= (msix_vector << index); 211 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 212 } 213 } 214 215 static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring) 216 { 217 return ring->stats.packets; 218 } 219 220 static u32 ixgbevf_get_tx_pending(struct ixgbevf_ring *ring) 221 { 222 struct ixgbevf_adapter *adapter = netdev_priv(ring->netdev); 223 struct ixgbe_hw *hw = &adapter->hw; 224 225 u32 head = IXGBE_READ_REG(hw, IXGBE_VFTDH(ring->reg_idx)); 226 u32 tail = IXGBE_READ_REG(hw, IXGBE_VFTDT(ring->reg_idx)); 227 228 if (head != tail) 229 return (head < tail) ? 230 tail - head : (tail + ring->count - head); 231 232 return 0; 233 } 234 235 static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring) 236 { 237 u32 tx_done = ixgbevf_get_tx_completed(tx_ring); 238 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 239 u32 tx_pending = ixgbevf_get_tx_pending(tx_ring); 240 241 clear_check_for_tx_hang(tx_ring); 242 243 /* Check for a hung queue, but be thorough. This verifies 244 * that a transmit has been completed since the previous 245 * check AND there is at least one packet pending. The 246 * ARMED bit is set to indicate a potential hang. 247 */ 248 if ((tx_done_old == tx_done) && tx_pending) { 249 /* make sure it is true for two checks in a row */ 250 return test_and_set_bit(__IXGBEVF_HANG_CHECK_ARMED, 251 &tx_ring->state); 252 } 253 /* reset the countdown */ 254 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &tx_ring->state); 255 256 /* update completed stats and continue */ 257 tx_ring->tx_stats.tx_done_old = tx_done; 258 259 return false; 260 } 261 262 static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter) 263 { 264 /* Do the reset outside of interrupt context */ 265 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 266 set_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state); 267 ixgbevf_service_event_schedule(adapter); 268 } 269 } 270 271 /** 272 * ixgbevf_tx_timeout - Respond to a Tx Hang 273 * @netdev: network interface device structure 274 **/ 275 static void ixgbevf_tx_timeout(struct net_device *netdev) 276 { 277 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 278 279 ixgbevf_tx_timeout_reset(adapter); 280 } 281 282 /** 283 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 284 * @q_vector: board private structure 285 * @tx_ring: tx ring to clean 286 * @napi_budget: Used to determine if we are in netpoll 287 **/ 288 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, 289 struct ixgbevf_ring *tx_ring, int napi_budget) 290 { 291 struct ixgbevf_adapter *adapter = q_vector->adapter; 292 struct ixgbevf_tx_buffer *tx_buffer; 293 union ixgbe_adv_tx_desc *tx_desc; 294 unsigned int total_bytes = 0, total_packets = 0; 295 unsigned int budget = tx_ring->count / 2; 296 unsigned int i = tx_ring->next_to_clean; 297 298 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 299 return true; 300 301 tx_buffer = &tx_ring->tx_buffer_info[i]; 302 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 303 i -= tx_ring->count; 304 305 do { 306 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 307 308 /* if next_to_watch is not set then there is no work pending */ 309 if (!eop_desc) 310 break; 311 312 /* prevent any other reads prior to eop_desc */ 313 smp_rmb(); 314 315 /* if DD is not set pending work has not been completed */ 316 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 317 break; 318 319 /* clear next_to_watch to prevent false hangs */ 320 tx_buffer->next_to_watch = NULL; 321 322 /* update the statistics for this packet */ 323 total_bytes += tx_buffer->bytecount; 324 total_packets += tx_buffer->gso_segs; 325 326 /* free the skb */ 327 if (ring_is_xdp(tx_ring)) 328 page_frag_free(tx_buffer->data); 329 else 330 napi_consume_skb(tx_buffer->skb, napi_budget); 331 332 /* unmap skb header data */ 333 dma_unmap_single(tx_ring->dev, 334 dma_unmap_addr(tx_buffer, dma), 335 dma_unmap_len(tx_buffer, len), 336 DMA_TO_DEVICE); 337 338 /* clear tx_buffer data */ 339 dma_unmap_len_set(tx_buffer, len, 0); 340 341 /* unmap remaining buffers */ 342 while (tx_desc != eop_desc) { 343 tx_buffer++; 344 tx_desc++; 345 i++; 346 if (unlikely(!i)) { 347 i -= tx_ring->count; 348 tx_buffer = tx_ring->tx_buffer_info; 349 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 350 } 351 352 /* unmap any remaining paged data */ 353 if (dma_unmap_len(tx_buffer, len)) { 354 dma_unmap_page(tx_ring->dev, 355 dma_unmap_addr(tx_buffer, dma), 356 dma_unmap_len(tx_buffer, len), 357 DMA_TO_DEVICE); 358 dma_unmap_len_set(tx_buffer, len, 0); 359 } 360 } 361 362 /* move us one more past the eop_desc for start of next pkt */ 363 tx_buffer++; 364 tx_desc++; 365 i++; 366 if (unlikely(!i)) { 367 i -= tx_ring->count; 368 tx_buffer = tx_ring->tx_buffer_info; 369 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 370 } 371 372 /* issue prefetch for next Tx descriptor */ 373 prefetch(tx_desc); 374 375 /* update budget accounting */ 376 budget--; 377 } while (likely(budget)); 378 379 i += tx_ring->count; 380 tx_ring->next_to_clean = i; 381 u64_stats_update_begin(&tx_ring->syncp); 382 tx_ring->stats.bytes += total_bytes; 383 tx_ring->stats.packets += total_packets; 384 u64_stats_update_end(&tx_ring->syncp); 385 q_vector->tx.total_bytes += total_bytes; 386 q_vector->tx.total_packets += total_packets; 387 388 if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) { 389 struct ixgbe_hw *hw = &adapter->hw; 390 union ixgbe_adv_tx_desc *eop_desc; 391 392 eop_desc = tx_ring->tx_buffer_info[i].next_to_watch; 393 394 pr_err("Detected Tx Unit Hang%s\n" 395 " Tx Queue <%d>\n" 396 " TDH, TDT <%x>, <%x>\n" 397 " next_to_use <%x>\n" 398 " next_to_clean <%x>\n" 399 "tx_buffer_info[next_to_clean]\n" 400 " next_to_watch <%p>\n" 401 " eop_desc->wb.status <%x>\n" 402 " time_stamp <%lx>\n" 403 " jiffies <%lx>\n", 404 ring_is_xdp(tx_ring) ? " XDP" : "", 405 tx_ring->queue_index, 406 IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)), 407 IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)), 408 tx_ring->next_to_use, i, 409 eop_desc, (eop_desc ? eop_desc->wb.status : 0), 410 tx_ring->tx_buffer_info[i].time_stamp, jiffies); 411 412 if (!ring_is_xdp(tx_ring)) 413 netif_stop_subqueue(tx_ring->netdev, 414 tx_ring->queue_index); 415 416 /* schedule immediate reset if we believe we hung */ 417 ixgbevf_tx_timeout_reset(adapter); 418 419 return true; 420 } 421 422 if (ring_is_xdp(tx_ring)) 423 return !!budget; 424 425 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 426 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 427 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 428 /* Make sure that anybody stopping the queue after this 429 * sees the new next_to_clean. 430 */ 431 smp_mb(); 432 433 if (__netif_subqueue_stopped(tx_ring->netdev, 434 tx_ring->queue_index) && 435 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 436 netif_wake_subqueue(tx_ring->netdev, 437 tx_ring->queue_index); 438 ++tx_ring->tx_stats.restart_queue; 439 } 440 } 441 442 return !!budget; 443 } 444 445 /** 446 * ixgbevf_rx_skb - Helper function to determine proper Rx method 447 * @q_vector: structure containing interrupt and ring information 448 * @skb: packet to send up 449 **/ 450 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, 451 struct sk_buff *skb) 452 { 453 napi_gro_receive(&q_vector->napi, skb); 454 } 455 456 #define IXGBE_RSS_L4_TYPES_MASK \ 457 ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ 458 (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ 459 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ 460 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) 461 462 static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring, 463 union ixgbe_adv_rx_desc *rx_desc, 464 struct sk_buff *skb) 465 { 466 u16 rss_type; 467 468 if (!(ring->netdev->features & NETIF_F_RXHASH)) 469 return; 470 471 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & 472 IXGBE_RXDADV_RSSTYPE_MASK; 473 474 if (!rss_type) 475 return; 476 477 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 478 (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? 479 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); 480 } 481 482 /** 483 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 484 * @ring: structure containig ring specific data 485 * @rx_desc: current Rx descriptor being processed 486 * @skb: skb currently being received and modified 487 **/ 488 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, 489 union ixgbe_adv_rx_desc *rx_desc, 490 struct sk_buff *skb) 491 { 492 skb_checksum_none_assert(skb); 493 494 /* Rx csum disabled */ 495 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 496 return; 497 498 /* if IP and error */ 499 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 500 ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 501 ring->rx_stats.csum_err++; 502 return; 503 } 504 505 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 506 return; 507 508 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 509 ring->rx_stats.csum_err++; 510 return; 511 } 512 513 /* It must be a TCP or UDP packet with a valid checksum */ 514 skb->ip_summed = CHECKSUM_UNNECESSARY; 515 } 516 517 /** 518 * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor 519 * @rx_ring: rx descriptor ring packet is being transacted on 520 * @rx_desc: pointer to the EOP Rx descriptor 521 * @skb: pointer to current skb being populated 522 * 523 * This function checks the ring, descriptor, and packet information in 524 * order to populate the checksum, VLAN, protocol, and other fields within 525 * the skb. 526 **/ 527 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, 528 union ixgbe_adv_rx_desc *rx_desc, 529 struct sk_buff *skb) 530 { 531 ixgbevf_rx_hash(rx_ring, rx_desc, skb); 532 ixgbevf_rx_checksum(rx_ring, rx_desc, skb); 533 534 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 535 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 536 unsigned long *active_vlans = netdev_priv(rx_ring->netdev); 537 538 if (test_bit(vid & VLAN_VID_MASK, active_vlans)) 539 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 540 } 541 542 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 543 } 544 545 static 546 struct ixgbevf_rx_buffer *ixgbevf_get_rx_buffer(struct ixgbevf_ring *rx_ring, 547 const unsigned int size) 548 { 549 struct ixgbevf_rx_buffer *rx_buffer; 550 551 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 552 prefetchw(rx_buffer->page); 553 554 /* we are reusing so sync this buffer for CPU use */ 555 dma_sync_single_range_for_cpu(rx_ring->dev, 556 rx_buffer->dma, 557 rx_buffer->page_offset, 558 size, 559 DMA_FROM_DEVICE); 560 561 rx_buffer->pagecnt_bias--; 562 563 return rx_buffer; 564 } 565 566 static void ixgbevf_put_rx_buffer(struct ixgbevf_ring *rx_ring, 567 struct ixgbevf_rx_buffer *rx_buffer, 568 struct sk_buff *skb) 569 { 570 if (ixgbevf_can_reuse_rx_page(rx_buffer)) { 571 /* hand second half of page back to the ring */ 572 ixgbevf_reuse_rx_page(rx_ring, rx_buffer); 573 } else { 574 if (IS_ERR(skb)) 575 /* We are not reusing the buffer so unmap it and free 576 * any references we are holding to it 577 */ 578 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 579 ixgbevf_rx_pg_size(rx_ring), 580 DMA_FROM_DEVICE, 581 IXGBEVF_RX_DMA_ATTR); 582 __page_frag_cache_drain(rx_buffer->page, 583 rx_buffer->pagecnt_bias); 584 } 585 586 /* clear contents of rx_buffer */ 587 rx_buffer->page = NULL; 588 } 589 590 /** 591 * ixgbevf_is_non_eop - process handling of non-EOP buffers 592 * @rx_ring: Rx ring being processed 593 * @rx_desc: Rx descriptor for current buffer 594 * 595 * This function updates next to clean. If the buffer is an EOP buffer 596 * this function exits returning false, otherwise it will place the 597 * sk_buff in the next buffer to be chained and return true indicating 598 * that this is in fact a non-EOP buffer. 599 **/ 600 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, 601 union ixgbe_adv_rx_desc *rx_desc) 602 { 603 u32 ntc = rx_ring->next_to_clean + 1; 604 605 /* fetch, update, and store next to clean */ 606 ntc = (ntc < rx_ring->count) ? ntc : 0; 607 rx_ring->next_to_clean = ntc; 608 609 prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); 610 611 if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 612 return false; 613 614 return true; 615 } 616 617 static inline unsigned int ixgbevf_rx_offset(struct ixgbevf_ring *rx_ring) 618 { 619 return ring_uses_build_skb(rx_ring) ? IXGBEVF_SKB_PAD : 0; 620 } 621 622 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, 623 struct ixgbevf_rx_buffer *bi) 624 { 625 struct page *page = bi->page; 626 dma_addr_t dma; 627 628 /* since we are recycling buffers we should seldom need to alloc */ 629 if (likely(page)) 630 return true; 631 632 /* alloc new page for storage */ 633 page = dev_alloc_pages(ixgbevf_rx_pg_order(rx_ring)); 634 if (unlikely(!page)) { 635 rx_ring->rx_stats.alloc_rx_page_failed++; 636 return false; 637 } 638 639 /* map page for use */ 640 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 641 ixgbevf_rx_pg_size(rx_ring), 642 DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR); 643 644 /* if mapping failed free memory back to system since 645 * there isn't much point in holding memory we can't use 646 */ 647 if (dma_mapping_error(rx_ring->dev, dma)) { 648 __free_pages(page, ixgbevf_rx_pg_order(rx_ring)); 649 650 rx_ring->rx_stats.alloc_rx_page_failed++; 651 return false; 652 } 653 654 bi->dma = dma; 655 bi->page = page; 656 bi->page_offset = ixgbevf_rx_offset(rx_ring); 657 bi->pagecnt_bias = 1; 658 rx_ring->rx_stats.alloc_rx_page++; 659 660 return true; 661 } 662 663 /** 664 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 665 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on 666 * @cleaned_count: number of buffers to replace 667 **/ 668 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, 669 u16 cleaned_count) 670 { 671 union ixgbe_adv_rx_desc *rx_desc; 672 struct ixgbevf_rx_buffer *bi; 673 unsigned int i = rx_ring->next_to_use; 674 675 /* nothing to do or no valid netdev defined */ 676 if (!cleaned_count || !rx_ring->netdev) 677 return; 678 679 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 680 bi = &rx_ring->rx_buffer_info[i]; 681 i -= rx_ring->count; 682 683 do { 684 if (!ixgbevf_alloc_mapped_page(rx_ring, bi)) 685 break; 686 687 /* sync the buffer for use by the device */ 688 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 689 bi->page_offset, 690 ixgbevf_rx_bufsz(rx_ring), 691 DMA_FROM_DEVICE); 692 693 /* Refresh the desc even if pkt_addr didn't change 694 * because each write-back erases this info. 695 */ 696 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 697 698 rx_desc++; 699 bi++; 700 i++; 701 if (unlikely(!i)) { 702 rx_desc = IXGBEVF_RX_DESC(rx_ring, 0); 703 bi = rx_ring->rx_buffer_info; 704 i -= rx_ring->count; 705 } 706 707 /* clear the length for the next_to_use descriptor */ 708 rx_desc->wb.upper.length = 0; 709 710 cleaned_count--; 711 } while (cleaned_count); 712 713 i += rx_ring->count; 714 715 if (rx_ring->next_to_use != i) { 716 /* record the next descriptor to use */ 717 rx_ring->next_to_use = i; 718 719 /* update next to alloc since we have filled the ring */ 720 rx_ring->next_to_alloc = i; 721 722 /* Force memory writes to complete before letting h/w 723 * know there are new descriptors to fetch. (Only 724 * applicable for weak-ordered memory model archs, 725 * such as IA-64). 726 */ 727 wmb(); 728 ixgbevf_write_tail(rx_ring, i); 729 } 730 } 731 732 /** 733 * ixgbevf_cleanup_headers - Correct corrupted or empty headers 734 * @rx_ring: rx descriptor ring packet is being transacted on 735 * @rx_desc: pointer to the EOP Rx descriptor 736 * @skb: pointer to current skb being fixed 737 * 738 * Check for corrupted packet headers caused by senders on the local L2 739 * embedded NIC switch not setting up their Tx Descriptors right. These 740 * should be very rare. 741 * 742 * Also address the case where we are pulling data in on pages only 743 * and as such no data is present in the skb header. 744 * 745 * In addition if skb is not at least 60 bytes we need to pad it so that 746 * it is large enough to qualify as a valid Ethernet frame. 747 * 748 * Returns true if an error was encountered and skb was freed. 749 **/ 750 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, 751 union ixgbe_adv_rx_desc *rx_desc, 752 struct sk_buff *skb) 753 { 754 /* XDP packets use error pointer so abort at this point */ 755 if (IS_ERR(skb)) 756 return true; 757 758 /* verify that the packet does not have any known errors */ 759 if (unlikely(ixgbevf_test_staterr(rx_desc, 760 IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { 761 struct net_device *netdev = rx_ring->netdev; 762 763 if (!(netdev->features & NETIF_F_RXALL)) { 764 dev_kfree_skb_any(skb); 765 return true; 766 } 767 } 768 769 /* if eth_skb_pad returns an error the skb was freed */ 770 if (eth_skb_pad(skb)) 771 return true; 772 773 return false; 774 } 775 776 /** 777 * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring 778 * @rx_ring: rx descriptor ring to store buffers on 779 * @old_buff: donor buffer to have page reused 780 * 781 * Synchronizes page for reuse by the adapter 782 **/ 783 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, 784 struct ixgbevf_rx_buffer *old_buff) 785 { 786 struct ixgbevf_rx_buffer *new_buff; 787 u16 nta = rx_ring->next_to_alloc; 788 789 new_buff = &rx_ring->rx_buffer_info[nta]; 790 791 /* update, and store next to alloc */ 792 nta++; 793 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 794 795 /* transfer page from old buffer to new buffer */ 796 new_buff->page = old_buff->page; 797 new_buff->dma = old_buff->dma; 798 new_buff->page_offset = old_buff->page_offset; 799 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 800 } 801 802 static inline bool ixgbevf_page_is_reserved(struct page *page) 803 { 804 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 805 } 806 807 static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer) 808 { 809 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 810 struct page *page = rx_buffer->page; 811 812 /* avoid re-using remote pages */ 813 if (unlikely(ixgbevf_page_is_reserved(page))) 814 return false; 815 816 #if (PAGE_SIZE < 8192) 817 /* if we are only owner of page we can reuse it */ 818 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 819 return false; 820 #else 821 #define IXGBEVF_LAST_OFFSET \ 822 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBEVF_RXBUFFER_2048) 823 824 if (rx_buffer->page_offset > IXGBEVF_LAST_OFFSET) 825 return false; 826 827 #endif 828 829 /* If we have drained the page fragment pool we need to update 830 * the pagecnt_bias and page count so that we fully restock the 831 * number of references the driver holds. 832 */ 833 if (unlikely(!pagecnt_bias)) { 834 page_ref_add(page, USHRT_MAX); 835 rx_buffer->pagecnt_bias = USHRT_MAX; 836 } 837 838 return true; 839 } 840 841 /** 842 * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff 843 * @rx_ring: rx descriptor ring to transact packets on 844 * @rx_buffer: buffer containing page to add 845 * @skb: sk_buff to place the data into 846 * @size: size of buffer to be added 847 * 848 * This function will add the data contained in rx_buffer->page to the skb. 849 **/ 850 static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, 851 struct ixgbevf_rx_buffer *rx_buffer, 852 struct sk_buff *skb, 853 unsigned int size) 854 { 855 #if (PAGE_SIZE < 8192) 856 unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; 857 #else 858 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 859 SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) : 860 SKB_DATA_ALIGN(size); 861 #endif 862 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 863 rx_buffer->page_offset, size, truesize); 864 #if (PAGE_SIZE < 8192) 865 rx_buffer->page_offset ^= truesize; 866 #else 867 rx_buffer->page_offset += truesize; 868 #endif 869 } 870 871 static 872 struct sk_buff *ixgbevf_construct_skb(struct ixgbevf_ring *rx_ring, 873 struct ixgbevf_rx_buffer *rx_buffer, 874 struct xdp_buff *xdp, 875 union ixgbe_adv_rx_desc *rx_desc) 876 { 877 unsigned int size = xdp->data_end - xdp->data; 878 #if (PAGE_SIZE < 8192) 879 unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; 880 #else 881 unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end - 882 xdp->data_hard_start); 883 #endif 884 unsigned int headlen; 885 struct sk_buff *skb; 886 887 /* prefetch first cache line of first page */ 888 prefetch(xdp->data); 889 #if L1_CACHE_BYTES < 128 890 prefetch(xdp->data + L1_CACHE_BYTES); 891 #endif 892 /* Note, we get here by enabling legacy-rx via: 893 * 894 * ethtool --set-priv-flags <dev> legacy-rx on 895 * 896 * In this mode, we currently get 0 extra XDP headroom as 897 * opposed to having legacy-rx off, where we process XDP 898 * packets going to stack via ixgbevf_build_skb(). 899 * 900 * For ixgbevf_construct_skb() mode it means that the 901 * xdp->data_meta will always point to xdp->data, since 902 * the helper cannot expand the head. Should this ever 903 * changed in future for legacy-rx mode on, then lets also 904 * add xdp->data_meta handling here. 905 */ 906 907 /* allocate a skb to store the frags */ 908 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBEVF_RX_HDR_SIZE); 909 if (unlikely(!skb)) 910 return NULL; 911 912 /* Determine available headroom for copy */ 913 headlen = size; 914 if (headlen > IXGBEVF_RX_HDR_SIZE) 915 headlen = eth_get_headlen(xdp->data, IXGBEVF_RX_HDR_SIZE); 916 917 /* align pull length to size of long to optimize memcpy performance */ 918 memcpy(__skb_put(skb, headlen), xdp->data, 919 ALIGN(headlen, sizeof(long))); 920 921 /* update all of the pointers */ 922 size -= headlen; 923 if (size) { 924 skb_add_rx_frag(skb, 0, rx_buffer->page, 925 (xdp->data + headlen) - 926 page_address(rx_buffer->page), 927 size, truesize); 928 #if (PAGE_SIZE < 8192) 929 rx_buffer->page_offset ^= truesize; 930 #else 931 rx_buffer->page_offset += truesize; 932 #endif 933 } else { 934 rx_buffer->pagecnt_bias++; 935 } 936 937 return skb; 938 } 939 940 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 941 u32 qmask) 942 { 943 struct ixgbe_hw *hw = &adapter->hw; 944 945 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); 946 } 947 948 static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring, 949 struct ixgbevf_rx_buffer *rx_buffer, 950 struct xdp_buff *xdp, 951 union ixgbe_adv_rx_desc *rx_desc) 952 { 953 unsigned int metasize = xdp->data - xdp->data_meta; 954 #if (PAGE_SIZE < 8192) 955 unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; 956 #else 957 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 958 SKB_DATA_ALIGN(xdp->data_end - 959 xdp->data_hard_start); 960 #endif 961 struct sk_buff *skb; 962 963 /* Prefetch first cache line of first page. If xdp->data_meta 964 * is unused, this points to xdp->data, otherwise, we likely 965 * have a consumer accessing first few bytes of meta data, 966 * and then actual data. 967 */ 968 prefetch(xdp->data_meta); 969 #if L1_CACHE_BYTES < 128 970 prefetch(xdp->data_meta + L1_CACHE_BYTES); 971 #endif 972 973 /* build an skb around the page buffer */ 974 skb = build_skb(xdp->data_hard_start, truesize); 975 if (unlikely(!skb)) 976 return NULL; 977 978 /* update pointers within the skb to store the data */ 979 skb_reserve(skb, xdp->data - xdp->data_hard_start); 980 __skb_put(skb, xdp->data_end - xdp->data); 981 if (metasize) 982 skb_metadata_set(skb, metasize); 983 984 /* update buffer offset */ 985 #if (PAGE_SIZE < 8192) 986 rx_buffer->page_offset ^= truesize; 987 #else 988 rx_buffer->page_offset += truesize; 989 #endif 990 991 return skb; 992 } 993 994 #define IXGBEVF_XDP_PASS 0 995 #define IXGBEVF_XDP_CONSUMED 1 996 #define IXGBEVF_XDP_TX 2 997 998 static int ixgbevf_xmit_xdp_ring(struct ixgbevf_ring *ring, 999 struct xdp_buff *xdp) 1000 { 1001 struct ixgbevf_tx_buffer *tx_buffer; 1002 union ixgbe_adv_tx_desc *tx_desc; 1003 u32 len, cmd_type; 1004 dma_addr_t dma; 1005 u16 i; 1006 1007 len = xdp->data_end - xdp->data; 1008 1009 if (unlikely(!ixgbevf_desc_unused(ring))) 1010 return IXGBEVF_XDP_CONSUMED; 1011 1012 dma = dma_map_single(ring->dev, xdp->data, len, DMA_TO_DEVICE); 1013 if (dma_mapping_error(ring->dev, dma)) 1014 return IXGBEVF_XDP_CONSUMED; 1015 1016 /* record the location of the first descriptor for this packet */ 1017 tx_buffer = &ring->tx_buffer_info[ring->next_to_use]; 1018 tx_buffer->bytecount = len; 1019 tx_buffer->gso_segs = 1; 1020 tx_buffer->protocol = 0; 1021 1022 i = ring->next_to_use; 1023 tx_desc = IXGBEVF_TX_DESC(ring, i); 1024 1025 dma_unmap_len_set(tx_buffer, len, len); 1026 dma_unmap_addr_set(tx_buffer, dma, dma); 1027 tx_buffer->data = xdp->data; 1028 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1029 1030 /* put descriptor type bits */ 1031 cmd_type = IXGBE_ADVTXD_DTYP_DATA | 1032 IXGBE_ADVTXD_DCMD_DEXT | 1033 IXGBE_ADVTXD_DCMD_IFCS; 1034 cmd_type |= len | IXGBE_TXD_CMD; 1035 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1036 tx_desc->read.olinfo_status = 1037 cpu_to_le32((len << IXGBE_ADVTXD_PAYLEN_SHIFT) | 1038 IXGBE_ADVTXD_CC); 1039 1040 /* Avoid any potential race with cleanup */ 1041 smp_wmb(); 1042 1043 /* set next_to_watch value indicating a packet is present */ 1044 i++; 1045 if (i == ring->count) 1046 i = 0; 1047 1048 tx_buffer->next_to_watch = tx_desc; 1049 ring->next_to_use = i; 1050 1051 return IXGBEVF_XDP_TX; 1052 } 1053 1054 static struct sk_buff *ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, 1055 struct ixgbevf_ring *rx_ring, 1056 struct xdp_buff *xdp) 1057 { 1058 int result = IXGBEVF_XDP_PASS; 1059 struct ixgbevf_ring *xdp_ring; 1060 struct bpf_prog *xdp_prog; 1061 u32 act; 1062 1063 rcu_read_lock(); 1064 xdp_prog = READ_ONCE(rx_ring->xdp_prog); 1065 1066 if (!xdp_prog) 1067 goto xdp_out; 1068 1069 act = bpf_prog_run_xdp(xdp_prog, xdp); 1070 switch (act) { 1071 case XDP_PASS: 1072 break; 1073 case XDP_TX: 1074 xdp_ring = adapter->xdp_ring[rx_ring->queue_index]; 1075 result = ixgbevf_xmit_xdp_ring(xdp_ring, xdp); 1076 break; 1077 default: 1078 bpf_warn_invalid_xdp_action(act); 1079 /* fallthrough */ 1080 case XDP_ABORTED: 1081 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); 1082 /* fallthrough -- handle aborts by dropping packet */ 1083 case XDP_DROP: 1084 result = IXGBEVF_XDP_CONSUMED; 1085 break; 1086 } 1087 xdp_out: 1088 rcu_read_unlock(); 1089 return ERR_PTR(-result); 1090 } 1091 1092 static void ixgbevf_rx_buffer_flip(struct ixgbevf_ring *rx_ring, 1093 struct ixgbevf_rx_buffer *rx_buffer, 1094 unsigned int size) 1095 { 1096 #if (PAGE_SIZE < 8192) 1097 unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; 1098 1099 rx_buffer->page_offset ^= truesize; 1100 #else 1101 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1102 SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) : 1103 SKB_DATA_ALIGN(size); 1104 1105 rx_buffer->page_offset += truesize; 1106 #endif 1107 } 1108 1109 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 1110 struct ixgbevf_ring *rx_ring, 1111 int budget) 1112 { 1113 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1114 struct ixgbevf_adapter *adapter = q_vector->adapter; 1115 u16 cleaned_count = ixgbevf_desc_unused(rx_ring); 1116 struct sk_buff *skb = rx_ring->skb; 1117 bool xdp_xmit = false; 1118 struct xdp_buff xdp; 1119 1120 xdp.rxq = &rx_ring->xdp_rxq; 1121 1122 while (likely(total_rx_packets < budget)) { 1123 struct ixgbevf_rx_buffer *rx_buffer; 1124 union ixgbe_adv_rx_desc *rx_desc; 1125 unsigned int size; 1126 1127 /* return some buffers to hardware, one at a time is too slow */ 1128 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 1129 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count); 1130 cleaned_count = 0; 1131 } 1132 1133 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); 1134 size = le16_to_cpu(rx_desc->wb.upper.length); 1135 if (!size) 1136 break; 1137 1138 /* This memory barrier is needed to keep us from reading 1139 * any other fields out of the rx_desc until we know the 1140 * RXD_STAT_DD bit is set 1141 */ 1142 rmb(); 1143 1144 rx_buffer = ixgbevf_get_rx_buffer(rx_ring, size); 1145 1146 /* retrieve a buffer from the ring */ 1147 if (!skb) { 1148 xdp.data = page_address(rx_buffer->page) + 1149 rx_buffer->page_offset; 1150 xdp.data_meta = xdp.data; 1151 xdp.data_hard_start = xdp.data - 1152 ixgbevf_rx_offset(rx_ring); 1153 xdp.data_end = xdp.data + size; 1154 1155 skb = ixgbevf_run_xdp(adapter, rx_ring, &xdp); 1156 } 1157 1158 if (IS_ERR(skb)) { 1159 if (PTR_ERR(skb) == -IXGBEVF_XDP_TX) { 1160 xdp_xmit = true; 1161 ixgbevf_rx_buffer_flip(rx_ring, rx_buffer, 1162 size); 1163 } else { 1164 rx_buffer->pagecnt_bias++; 1165 } 1166 total_rx_packets++; 1167 total_rx_bytes += size; 1168 } else if (skb) { 1169 ixgbevf_add_rx_frag(rx_ring, rx_buffer, skb, size); 1170 } else if (ring_uses_build_skb(rx_ring)) { 1171 skb = ixgbevf_build_skb(rx_ring, rx_buffer, 1172 &xdp, rx_desc); 1173 } else { 1174 skb = ixgbevf_construct_skb(rx_ring, rx_buffer, 1175 &xdp, rx_desc); 1176 } 1177 1178 /* exit if we failed to retrieve a buffer */ 1179 if (!skb) { 1180 rx_ring->rx_stats.alloc_rx_buff_failed++; 1181 rx_buffer->pagecnt_bias++; 1182 break; 1183 } 1184 1185 ixgbevf_put_rx_buffer(rx_ring, rx_buffer, skb); 1186 cleaned_count++; 1187 1188 /* fetch next buffer in frame if non-eop */ 1189 if (ixgbevf_is_non_eop(rx_ring, rx_desc)) 1190 continue; 1191 1192 /* verify the packet layout is correct */ 1193 if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) { 1194 skb = NULL; 1195 continue; 1196 } 1197 1198 /* probably a little skewed due to removing CRC */ 1199 total_rx_bytes += skb->len; 1200 1201 /* Workaround hardware that can't do proper VEPA multicast 1202 * source pruning. 1203 */ 1204 if ((skb->pkt_type == PACKET_BROADCAST || 1205 skb->pkt_type == PACKET_MULTICAST) && 1206 ether_addr_equal(rx_ring->netdev->dev_addr, 1207 eth_hdr(skb)->h_source)) { 1208 dev_kfree_skb_irq(skb); 1209 continue; 1210 } 1211 1212 /* populate checksum, VLAN, and protocol */ 1213 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); 1214 1215 ixgbevf_rx_skb(q_vector, skb); 1216 1217 /* reset skb pointer */ 1218 skb = NULL; 1219 1220 /* update budget accounting */ 1221 total_rx_packets++; 1222 } 1223 1224 /* place incomplete frames back on ring for completion */ 1225 rx_ring->skb = skb; 1226 1227 if (xdp_xmit) { 1228 struct ixgbevf_ring *xdp_ring = 1229 adapter->xdp_ring[rx_ring->queue_index]; 1230 1231 /* Force memory writes to complete before letting h/w 1232 * know there are new descriptors to fetch. 1233 */ 1234 wmb(); 1235 ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); 1236 } 1237 1238 u64_stats_update_begin(&rx_ring->syncp); 1239 rx_ring->stats.packets += total_rx_packets; 1240 rx_ring->stats.bytes += total_rx_bytes; 1241 u64_stats_update_end(&rx_ring->syncp); 1242 q_vector->rx.total_packets += total_rx_packets; 1243 q_vector->rx.total_bytes += total_rx_bytes; 1244 1245 return total_rx_packets; 1246 } 1247 1248 /** 1249 * ixgbevf_poll - NAPI polling calback 1250 * @napi: napi struct with our devices info in it 1251 * @budget: amount of work driver is allowed to do this pass, in packets 1252 * 1253 * This function will clean more than one or more rings associated with a 1254 * q_vector. 1255 **/ 1256 static int ixgbevf_poll(struct napi_struct *napi, int budget) 1257 { 1258 struct ixgbevf_q_vector *q_vector = 1259 container_of(napi, struct ixgbevf_q_vector, napi); 1260 struct ixgbevf_adapter *adapter = q_vector->adapter; 1261 struct ixgbevf_ring *ring; 1262 int per_ring_budget, work_done = 0; 1263 bool clean_complete = true; 1264 1265 ixgbevf_for_each_ring(ring, q_vector->tx) { 1266 if (!ixgbevf_clean_tx_irq(q_vector, ring, budget)) 1267 clean_complete = false; 1268 } 1269 1270 if (budget <= 0) 1271 return budget; 1272 1273 /* attempt to distribute budget to each queue fairly, but don't allow 1274 * the budget to go below 1 because we'll exit polling 1275 */ 1276 if (q_vector->rx.count > 1) 1277 per_ring_budget = max(budget/q_vector->rx.count, 1); 1278 else 1279 per_ring_budget = budget; 1280 1281 ixgbevf_for_each_ring(ring, q_vector->rx) { 1282 int cleaned = ixgbevf_clean_rx_irq(q_vector, ring, 1283 per_ring_budget); 1284 work_done += cleaned; 1285 if (cleaned >= per_ring_budget) 1286 clean_complete = false; 1287 } 1288 1289 /* If all work not completed, return budget and keep polling */ 1290 if (!clean_complete) 1291 return budget; 1292 /* all work done, exit the polling mode */ 1293 napi_complete_done(napi, work_done); 1294 if (adapter->rx_itr_setting == 1) 1295 ixgbevf_set_itr(q_vector); 1296 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 1297 !test_bit(__IXGBEVF_REMOVING, &adapter->state)) 1298 ixgbevf_irq_enable_queues(adapter, 1299 BIT(q_vector->v_idx)); 1300 1301 return 0; 1302 } 1303 1304 /** 1305 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 1306 * @q_vector: structure containing interrupt and ring information 1307 **/ 1308 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) 1309 { 1310 struct ixgbevf_adapter *adapter = q_vector->adapter; 1311 struct ixgbe_hw *hw = &adapter->hw; 1312 int v_idx = q_vector->v_idx; 1313 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 1314 1315 /* set the WDIS bit to not clear the timer bits and cause an 1316 * immediate assertion of the interrupt 1317 */ 1318 itr_reg |= IXGBE_EITR_CNT_WDIS; 1319 1320 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 1321 } 1322 1323 /** 1324 * ixgbevf_configure_msix - Configure MSI-X hardware 1325 * @adapter: board private structure 1326 * 1327 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 1328 * interrupts. 1329 **/ 1330 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 1331 { 1332 struct ixgbevf_q_vector *q_vector; 1333 int q_vectors, v_idx; 1334 1335 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1336 adapter->eims_enable_mask = 0; 1337 1338 /* Populate the IVAR table and set the ITR values to the 1339 * corresponding register. 1340 */ 1341 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 1342 struct ixgbevf_ring *ring; 1343 1344 q_vector = adapter->q_vector[v_idx]; 1345 1346 ixgbevf_for_each_ring(ring, q_vector->rx) 1347 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx); 1348 1349 ixgbevf_for_each_ring(ring, q_vector->tx) 1350 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx); 1351 1352 if (q_vector->tx.ring && !q_vector->rx.ring) { 1353 /* Tx only vector */ 1354 if (adapter->tx_itr_setting == 1) 1355 q_vector->itr = IXGBE_12K_ITR; 1356 else 1357 q_vector->itr = adapter->tx_itr_setting; 1358 } else { 1359 /* Rx or Rx/Tx vector */ 1360 if (adapter->rx_itr_setting == 1) 1361 q_vector->itr = IXGBE_20K_ITR; 1362 else 1363 q_vector->itr = adapter->rx_itr_setting; 1364 } 1365 1366 /* add q_vector eims value to global eims_enable_mask */ 1367 adapter->eims_enable_mask |= BIT(v_idx); 1368 1369 ixgbevf_write_eitr(q_vector); 1370 } 1371 1372 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 1373 /* setup eims_other and add value to global eims_enable_mask */ 1374 adapter->eims_other = BIT(v_idx); 1375 adapter->eims_enable_mask |= adapter->eims_other; 1376 } 1377 1378 enum latency_range { 1379 lowest_latency = 0, 1380 low_latency = 1, 1381 bulk_latency = 2, 1382 latency_invalid = 255 1383 }; 1384 1385 /** 1386 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 1387 * @q_vector: structure containing interrupt and ring information 1388 * @ring_container: structure containing ring performance data 1389 * 1390 * Stores a new ITR value based on packets and byte 1391 * counts during the last interrupt. The advantage of per interrupt 1392 * computation is faster updates and more accurate ITR for the current 1393 * traffic pattern. Constants in this function were computed 1394 * based on theoretical maximum wire speed and thresholds were set based 1395 * on testing data as well as attempting to minimize response time 1396 * while increasing bulk throughput. 1397 **/ 1398 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector, 1399 struct ixgbevf_ring_container *ring_container) 1400 { 1401 int bytes = ring_container->total_bytes; 1402 int packets = ring_container->total_packets; 1403 u32 timepassed_us; 1404 u64 bytes_perint; 1405 u8 itr_setting = ring_container->itr; 1406 1407 if (packets == 0) 1408 return; 1409 1410 /* simple throttle rate management 1411 * 0-20MB/s lowest (100000 ints/s) 1412 * 20-100MB/s low (20000 ints/s) 1413 * 100-1249MB/s bulk (12000 ints/s) 1414 */ 1415 /* what was last interrupt timeslice? */ 1416 timepassed_us = q_vector->itr >> 2; 1417 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 1418 1419 switch (itr_setting) { 1420 case lowest_latency: 1421 if (bytes_perint > 10) 1422 itr_setting = low_latency; 1423 break; 1424 case low_latency: 1425 if (bytes_perint > 20) 1426 itr_setting = bulk_latency; 1427 else if (bytes_perint <= 10) 1428 itr_setting = lowest_latency; 1429 break; 1430 case bulk_latency: 1431 if (bytes_perint <= 20) 1432 itr_setting = low_latency; 1433 break; 1434 } 1435 1436 /* clear work counters since we have the values we need */ 1437 ring_container->total_bytes = 0; 1438 ring_container->total_packets = 0; 1439 1440 /* write updated itr to ring container */ 1441 ring_container->itr = itr_setting; 1442 } 1443 1444 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector) 1445 { 1446 u32 new_itr = q_vector->itr; 1447 u8 current_itr; 1448 1449 ixgbevf_update_itr(q_vector, &q_vector->tx); 1450 ixgbevf_update_itr(q_vector, &q_vector->rx); 1451 1452 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 1453 1454 switch (current_itr) { 1455 /* counts and packets in update_itr are dependent on these numbers */ 1456 case lowest_latency: 1457 new_itr = IXGBE_100K_ITR; 1458 break; 1459 case low_latency: 1460 new_itr = IXGBE_20K_ITR; 1461 break; 1462 case bulk_latency: 1463 new_itr = IXGBE_12K_ITR; 1464 break; 1465 default: 1466 break; 1467 } 1468 1469 if (new_itr != q_vector->itr) { 1470 /* do an exponential smoothing */ 1471 new_itr = (10 * new_itr * q_vector->itr) / 1472 ((9 * new_itr) + q_vector->itr); 1473 1474 /* save the algorithm value here */ 1475 q_vector->itr = new_itr; 1476 1477 ixgbevf_write_eitr(q_vector); 1478 } 1479 } 1480 1481 static irqreturn_t ixgbevf_msix_other(int irq, void *data) 1482 { 1483 struct ixgbevf_adapter *adapter = data; 1484 struct ixgbe_hw *hw = &adapter->hw; 1485 1486 hw->mac.get_link_status = 1; 1487 1488 ixgbevf_service_event_schedule(adapter); 1489 1490 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); 1491 1492 return IRQ_HANDLED; 1493 } 1494 1495 /** 1496 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) 1497 * @irq: unused 1498 * @data: pointer to our q_vector struct for this interrupt vector 1499 **/ 1500 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data) 1501 { 1502 struct ixgbevf_q_vector *q_vector = data; 1503 1504 /* EIAM disabled interrupts (on this vector) for us */ 1505 if (q_vector->rx.ring || q_vector->tx.ring) 1506 napi_schedule_irqoff(&q_vector->napi); 1507 1508 return IRQ_HANDLED; 1509 } 1510 1511 /** 1512 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 1513 * @adapter: board private structure 1514 * 1515 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 1516 * interrupts from the kernel. 1517 **/ 1518 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 1519 { 1520 struct net_device *netdev = adapter->netdev; 1521 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1522 unsigned int ri = 0, ti = 0; 1523 int vector, err; 1524 1525 for (vector = 0; vector < q_vectors; vector++) { 1526 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector]; 1527 struct msix_entry *entry = &adapter->msix_entries[vector]; 1528 1529 if (q_vector->tx.ring && q_vector->rx.ring) { 1530 snprintf(q_vector->name, sizeof(q_vector->name), 1531 "%s-TxRx-%u", netdev->name, ri++); 1532 ti++; 1533 } else if (q_vector->rx.ring) { 1534 snprintf(q_vector->name, sizeof(q_vector->name), 1535 "%s-rx-%u", netdev->name, ri++); 1536 } else if (q_vector->tx.ring) { 1537 snprintf(q_vector->name, sizeof(q_vector->name), 1538 "%s-tx-%u", netdev->name, ti++); 1539 } else { 1540 /* skip this unused q_vector */ 1541 continue; 1542 } 1543 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0, 1544 q_vector->name, q_vector); 1545 if (err) { 1546 hw_dbg(&adapter->hw, 1547 "request_irq failed for MSIX interrupt Error: %d\n", 1548 err); 1549 goto free_queue_irqs; 1550 } 1551 } 1552 1553 err = request_irq(adapter->msix_entries[vector].vector, 1554 &ixgbevf_msix_other, 0, netdev->name, adapter); 1555 if (err) { 1556 hw_dbg(&adapter->hw, "request_irq for msix_other failed: %d\n", 1557 err); 1558 goto free_queue_irqs; 1559 } 1560 1561 return 0; 1562 1563 free_queue_irqs: 1564 while (vector) { 1565 vector--; 1566 free_irq(adapter->msix_entries[vector].vector, 1567 adapter->q_vector[vector]); 1568 } 1569 /* This failure is non-recoverable - it indicates the system is 1570 * out of MSIX vector resources and the VF driver cannot run 1571 * without them. Set the number of msix vectors to zero 1572 * indicating that not enough can be allocated. The error 1573 * will be returned to the user indicating device open failed. 1574 * Any further attempts to force the driver to open will also 1575 * fail. The only way to recover is to unload the driver and 1576 * reload it again. If the system has recovered some MSIX 1577 * vectors then it may succeed. 1578 */ 1579 adapter->num_msix_vectors = 0; 1580 return err; 1581 } 1582 1583 /** 1584 * ixgbevf_request_irq - initialize interrupts 1585 * @adapter: board private structure 1586 * 1587 * Attempts to configure interrupts using the best available 1588 * capabilities of the hardware and kernel. 1589 **/ 1590 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 1591 { 1592 int err = ixgbevf_request_msix_irqs(adapter); 1593 1594 if (err) 1595 hw_dbg(&adapter->hw, "request_irq failed, Error %d\n", err); 1596 1597 return err; 1598 } 1599 1600 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 1601 { 1602 int i, q_vectors; 1603 1604 if (!adapter->msix_entries) 1605 return; 1606 1607 q_vectors = adapter->num_msix_vectors; 1608 i = q_vectors - 1; 1609 1610 free_irq(adapter->msix_entries[i].vector, adapter); 1611 i--; 1612 1613 for (; i >= 0; i--) { 1614 /* free only the irqs that were actually requested */ 1615 if (!adapter->q_vector[i]->rx.ring && 1616 !adapter->q_vector[i]->tx.ring) 1617 continue; 1618 1619 free_irq(adapter->msix_entries[i].vector, 1620 adapter->q_vector[i]); 1621 } 1622 } 1623 1624 /** 1625 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1626 * @adapter: board private structure 1627 **/ 1628 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1629 { 1630 struct ixgbe_hw *hw = &adapter->hw; 1631 int i; 1632 1633 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0); 1634 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1635 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0); 1636 1637 IXGBE_WRITE_FLUSH(hw); 1638 1639 for (i = 0; i < adapter->num_msix_vectors; i++) 1640 synchronize_irq(adapter->msix_entries[i].vector); 1641 } 1642 1643 /** 1644 * ixgbevf_irq_enable - Enable default interrupt generation settings 1645 * @adapter: board private structure 1646 **/ 1647 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) 1648 { 1649 struct ixgbe_hw *hw = &adapter->hw; 1650 1651 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask); 1652 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask); 1653 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); 1654 } 1655 1656 /** 1657 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset 1658 * @adapter: board private structure 1659 * @ring: structure containing ring specific data 1660 * 1661 * Configure the Tx descriptor ring after a reset. 1662 **/ 1663 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 1664 struct ixgbevf_ring *ring) 1665 { 1666 struct ixgbe_hw *hw = &adapter->hw; 1667 u64 tdba = ring->dma; 1668 int wait_loop = 10; 1669 u32 txdctl = IXGBE_TXDCTL_ENABLE; 1670 u8 reg_idx = ring->reg_idx; 1671 1672 /* disable queue to avoid issues while updating state */ 1673 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 1674 IXGBE_WRITE_FLUSH(hw); 1675 1676 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 1677 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32); 1678 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx), 1679 ring->count * sizeof(union ixgbe_adv_tx_desc)); 1680 1681 /* disable head writeback */ 1682 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0); 1683 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0); 1684 1685 /* enable relaxed ordering */ 1686 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx), 1687 (IXGBE_DCA_TXCTRL_DESC_RRO_EN | 1688 IXGBE_DCA_TXCTRL_DATA_RRO_EN)); 1689 1690 /* reset head and tail pointers */ 1691 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0); 1692 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0); 1693 ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx); 1694 1695 /* reset ntu and ntc to place SW in sync with hardwdare */ 1696 ring->next_to_clean = 0; 1697 ring->next_to_use = 0; 1698 1699 /* In order to avoid issues WTHRESH + PTHRESH should always be equal 1700 * to or less than the number of on chip descriptors, which is 1701 * currently 40. 1702 */ 1703 txdctl |= (8 << 16); /* WTHRESH = 8 */ 1704 1705 /* Setting PTHRESH to 32 both improves performance */ 1706 txdctl |= (1u << 8) | /* HTHRESH = 1 */ 1707 32; /* PTHRESH = 32 */ 1708 1709 /* reinitialize tx_buffer_info */ 1710 memset(ring->tx_buffer_info, 0, 1711 sizeof(struct ixgbevf_tx_buffer) * ring->count); 1712 1713 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); 1714 1715 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl); 1716 1717 /* poll to verify queue is enabled */ 1718 do { 1719 usleep_range(1000, 2000); 1720 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx)); 1721 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 1722 if (!wait_loop) 1723 hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx); 1724 } 1725 1726 /** 1727 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1728 * @adapter: board private structure 1729 * 1730 * Configure the Tx unit of the MAC after a reset. 1731 **/ 1732 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1733 { 1734 u32 i; 1735 1736 /* Setup the HW Tx Head and Tail descriptor pointers */ 1737 for (i = 0; i < adapter->num_tx_queues; i++) 1738 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]); 1739 for (i = 0; i < adapter->num_xdp_queues; i++) 1740 ixgbevf_configure_tx_ring(adapter, adapter->xdp_ring[i]); 1741 } 1742 1743 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1744 1745 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, 1746 struct ixgbevf_ring *ring, int index) 1747 { 1748 struct ixgbe_hw *hw = &adapter->hw; 1749 u32 srrctl; 1750 1751 srrctl = IXGBE_SRRCTL_DROP_EN; 1752 1753 srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; 1754 if (ring_uses_large_buffer(ring)) 1755 srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1756 else 1757 srrctl |= IXGBEVF_RXBUFFER_2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1758 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1759 1760 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1761 } 1762 1763 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) 1764 { 1765 struct ixgbe_hw *hw = &adapter->hw; 1766 1767 /* PSRTYPE must be initialized in 82599 */ 1768 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 1769 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR | 1770 IXGBE_PSRTYPE_L2HDR; 1771 1772 if (adapter->num_rx_queues > 1) 1773 psrtype |= BIT(29); 1774 1775 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1776 } 1777 1778 #define IXGBEVF_MAX_RX_DESC_POLL 10 1779 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, 1780 struct ixgbevf_ring *ring) 1781 { 1782 struct ixgbe_hw *hw = &adapter->hw; 1783 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1784 u32 rxdctl; 1785 u8 reg_idx = ring->reg_idx; 1786 1787 if (IXGBE_REMOVED(hw->hw_addr)) 1788 return; 1789 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1790 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1791 1792 /* write value back with RXDCTL.ENABLE bit cleared */ 1793 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1794 1795 /* the hardware may take up to 100us to really disable the Rx queue */ 1796 do { 1797 udelay(10); 1798 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1799 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 1800 1801 if (!wait_loop) 1802 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", 1803 reg_idx); 1804 } 1805 1806 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1807 struct ixgbevf_ring *ring) 1808 { 1809 struct ixgbe_hw *hw = &adapter->hw; 1810 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1811 u32 rxdctl; 1812 u8 reg_idx = ring->reg_idx; 1813 1814 if (IXGBE_REMOVED(hw->hw_addr)) 1815 return; 1816 do { 1817 usleep_range(1000, 2000); 1818 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1819 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 1820 1821 if (!wait_loop) 1822 pr_err("RXDCTL.ENABLE queue %d not set while polling\n", 1823 reg_idx); 1824 } 1825 1826 /** 1827 * ixgbevf_init_rss_key - Initialize adapter RSS key 1828 * @adapter: device handle 1829 * 1830 * Allocates and initializes the RSS key if it is not allocated. 1831 **/ 1832 static inline int ixgbevf_init_rss_key(struct ixgbevf_adapter *adapter) 1833 { 1834 u32 *rss_key; 1835 1836 if (!adapter->rss_key) { 1837 rss_key = kzalloc(IXGBEVF_RSS_HASH_KEY_SIZE, GFP_KERNEL); 1838 if (unlikely(!rss_key)) 1839 return -ENOMEM; 1840 1841 netdev_rss_key_fill(rss_key, IXGBEVF_RSS_HASH_KEY_SIZE); 1842 adapter->rss_key = rss_key; 1843 } 1844 1845 return 0; 1846 } 1847 1848 static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) 1849 { 1850 struct ixgbe_hw *hw = &adapter->hw; 1851 u32 vfmrqc = 0, vfreta = 0; 1852 u16 rss_i = adapter->num_rx_queues; 1853 u8 i, j; 1854 1855 /* Fill out hash function seeds */ 1856 for (i = 0; i < IXGBEVF_VFRSSRK_REGS; i++) 1857 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), *(adapter->rss_key + i)); 1858 1859 for (i = 0, j = 0; i < IXGBEVF_X550_VFRETA_SIZE; i++, j++) { 1860 if (j == rss_i) 1861 j = 0; 1862 1863 adapter->rss_indir_tbl[i] = j; 1864 1865 vfreta |= j << (i & 0x3) * 8; 1866 if ((i & 3) == 3) { 1867 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), vfreta); 1868 vfreta = 0; 1869 } 1870 } 1871 1872 /* Perform hash on these packet types */ 1873 vfmrqc |= IXGBE_VFMRQC_RSS_FIELD_IPV4 | 1874 IXGBE_VFMRQC_RSS_FIELD_IPV4_TCP | 1875 IXGBE_VFMRQC_RSS_FIELD_IPV6 | 1876 IXGBE_VFMRQC_RSS_FIELD_IPV6_TCP; 1877 1878 vfmrqc |= IXGBE_VFMRQC_RSSEN; 1879 1880 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); 1881 } 1882 1883 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, 1884 struct ixgbevf_ring *ring) 1885 { 1886 struct ixgbe_hw *hw = &adapter->hw; 1887 union ixgbe_adv_rx_desc *rx_desc; 1888 u64 rdba = ring->dma; 1889 u32 rxdctl; 1890 u8 reg_idx = ring->reg_idx; 1891 1892 /* disable queue to avoid issues while updating state */ 1893 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1894 ixgbevf_disable_rx_queue(adapter, ring); 1895 1896 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 1897 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32); 1898 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx), 1899 ring->count * sizeof(union ixgbe_adv_rx_desc)); 1900 1901 #ifndef CONFIG_SPARC 1902 /* enable relaxed ordering */ 1903 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1904 IXGBE_DCA_RXCTRL_DESC_RRO_EN); 1905 #else 1906 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1907 IXGBE_DCA_RXCTRL_DESC_RRO_EN | 1908 IXGBE_DCA_RXCTRL_DATA_WRO_EN); 1909 #endif 1910 1911 /* reset head and tail pointers */ 1912 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0); 1913 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); 1914 ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); 1915 1916 /* initialize rx_buffer_info */ 1917 memset(ring->rx_buffer_info, 0, 1918 sizeof(struct ixgbevf_rx_buffer) * ring->count); 1919 1920 /* initialize Rx descriptor 0 */ 1921 rx_desc = IXGBEVF_RX_DESC(ring, 0); 1922 rx_desc->wb.upper.length = 0; 1923 1924 /* reset ntu and ntc to place SW in sync with hardwdare */ 1925 ring->next_to_clean = 0; 1926 ring->next_to_use = 0; 1927 ring->next_to_alloc = 0; 1928 1929 ixgbevf_configure_srrctl(adapter, ring, reg_idx); 1930 1931 /* RXDCTL.RLPML does not work on 82599 */ 1932 if (adapter->hw.mac.type != ixgbe_mac_82599_vf) { 1933 rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | 1934 IXGBE_RXDCTL_RLPML_EN); 1935 1936 #if (PAGE_SIZE < 8192) 1937 /* Limit the maximum frame size so we don't overrun the skb */ 1938 if (ring_uses_build_skb(ring) && 1939 !ring_uses_large_buffer(ring)) 1940 rxdctl |= IXGBEVF_MAX_FRAME_BUILD_SKB | 1941 IXGBE_RXDCTL_RLPML_EN; 1942 #endif 1943 } 1944 1945 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1946 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1947 1948 ixgbevf_rx_desc_queue_enable(adapter, ring); 1949 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); 1950 } 1951 1952 static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter, 1953 struct ixgbevf_ring *rx_ring) 1954 { 1955 struct net_device *netdev = adapter->netdev; 1956 unsigned int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 1957 1958 /* set build_skb and buffer size flags */ 1959 clear_ring_build_skb_enabled(rx_ring); 1960 clear_ring_uses_large_buffer(rx_ring); 1961 1962 if (adapter->flags & IXGBEVF_FLAGS_LEGACY_RX) 1963 return; 1964 1965 set_ring_build_skb_enabled(rx_ring); 1966 1967 if (PAGE_SIZE < 8192) { 1968 if (max_frame <= IXGBEVF_MAX_FRAME_BUILD_SKB) 1969 return; 1970 1971 set_ring_uses_large_buffer(rx_ring); 1972 } 1973 } 1974 1975 /** 1976 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1977 * @adapter: board private structure 1978 * 1979 * Configure the Rx unit of the MAC after a reset. 1980 **/ 1981 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1982 { 1983 struct ixgbe_hw *hw = &adapter->hw; 1984 struct net_device *netdev = adapter->netdev; 1985 int i, ret; 1986 1987 ixgbevf_setup_psrtype(adapter); 1988 if (hw->mac.type >= ixgbe_mac_X550_vf) 1989 ixgbevf_setup_vfmrqc(adapter); 1990 1991 spin_lock_bh(&adapter->mbx_lock); 1992 /* notify the PF of our intent to use this size of frame */ 1993 ret = hw->mac.ops.set_rlpml(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN); 1994 spin_unlock_bh(&adapter->mbx_lock); 1995 if (ret) 1996 dev_err(&adapter->pdev->dev, 1997 "Failed to set MTU at %d\n", netdev->mtu); 1998 1999 /* Setup the HW Rx Head and Tail Descriptor Pointers and 2000 * the Base and Length of the Rx Descriptor Ring 2001 */ 2002 for (i = 0; i < adapter->num_rx_queues; i++) { 2003 struct ixgbevf_ring *rx_ring = adapter->rx_ring[i]; 2004 2005 ixgbevf_set_rx_buffer_len(adapter, rx_ring); 2006 ixgbevf_configure_rx_ring(adapter, rx_ring); 2007 } 2008 } 2009 2010 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, 2011 __be16 proto, u16 vid) 2012 { 2013 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2014 struct ixgbe_hw *hw = &adapter->hw; 2015 int err; 2016 2017 spin_lock_bh(&adapter->mbx_lock); 2018 2019 /* add VID to filter table */ 2020 err = hw->mac.ops.set_vfta(hw, vid, 0, true); 2021 2022 spin_unlock_bh(&adapter->mbx_lock); 2023 2024 /* translate error return types so error makes sense */ 2025 if (err == IXGBE_ERR_MBX) 2026 return -EIO; 2027 2028 if (err == IXGBE_ERR_INVALID_ARGUMENT) 2029 return -EACCES; 2030 2031 set_bit(vid, adapter->active_vlans); 2032 2033 return err; 2034 } 2035 2036 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, 2037 __be16 proto, u16 vid) 2038 { 2039 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2040 struct ixgbe_hw *hw = &adapter->hw; 2041 int err; 2042 2043 spin_lock_bh(&adapter->mbx_lock); 2044 2045 /* remove VID from filter table */ 2046 err = hw->mac.ops.set_vfta(hw, vid, 0, false); 2047 2048 spin_unlock_bh(&adapter->mbx_lock); 2049 2050 clear_bit(vid, adapter->active_vlans); 2051 2052 return err; 2053 } 2054 2055 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 2056 { 2057 u16 vid; 2058 2059 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2060 ixgbevf_vlan_rx_add_vid(adapter->netdev, 2061 htons(ETH_P_8021Q), vid); 2062 } 2063 2064 static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 2065 { 2066 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2067 struct ixgbe_hw *hw = &adapter->hw; 2068 int count = 0; 2069 2070 if ((netdev_uc_count(netdev)) > 10) { 2071 pr_err("Too many unicast filters - No Space\n"); 2072 return -ENOSPC; 2073 } 2074 2075 if (!netdev_uc_empty(netdev)) { 2076 struct netdev_hw_addr *ha; 2077 2078 netdev_for_each_uc_addr(ha, netdev) { 2079 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 2080 udelay(200); 2081 } 2082 } else { 2083 /* If the list is empty then send message to PF driver to 2084 * clear all MAC VLANs on this VF. 2085 */ 2086 hw->mac.ops.set_uc_addr(hw, 0, NULL); 2087 } 2088 2089 return count; 2090 } 2091 2092 /** 2093 * ixgbevf_set_rx_mode - Multicast and unicast set 2094 * @netdev: network interface device structure 2095 * 2096 * The set_rx_method entry point is called whenever the multicast address 2097 * list, unicast address list or the network interface flags are updated. 2098 * This routine is responsible for configuring the hardware for proper 2099 * multicast mode and configuring requested unicast filters. 2100 **/ 2101 static void ixgbevf_set_rx_mode(struct net_device *netdev) 2102 { 2103 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2104 struct ixgbe_hw *hw = &adapter->hw; 2105 unsigned int flags = netdev->flags; 2106 int xcast_mode; 2107 2108 /* request the most inclusive mode we need */ 2109 if (flags & IFF_PROMISC) 2110 xcast_mode = IXGBEVF_XCAST_MODE_PROMISC; 2111 else if (flags & IFF_ALLMULTI) 2112 xcast_mode = IXGBEVF_XCAST_MODE_ALLMULTI; 2113 else if (flags & (IFF_BROADCAST | IFF_MULTICAST)) 2114 xcast_mode = IXGBEVF_XCAST_MODE_MULTI; 2115 else 2116 xcast_mode = IXGBEVF_XCAST_MODE_NONE; 2117 2118 spin_lock_bh(&adapter->mbx_lock); 2119 2120 hw->mac.ops.update_xcast_mode(hw, xcast_mode); 2121 2122 /* reprogram multicast list */ 2123 hw->mac.ops.update_mc_addr_list(hw, netdev); 2124 2125 ixgbevf_write_uc_addr_list(netdev); 2126 2127 spin_unlock_bh(&adapter->mbx_lock); 2128 } 2129 2130 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 2131 { 2132 int q_idx; 2133 struct ixgbevf_q_vector *q_vector; 2134 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2135 2136 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 2137 q_vector = adapter->q_vector[q_idx]; 2138 napi_enable(&q_vector->napi); 2139 } 2140 } 2141 2142 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 2143 { 2144 int q_idx; 2145 struct ixgbevf_q_vector *q_vector; 2146 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2147 2148 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 2149 q_vector = adapter->q_vector[q_idx]; 2150 napi_disable(&q_vector->napi); 2151 } 2152 } 2153 2154 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter) 2155 { 2156 struct ixgbe_hw *hw = &adapter->hw; 2157 unsigned int def_q = 0; 2158 unsigned int num_tcs = 0; 2159 unsigned int num_rx_queues = adapter->num_rx_queues; 2160 unsigned int num_tx_queues = adapter->num_tx_queues; 2161 int err; 2162 2163 spin_lock_bh(&adapter->mbx_lock); 2164 2165 /* fetch queue configuration from the PF */ 2166 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2167 2168 spin_unlock_bh(&adapter->mbx_lock); 2169 2170 if (err) 2171 return err; 2172 2173 if (num_tcs > 1) { 2174 /* we need only one Tx queue */ 2175 num_tx_queues = 1; 2176 2177 /* update default Tx ring register index */ 2178 adapter->tx_ring[0]->reg_idx = def_q; 2179 2180 /* we need as many queues as traffic classes */ 2181 num_rx_queues = num_tcs; 2182 } 2183 2184 /* if we have a bad config abort request queue reset */ 2185 if ((adapter->num_rx_queues != num_rx_queues) || 2186 (adapter->num_tx_queues != num_tx_queues)) { 2187 /* force mailbox timeout to prevent further messages */ 2188 hw->mbx.timeout = 0; 2189 2190 /* wait for watchdog to come around and bail us out */ 2191 set_bit(__IXGBEVF_QUEUE_RESET_REQUESTED, &adapter->state); 2192 } 2193 2194 return 0; 2195 } 2196 2197 static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 2198 { 2199 ixgbevf_configure_dcb(adapter); 2200 2201 ixgbevf_set_rx_mode(adapter->netdev); 2202 2203 ixgbevf_restore_vlan(adapter); 2204 2205 ixgbevf_configure_tx(adapter); 2206 ixgbevf_configure_rx(adapter); 2207 } 2208 2209 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 2210 { 2211 /* Only save pre-reset stats if there are some */ 2212 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 2213 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 2214 adapter->stats.base_vfgprc; 2215 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 2216 adapter->stats.base_vfgptc; 2217 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 2218 adapter->stats.base_vfgorc; 2219 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 2220 adapter->stats.base_vfgotc; 2221 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 2222 adapter->stats.base_vfmprc; 2223 } 2224 } 2225 2226 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 2227 { 2228 struct ixgbe_hw *hw = &adapter->hw; 2229 2230 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2231 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2232 adapter->stats.last_vfgorc |= 2233 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2234 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2235 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2236 adapter->stats.last_vfgotc |= 2237 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2238 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2239 2240 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 2241 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 2242 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 2243 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 2244 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 2245 } 2246 2247 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) 2248 { 2249 struct ixgbe_hw *hw = &adapter->hw; 2250 int api[] = { ixgbe_mbox_api_13, 2251 ixgbe_mbox_api_12, 2252 ixgbe_mbox_api_11, 2253 ixgbe_mbox_api_10, 2254 ixgbe_mbox_api_unknown }; 2255 int err, idx = 0; 2256 2257 spin_lock_bh(&adapter->mbx_lock); 2258 2259 while (api[idx] != ixgbe_mbox_api_unknown) { 2260 err = hw->mac.ops.negotiate_api_version(hw, api[idx]); 2261 if (!err) 2262 break; 2263 idx++; 2264 } 2265 2266 spin_unlock_bh(&adapter->mbx_lock); 2267 } 2268 2269 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 2270 { 2271 struct net_device *netdev = adapter->netdev; 2272 struct ixgbe_hw *hw = &adapter->hw; 2273 2274 ixgbevf_configure_msix(adapter); 2275 2276 spin_lock_bh(&adapter->mbx_lock); 2277 2278 if (is_valid_ether_addr(hw->mac.addr)) 2279 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 2280 else 2281 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 2282 2283 spin_unlock_bh(&adapter->mbx_lock); 2284 2285 smp_mb__before_atomic(); 2286 clear_bit(__IXGBEVF_DOWN, &adapter->state); 2287 ixgbevf_napi_enable_all(adapter); 2288 2289 /* clear any pending interrupts, may auto mask */ 2290 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2291 ixgbevf_irq_enable(adapter); 2292 2293 /* enable transmits */ 2294 netif_tx_start_all_queues(netdev); 2295 2296 ixgbevf_save_reset_stats(adapter); 2297 ixgbevf_init_last_counter_stats(adapter); 2298 2299 hw->mac.get_link_status = 1; 2300 mod_timer(&adapter->service_timer, jiffies); 2301 } 2302 2303 void ixgbevf_up(struct ixgbevf_adapter *adapter) 2304 { 2305 ixgbevf_configure(adapter); 2306 2307 ixgbevf_up_complete(adapter); 2308 } 2309 2310 /** 2311 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 2312 * @rx_ring: ring to free buffers from 2313 **/ 2314 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) 2315 { 2316 u16 i = rx_ring->next_to_clean; 2317 2318 /* Free Rx ring sk_buff */ 2319 if (rx_ring->skb) { 2320 dev_kfree_skb(rx_ring->skb); 2321 rx_ring->skb = NULL; 2322 } 2323 2324 /* Free all the Rx ring pages */ 2325 while (i != rx_ring->next_to_alloc) { 2326 struct ixgbevf_rx_buffer *rx_buffer; 2327 2328 rx_buffer = &rx_ring->rx_buffer_info[i]; 2329 2330 /* Invalidate cache lines that may have been written to by 2331 * device so that we avoid corrupting memory. 2332 */ 2333 dma_sync_single_range_for_cpu(rx_ring->dev, 2334 rx_buffer->dma, 2335 rx_buffer->page_offset, 2336 ixgbevf_rx_bufsz(rx_ring), 2337 DMA_FROM_DEVICE); 2338 2339 /* free resources associated with mapping */ 2340 dma_unmap_page_attrs(rx_ring->dev, 2341 rx_buffer->dma, 2342 ixgbevf_rx_pg_size(rx_ring), 2343 DMA_FROM_DEVICE, 2344 IXGBEVF_RX_DMA_ATTR); 2345 2346 __page_frag_cache_drain(rx_buffer->page, 2347 rx_buffer->pagecnt_bias); 2348 2349 i++; 2350 if (i == rx_ring->count) 2351 i = 0; 2352 } 2353 2354 rx_ring->next_to_alloc = 0; 2355 rx_ring->next_to_clean = 0; 2356 rx_ring->next_to_use = 0; 2357 } 2358 2359 /** 2360 * ixgbevf_clean_tx_ring - Free Tx Buffers 2361 * @tx_ring: ring to be cleaned 2362 **/ 2363 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) 2364 { 2365 u16 i = tx_ring->next_to_clean; 2366 struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 2367 2368 while (i != tx_ring->next_to_use) { 2369 union ixgbe_adv_tx_desc *eop_desc, *tx_desc; 2370 2371 /* Free all the Tx ring sk_buffs */ 2372 if (ring_is_xdp(tx_ring)) 2373 page_frag_free(tx_buffer->data); 2374 else 2375 dev_kfree_skb_any(tx_buffer->skb); 2376 2377 /* unmap skb header data */ 2378 dma_unmap_single(tx_ring->dev, 2379 dma_unmap_addr(tx_buffer, dma), 2380 dma_unmap_len(tx_buffer, len), 2381 DMA_TO_DEVICE); 2382 2383 /* check for eop_desc to determine the end of the packet */ 2384 eop_desc = tx_buffer->next_to_watch; 2385 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 2386 2387 /* unmap remaining buffers */ 2388 while (tx_desc != eop_desc) { 2389 tx_buffer++; 2390 tx_desc++; 2391 i++; 2392 if (unlikely(i == tx_ring->count)) { 2393 i = 0; 2394 tx_buffer = tx_ring->tx_buffer_info; 2395 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 2396 } 2397 2398 /* unmap any remaining paged data */ 2399 if (dma_unmap_len(tx_buffer, len)) 2400 dma_unmap_page(tx_ring->dev, 2401 dma_unmap_addr(tx_buffer, dma), 2402 dma_unmap_len(tx_buffer, len), 2403 DMA_TO_DEVICE); 2404 } 2405 2406 /* move us one more past the eop_desc for start of next pkt */ 2407 tx_buffer++; 2408 i++; 2409 if (unlikely(i == tx_ring->count)) { 2410 i = 0; 2411 tx_buffer = tx_ring->tx_buffer_info; 2412 } 2413 } 2414 2415 /* reset next_to_use and next_to_clean */ 2416 tx_ring->next_to_use = 0; 2417 tx_ring->next_to_clean = 0; 2418 2419 } 2420 2421 /** 2422 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 2423 * @adapter: board private structure 2424 **/ 2425 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 2426 { 2427 int i; 2428 2429 for (i = 0; i < adapter->num_rx_queues; i++) 2430 ixgbevf_clean_rx_ring(adapter->rx_ring[i]); 2431 } 2432 2433 /** 2434 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 2435 * @adapter: board private structure 2436 **/ 2437 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 2438 { 2439 int i; 2440 2441 for (i = 0; i < adapter->num_tx_queues; i++) 2442 ixgbevf_clean_tx_ring(adapter->tx_ring[i]); 2443 for (i = 0; i < adapter->num_xdp_queues; i++) 2444 ixgbevf_clean_tx_ring(adapter->xdp_ring[i]); 2445 } 2446 2447 void ixgbevf_down(struct ixgbevf_adapter *adapter) 2448 { 2449 struct net_device *netdev = adapter->netdev; 2450 struct ixgbe_hw *hw = &adapter->hw; 2451 int i; 2452 2453 /* signal that we are down to the interrupt handler */ 2454 if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state)) 2455 return; /* do nothing if already down */ 2456 2457 /* disable all enabled Rx queues */ 2458 for (i = 0; i < adapter->num_rx_queues; i++) 2459 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]); 2460 2461 usleep_range(10000, 20000); 2462 2463 netif_tx_stop_all_queues(netdev); 2464 2465 /* call carrier off first to avoid false dev_watchdog timeouts */ 2466 netif_carrier_off(netdev); 2467 netif_tx_disable(netdev); 2468 2469 ixgbevf_irq_disable(adapter); 2470 2471 ixgbevf_napi_disable_all(adapter); 2472 2473 del_timer_sync(&adapter->service_timer); 2474 2475 /* disable transmits in the hardware now that interrupts are off */ 2476 for (i = 0; i < adapter->num_tx_queues; i++) { 2477 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 2478 2479 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), 2480 IXGBE_TXDCTL_SWFLSH); 2481 } 2482 2483 for (i = 0; i < adapter->num_xdp_queues; i++) { 2484 u8 reg_idx = adapter->xdp_ring[i]->reg_idx; 2485 2486 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), 2487 IXGBE_TXDCTL_SWFLSH); 2488 } 2489 2490 if (!pci_channel_offline(adapter->pdev)) 2491 ixgbevf_reset(adapter); 2492 2493 ixgbevf_clean_all_tx_rings(adapter); 2494 ixgbevf_clean_all_rx_rings(adapter); 2495 } 2496 2497 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 2498 { 2499 WARN_ON(in_interrupt()); 2500 2501 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 2502 msleep(1); 2503 2504 ixgbevf_down(adapter); 2505 ixgbevf_up(adapter); 2506 2507 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 2508 } 2509 2510 void ixgbevf_reset(struct ixgbevf_adapter *adapter) 2511 { 2512 struct ixgbe_hw *hw = &adapter->hw; 2513 struct net_device *netdev = adapter->netdev; 2514 2515 if (hw->mac.ops.reset_hw(hw)) { 2516 hw_dbg(hw, "PF still resetting\n"); 2517 } else { 2518 hw->mac.ops.init_hw(hw); 2519 ixgbevf_negotiate_api(adapter); 2520 } 2521 2522 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 2523 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr); 2524 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr); 2525 } 2526 2527 adapter->last_reset = jiffies; 2528 } 2529 2530 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 2531 int vectors) 2532 { 2533 int vector_threshold; 2534 2535 /* We'll want at least 2 (vector_threshold): 2536 * 1) TxQ[0] + RxQ[0] handler 2537 * 2) Other (Link Status Change, etc.) 2538 */ 2539 vector_threshold = MIN_MSIX_COUNT; 2540 2541 /* The more we get, the more we will assign to Tx/Rx Cleanup 2542 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 2543 * Right now, we simply care about how many we'll get; we'll 2544 * set them up later while requesting irq's. 2545 */ 2546 vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 2547 vector_threshold, vectors); 2548 2549 if (vectors < 0) { 2550 dev_err(&adapter->pdev->dev, 2551 "Unable to allocate MSI-X interrupts\n"); 2552 kfree(adapter->msix_entries); 2553 adapter->msix_entries = NULL; 2554 return vectors; 2555 } 2556 2557 /* Adjust for only the vectors we'll use, which is minimum 2558 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 2559 * vectors we were allocated. 2560 */ 2561 adapter->num_msix_vectors = vectors; 2562 2563 return 0; 2564 } 2565 2566 /** 2567 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent 2568 * @adapter: board private structure to initialize 2569 * 2570 * This is the top level queue allocation routine. The order here is very 2571 * important, starting with the "most" number of features turned on at once, 2572 * and ending with the smallest set of features. This way large combinations 2573 * can be allocated if they're turned on, and smaller combinations are the 2574 * fallthrough conditions. 2575 * 2576 **/ 2577 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 2578 { 2579 struct ixgbe_hw *hw = &adapter->hw; 2580 unsigned int def_q = 0; 2581 unsigned int num_tcs = 0; 2582 int err; 2583 2584 /* Start with base case */ 2585 adapter->num_rx_queues = 1; 2586 adapter->num_tx_queues = 1; 2587 adapter->num_xdp_queues = 0; 2588 2589 spin_lock_bh(&adapter->mbx_lock); 2590 2591 /* fetch queue configuration from the PF */ 2592 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2593 2594 spin_unlock_bh(&adapter->mbx_lock); 2595 2596 if (err) 2597 return; 2598 2599 /* we need as many queues as traffic classes */ 2600 if (num_tcs > 1) { 2601 adapter->num_rx_queues = num_tcs; 2602 } else { 2603 u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES); 2604 2605 switch (hw->api_version) { 2606 case ixgbe_mbox_api_11: 2607 case ixgbe_mbox_api_12: 2608 case ixgbe_mbox_api_13: 2609 if (adapter->xdp_prog && 2610 hw->mac.max_tx_queues == rss) 2611 rss = rss > 3 ? 2 : 1; 2612 2613 adapter->num_rx_queues = rss; 2614 adapter->num_tx_queues = rss; 2615 adapter->num_xdp_queues = adapter->xdp_prog ? rss : 0; 2616 default: 2617 break; 2618 } 2619 } 2620 } 2621 2622 /** 2623 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 2624 * @adapter: board private structure to initialize 2625 * 2626 * Attempt to configure the interrupts using the best available 2627 * capabilities of the hardware and the kernel. 2628 **/ 2629 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 2630 { 2631 int vector, v_budget; 2632 2633 /* It's easy to be greedy for MSI-X vectors, but it really 2634 * doesn't do us much good if we have a lot more vectors 2635 * than CPU's. So let's be conservative and only ask for 2636 * (roughly) the same number of vectors as there are CPU's. 2637 * The default is to use pairs of vectors. 2638 */ 2639 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); 2640 v_budget = min_t(int, v_budget, num_online_cpus()); 2641 v_budget += NON_Q_VECTORS; 2642 2643 adapter->msix_entries = kcalloc(v_budget, 2644 sizeof(struct msix_entry), GFP_KERNEL); 2645 if (!adapter->msix_entries) 2646 return -ENOMEM; 2647 2648 for (vector = 0; vector < v_budget; vector++) 2649 adapter->msix_entries[vector].entry = vector; 2650 2651 /* A failure in MSI-X entry allocation isn't fatal, but the VF driver 2652 * does not support any other modes, so we will simply fail here. Note 2653 * that we clean up the msix_entries pointer else-where. 2654 */ 2655 return ixgbevf_acquire_msix_vectors(adapter, v_budget); 2656 } 2657 2658 static void ixgbevf_add_ring(struct ixgbevf_ring *ring, 2659 struct ixgbevf_ring_container *head) 2660 { 2661 ring->next = head->ring; 2662 head->ring = ring; 2663 head->count++; 2664 } 2665 2666 /** 2667 * ixgbevf_alloc_q_vector - Allocate memory for a single interrupt vector 2668 * @adapter: board private structure to initialize 2669 * @v_idx: index of vector in adapter struct 2670 * @txr_count: number of Tx rings for q vector 2671 * @txr_idx: index of first Tx ring to assign 2672 * @xdp_count: total number of XDP rings to allocate 2673 * @xdp_idx: index of first XDP ring to allocate 2674 * @rxr_count: number of Rx rings for q vector 2675 * @rxr_idx: index of first Rx ring to assign 2676 * 2677 * We allocate one q_vector. If allocation fails we return -ENOMEM. 2678 **/ 2679 static int ixgbevf_alloc_q_vector(struct ixgbevf_adapter *adapter, int v_idx, 2680 int txr_count, int txr_idx, 2681 int xdp_count, int xdp_idx, 2682 int rxr_count, int rxr_idx) 2683 { 2684 struct ixgbevf_q_vector *q_vector; 2685 int reg_idx = txr_idx + xdp_idx; 2686 struct ixgbevf_ring *ring; 2687 int ring_count, size; 2688 2689 ring_count = txr_count + xdp_count + rxr_count; 2690 size = sizeof(*q_vector) + (sizeof(*ring) * ring_count); 2691 2692 /* allocate q_vector and rings */ 2693 q_vector = kzalloc(size, GFP_KERNEL); 2694 if (!q_vector) 2695 return -ENOMEM; 2696 2697 /* initialize NAPI */ 2698 netif_napi_add(adapter->netdev, &q_vector->napi, ixgbevf_poll, 64); 2699 2700 /* tie q_vector and adapter together */ 2701 adapter->q_vector[v_idx] = q_vector; 2702 q_vector->adapter = adapter; 2703 q_vector->v_idx = v_idx; 2704 2705 /* initialize pointer to rings */ 2706 ring = q_vector->ring; 2707 2708 while (txr_count) { 2709 /* assign generic ring traits */ 2710 ring->dev = &adapter->pdev->dev; 2711 ring->netdev = adapter->netdev; 2712 2713 /* configure backlink on ring */ 2714 ring->q_vector = q_vector; 2715 2716 /* update q_vector Tx values */ 2717 ixgbevf_add_ring(ring, &q_vector->tx); 2718 2719 /* apply Tx specific ring traits */ 2720 ring->count = adapter->tx_ring_count; 2721 ring->queue_index = txr_idx; 2722 ring->reg_idx = reg_idx; 2723 2724 /* assign ring to adapter */ 2725 adapter->tx_ring[txr_idx] = ring; 2726 2727 /* update count and index */ 2728 txr_count--; 2729 txr_idx++; 2730 reg_idx++; 2731 2732 /* push pointer to next ring */ 2733 ring++; 2734 } 2735 2736 while (xdp_count) { 2737 /* assign generic ring traits */ 2738 ring->dev = &adapter->pdev->dev; 2739 ring->netdev = adapter->netdev; 2740 2741 /* configure backlink on ring */ 2742 ring->q_vector = q_vector; 2743 2744 /* update q_vector Tx values */ 2745 ixgbevf_add_ring(ring, &q_vector->tx); 2746 2747 /* apply Tx specific ring traits */ 2748 ring->count = adapter->tx_ring_count; 2749 ring->queue_index = xdp_idx; 2750 ring->reg_idx = reg_idx; 2751 set_ring_xdp(ring); 2752 2753 /* assign ring to adapter */ 2754 adapter->xdp_ring[xdp_idx] = ring; 2755 2756 /* update count and index */ 2757 xdp_count--; 2758 xdp_idx++; 2759 reg_idx++; 2760 2761 /* push pointer to next ring */ 2762 ring++; 2763 } 2764 2765 while (rxr_count) { 2766 /* assign generic ring traits */ 2767 ring->dev = &adapter->pdev->dev; 2768 ring->netdev = adapter->netdev; 2769 2770 /* configure backlink on ring */ 2771 ring->q_vector = q_vector; 2772 2773 /* update q_vector Rx values */ 2774 ixgbevf_add_ring(ring, &q_vector->rx); 2775 2776 /* apply Rx specific ring traits */ 2777 ring->count = adapter->rx_ring_count; 2778 ring->queue_index = rxr_idx; 2779 ring->reg_idx = rxr_idx; 2780 2781 /* assign ring to adapter */ 2782 adapter->rx_ring[rxr_idx] = ring; 2783 2784 /* update count and index */ 2785 rxr_count--; 2786 rxr_idx++; 2787 2788 /* push pointer to next ring */ 2789 ring++; 2790 } 2791 2792 return 0; 2793 } 2794 2795 /** 2796 * ixgbevf_free_q_vector - Free memory allocated for specific interrupt vector 2797 * @adapter: board private structure to initialize 2798 * @v_idx: index of vector in adapter struct 2799 * 2800 * This function frees the memory allocated to the q_vector. In addition if 2801 * NAPI is enabled it will delete any references to the NAPI struct prior 2802 * to freeing the q_vector. 2803 **/ 2804 static void ixgbevf_free_q_vector(struct ixgbevf_adapter *adapter, int v_idx) 2805 { 2806 struct ixgbevf_q_vector *q_vector = adapter->q_vector[v_idx]; 2807 struct ixgbevf_ring *ring; 2808 2809 ixgbevf_for_each_ring(ring, q_vector->tx) { 2810 if (ring_is_xdp(ring)) 2811 adapter->xdp_ring[ring->queue_index] = NULL; 2812 else 2813 adapter->tx_ring[ring->queue_index] = NULL; 2814 } 2815 2816 ixgbevf_for_each_ring(ring, q_vector->rx) 2817 adapter->rx_ring[ring->queue_index] = NULL; 2818 2819 adapter->q_vector[v_idx] = NULL; 2820 netif_napi_del(&q_vector->napi); 2821 2822 /* ixgbevf_get_stats() might access the rings on this vector, 2823 * we must wait a grace period before freeing it. 2824 */ 2825 kfree_rcu(q_vector, rcu); 2826 } 2827 2828 /** 2829 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 2830 * @adapter: board private structure to initialize 2831 * 2832 * We allocate one q_vector per queue interrupt. If allocation fails we 2833 * return -ENOMEM. 2834 **/ 2835 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 2836 { 2837 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2838 int rxr_remaining = adapter->num_rx_queues; 2839 int txr_remaining = adapter->num_tx_queues; 2840 int xdp_remaining = adapter->num_xdp_queues; 2841 int rxr_idx = 0, txr_idx = 0, xdp_idx = 0, v_idx = 0; 2842 int err; 2843 2844 if (q_vectors >= (rxr_remaining + txr_remaining + xdp_remaining)) { 2845 for (; rxr_remaining; v_idx++, q_vectors--) { 2846 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors); 2847 2848 err = ixgbevf_alloc_q_vector(adapter, v_idx, 2849 0, 0, 0, 0, rqpv, rxr_idx); 2850 if (err) 2851 goto err_out; 2852 2853 /* update counts and index */ 2854 rxr_remaining -= rqpv; 2855 rxr_idx += rqpv; 2856 } 2857 } 2858 2859 for (; q_vectors; v_idx++, q_vectors--) { 2860 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors); 2861 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors); 2862 int xqpv = DIV_ROUND_UP(xdp_remaining, q_vectors); 2863 2864 err = ixgbevf_alloc_q_vector(adapter, v_idx, 2865 tqpv, txr_idx, 2866 xqpv, xdp_idx, 2867 rqpv, rxr_idx); 2868 2869 if (err) 2870 goto err_out; 2871 2872 /* update counts and index */ 2873 rxr_remaining -= rqpv; 2874 rxr_idx += rqpv; 2875 txr_remaining -= tqpv; 2876 txr_idx += tqpv; 2877 xdp_remaining -= xqpv; 2878 xdp_idx += xqpv; 2879 } 2880 2881 return 0; 2882 2883 err_out: 2884 while (v_idx) { 2885 v_idx--; 2886 ixgbevf_free_q_vector(adapter, v_idx); 2887 } 2888 2889 return -ENOMEM; 2890 } 2891 2892 /** 2893 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 2894 * @adapter: board private structure to initialize 2895 * 2896 * This function frees the memory allocated to the q_vectors. In addition if 2897 * NAPI is enabled it will delete any references to the NAPI struct prior 2898 * to freeing the q_vector. 2899 **/ 2900 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 2901 { 2902 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2903 2904 while (q_vectors) { 2905 q_vectors--; 2906 ixgbevf_free_q_vector(adapter, q_vectors); 2907 } 2908 } 2909 2910 /** 2911 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 2912 * @adapter: board private structure 2913 * 2914 **/ 2915 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 2916 { 2917 if (!adapter->msix_entries) 2918 return; 2919 2920 pci_disable_msix(adapter->pdev); 2921 kfree(adapter->msix_entries); 2922 adapter->msix_entries = NULL; 2923 } 2924 2925 /** 2926 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 2927 * @adapter: board private structure to initialize 2928 * 2929 **/ 2930 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 2931 { 2932 int err; 2933 2934 /* Number of supported queues */ 2935 ixgbevf_set_num_queues(adapter); 2936 2937 err = ixgbevf_set_interrupt_capability(adapter); 2938 if (err) { 2939 hw_dbg(&adapter->hw, 2940 "Unable to setup interrupt capabilities\n"); 2941 goto err_set_interrupt; 2942 } 2943 2944 err = ixgbevf_alloc_q_vectors(adapter); 2945 if (err) { 2946 hw_dbg(&adapter->hw, "Unable to allocate memory for queue vectors\n"); 2947 goto err_alloc_q_vectors; 2948 } 2949 2950 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u XDP Queue count %u\n", 2951 (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled", 2952 adapter->num_rx_queues, adapter->num_tx_queues, 2953 adapter->num_xdp_queues); 2954 2955 set_bit(__IXGBEVF_DOWN, &adapter->state); 2956 2957 return 0; 2958 err_alloc_q_vectors: 2959 ixgbevf_reset_interrupt_capability(adapter); 2960 err_set_interrupt: 2961 return err; 2962 } 2963 2964 /** 2965 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings 2966 * @adapter: board private structure to clear interrupt scheme on 2967 * 2968 * We go through and clear interrupt specific resources and reset the structure 2969 * to pre-load conditions 2970 **/ 2971 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) 2972 { 2973 adapter->num_tx_queues = 0; 2974 adapter->num_xdp_queues = 0; 2975 adapter->num_rx_queues = 0; 2976 2977 ixgbevf_free_q_vectors(adapter); 2978 ixgbevf_reset_interrupt_capability(adapter); 2979 } 2980 2981 /** 2982 * ixgbevf_sw_init - Initialize general software structures 2983 * @adapter: board private structure to initialize 2984 * 2985 * ixgbevf_sw_init initializes the Adapter private data structure. 2986 * Fields are initialized based on PCI device information and 2987 * OS network device settings (MTU size). 2988 **/ 2989 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 2990 { 2991 struct ixgbe_hw *hw = &adapter->hw; 2992 struct pci_dev *pdev = adapter->pdev; 2993 struct net_device *netdev = adapter->netdev; 2994 int err; 2995 2996 /* PCI config space info */ 2997 hw->vendor_id = pdev->vendor; 2998 hw->device_id = pdev->device; 2999 hw->revision_id = pdev->revision; 3000 hw->subsystem_vendor_id = pdev->subsystem_vendor; 3001 hw->subsystem_device_id = pdev->subsystem_device; 3002 3003 hw->mbx.ops.init_params(hw); 3004 3005 if (hw->mac.type >= ixgbe_mac_X550_vf) { 3006 err = ixgbevf_init_rss_key(adapter); 3007 if (err) 3008 goto out; 3009 } 3010 3011 /* assume legacy case in which PF would only give VF 2 queues */ 3012 hw->mac.max_tx_queues = 2; 3013 hw->mac.max_rx_queues = 2; 3014 3015 /* lock to protect mailbox accesses */ 3016 spin_lock_init(&adapter->mbx_lock); 3017 3018 err = hw->mac.ops.reset_hw(hw); 3019 if (err) { 3020 dev_info(&pdev->dev, 3021 "PF still in reset state. Is the PF interface up?\n"); 3022 } else { 3023 err = hw->mac.ops.init_hw(hw); 3024 if (err) { 3025 pr_err("init_shared_code failed: %d\n", err); 3026 goto out; 3027 } 3028 ixgbevf_negotiate_api(adapter); 3029 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr); 3030 if (err) 3031 dev_info(&pdev->dev, "Error reading MAC address\n"); 3032 else if (is_zero_ether_addr(adapter->hw.mac.addr)) 3033 dev_info(&pdev->dev, 3034 "MAC address not assigned by administrator.\n"); 3035 ether_addr_copy(netdev->dev_addr, hw->mac.addr); 3036 } 3037 3038 if (!is_valid_ether_addr(netdev->dev_addr)) { 3039 dev_info(&pdev->dev, "Assigning random MAC address\n"); 3040 eth_hw_addr_random(netdev); 3041 ether_addr_copy(hw->mac.addr, netdev->dev_addr); 3042 ether_addr_copy(hw->mac.perm_addr, netdev->dev_addr); 3043 } 3044 3045 /* Enable dynamic interrupt throttling rates */ 3046 adapter->rx_itr_setting = 1; 3047 adapter->tx_itr_setting = 1; 3048 3049 /* set default ring sizes */ 3050 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 3051 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 3052 3053 set_bit(__IXGBEVF_DOWN, &adapter->state); 3054 return 0; 3055 3056 out: 3057 return err; 3058 } 3059 3060 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 3061 { \ 3062 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 3063 if (current_counter < last_counter) \ 3064 counter += 0x100000000LL; \ 3065 last_counter = current_counter; \ 3066 counter &= 0xFFFFFFFF00000000LL; \ 3067 counter |= current_counter; \ 3068 } 3069 3070 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 3071 { \ 3072 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 3073 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 3074 u64 current_counter = (current_counter_msb << 32) | \ 3075 current_counter_lsb; \ 3076 if (current_counter < last_counter) \ 3077 counter += 0x1000000000LL; \ 3078 last_counter = current_counter; \ 3079 counter &= 0xFFFFFFF000000000LL; \ 3080 counter |= current_counter; \ 3081 } 3082 /** 3083 * ixgbevf_update_stats - Update the board statistics counters. 3084 * @adapter: board private structure 3085 **/ 3086 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 3087 { 3088 struct ixgbe_hw *hw = &adapter->hw; 3089 u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0; 3090 u64 alloc_rx_page = 0, hw_csum_rx_error = 0; 3091 int i; 3092 3093 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3094 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3095 return; 3096 3097 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 3098 adapter->stats.vfgprc); 3099 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 3100 adapter->stats.vfgptc); 3101 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3102 adapter->stats.last_vfgorc, 3103 adapter->stats.vfgorc); 3104 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3105 adapter->stats.last_vfgotc, 3106 adapter->stats.vfgotc); 3107 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 3108 adapter->stats.vfmprc); 3109 3110 for (i = 0; i < adapter->num_rx_queues; i++) { 3111 struct ixgbevf_ring *rx_ring = adapter->rx_ring[i]; 3112 3113 hw_csum_rx_error += rx_ring->rx_stats.csum_err; 3114 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed; 3115 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed; 3116 alloc_rx_page += rx_ring->rx_stats.alloc_rx_page; 3117 } 3118 3119 adapter->hw_csum_rx_error = hw_csum_rx_error; 3120 adapter->alloc_rx_page_failed = alloc_rx_page_failed; 3121 adapter->alloc_rx_buff_failed = alloc_rx_buff_failed; 3122 adapter->alloc_rx_page = alloc_rx_page; 3123 } 3124 3125 /** 3126 * ixgbevf_service_timer - Timer Call-back 3127 * @t: pointer to timer_list struct 3128 **/ 3129 static void ixgbevf_service_timer(struct timer_list *t) 3130 { 3131 struct ixgbevf_adapter *adapter = from_timer(adapter, t, 3132 service_timer); 3133 3134 /* Reset the timer */ 3135 mod_timer(&adapter->service_timer, (HZ * 2) + jiffies); 3136 3137 ixgbevf_service_event_schedule(adapter); 3138 } 3139 3140 static void ixgbevf_reset_subtask(struct ixgbevf_adapter *adapter) 3141 { 3142 if (!test_and_clear_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state)) 3143 return; 3144 3145 /* If we're already down or resetting, just bail */ 3146 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3147 test_bit(__IXGBEVF_REMOVING, &adapter->state) || 3148 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3149 return; 3150 3151 adapter->tx_timeout_count++; 3152 3153 rtnl_lock(); 3154 ixgbevf_reinit_locked(adapter); 3155 rtnl_unlock(); 3156 } 3157 3158 /** 3159 * ixgbevf_check_hang_subtask - check for hung queues and dropped interrupts 3160 * @adapter: pointer to the device adapter structure 3161 * 3162 * This function serves two purposes. First it strobes the interrupt lines 3163 * in order to make certain interrupts are occurring. Secondly it sets the 3164 * bits needed to check for TX hangs. As a result we should immediately 3165 * determine if a hang has occurred. 3166 **/ 3167 static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter) 3168 { 3169 struct ixgbe_hw *hw = &adapter->hw; 3170 u32 eics = 0; 3171 int i; 3172 3173 /* If we're down or resetting, just bail */ 3174 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3175 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3176 return; 3177 3178 /* Force detection of hung controller */ 3179 if (netif_carrier_ok(adapter->netdev)) { 3180 for (i = 0; i < adapter->num_tx_queues; i++) 3181 set_check_for_tx_hang(adapter->tx_ring[i]); 3182 for (i = 0; i < adapter->num_xdp_queues; i++) 3183 set_check_for_tx_hang(adapter->xdp_ring[i]); 3184 } 3185 3186 /* get one bit for every active Tx/Rx interrupt vector */ 3187 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 3188 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 3189 3190 if (qv->rx.ring || qv->tx.ring) 3191 eics |= BIT(i); 3192 } 3193 3194 /* Cause software interrupt to ensure rings are cleaned */ 3195 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics); 3196 } 3197 3198 /** 3199 * ixgbevf_watchdog_update_link - update the link status 3200 * @adapter: pointer to the device adapter structure 3201 **/ 3202 static void ixgbevf_watchdog_update_link(struct ixgbevf_adapter *adapter) 3203 { 3204 struct ixgbe_hw *hw = &adapter->hw; 3205 u32 link_speed = adapter->link_speed; 3206 bool link_up = adapter->link_up; 3207 s32 err; 3208 3209 spin_lock_bh(&adapter->mbx_lock); 3210 3211 err = hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 3212 3213 spin_unlock_bh(&adapter->mbx_lock); 3214 3215 /* if check for link returns error we will need to reset */ 3216 if (err && time_after(jiffies, adapter->last_reset + (10 * HZ))) { 3217 set_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state); 3218 link_up = false; 3219 } 3220 3221 adapter->link_up = link_up; 3222 adapter->link_speed = link_speed; 3223 } 3224 3225 /** 3226 * ixgbevf_watchdog_link_is_up - update netif_carrier status and 3227 * print link up message 3228 * @adapter: pointer to the device adapter structure 3229 **/ 3230 static void ixgbevf_watchdog_link_is_up(struct ixgbevf_adapter *adapter) 3231 { 3232 struct net_device *netdev = adapter->netdev; 3233 3234 /* only continue if link was previously down */ 3235 if (netif_carrier_ok(netdev)) 3236 return; 3237 3238 dev_info(&adapter->pdev->dev, "NIC Link is Up %s\n", 3239 (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 3240 "10 Gbps" : 3241 (adapter->link_speed == IXGBE_LINK_SPEED_1GB_FULL) ? 3242 "1 Gbps" : 3243 (adapter->link_speed == IXGBE_LINK_SPEED_100_FULL) ? 3244 "100 Mbps" : 3245 "unknown speed"); 3246 3247 netif_carrier_on(netdev); 3248 } 3249 3250 /** 3251 * ixgbevf_watchdog_link_is_down - update netif_carrier status and 3252 * print link down message 3253 * @adapter: pointer to the adapter structure 3254 **/ 3255 static void ixgbevf_watchdog_link_is_down(struct ixgbevf_adapter *adapter) 3256 { 3257 struct net_device *netdev = adapter->netdev; 3258 3259 adapter->link_speed = 0; 3260 3261 /* only continue if link was up previously */ 3262 if (!netif_carrier_ok(netdev)) 3263 return; 3264 3265 dev_info(&adapter->pdev->dev, "NIC Link is Down\n"); 3266 3267 netif_carrier_off(netdev); 3268 } 3269 3270 /** 3271 * ixgbevf_watchdog_subtask - worker thread to bring link up 3272 * @adapter: board private structure 3273 **/ 3274 static void ixgbevf_watchdog_subtask(struct ixgbevf_adapter *adapter) 3275 { 3276 /* if interface is down do nothing */ 3277 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3278 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3279 return; 3280 3281 ixgbevf_watchdog_update_link(adapter); 3282 3283 if (adapter->link_up) 3284 ixgbevf_watchdog_link_is_up(adapter); 3285 else 3286 ixgbevf_watchdog_link_is_down(adapter); 3287 3288 ixgbevf_update_stats(adapter); 3289 } 3290 3291 /** 3292 * ixgbevf_service_task - manages and runs subtasks 3293 * @work: pointer to work_struct containing our data 3294 **/ 3295 static void ixgbevf_service_task(struct work_struct *work) 3296 { 3297 struct ixgbevf_adapter *adapter = container_of(work, 3298 struct ixgbevf_adapter, 3299 service_task); 3300 struct ixgbe_hw *hw = &adapter->hw; 3301 3302 if (IXGBE_REMOVED(hw->hw_addr)) { 3303 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 3304 rtnl_lock(); 3305 ixgbevf_down(adapter); 3306 rtnl_unlock(); 3307 } 3308 return; 3309 } 3310 3311 ixgbevf_queue_reset_subtask(adapter); 3312 ixgbevf_reset_subtask(adapter); 3313 ixgbevf_watchdog_subtask(adapter); 3314 ixgbevf_check_hang_subtask(adapter); 3315 3316 ixgbevf_service_event_complete(adapter); 3317 } 3318 3319 /** 3320 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 3321 * @tx_ring: Tx descriptor ring for a specific queue 3322 * 3323 * Free all transmit software resources 3324 **/ 3325 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) 3326 { 3327 ixgbevf_clean_tx_ring(tx_ring); 3328 3329 vfree(tx_ring->tx_buffer_info); 3330 tx_ring->tx_buffer_info = NULL; 3331 3332 /* if not set, then don't free */ 3333 if (!tx_ring->desc) 3334 return; 3335 3336 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, 3337 tx_ring->dma); 3338 3339 tx_ring->desc = NULL; 3340 } 3341 3342 /** 3343 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 3344 * @adapter: board private structure 3345 * 3346 * Free all transmit software resources 3347 **/ 3348 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 3349 { 3350 int i; 3351 3352 for (i = 0; i < adapter->num_tx_queues; i++) 3353 if (adapter->tx_ring[i]->desc) 3354 ixgbevf_free_tx_resources(adapter->tx_ring[i]); 3355 for (i = 0; i < adapter->num_xdp_queues; i++) 3356 if (adapter->xdp_ring[i]->desc) 3357 ixgbevf_free_tx_resources(adapter->xdp_ring[i]); 3358 } 3359 3360 /** 3361 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 3362 * @tx_ring: Tx descriptor ring (for a specific queue) to setup 3363 * 3364 * Return 0 on success, negative on failure 3365 **/ 3366 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) 3367 { 3368 struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); 3369 int size; 3370 3371 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 3372 tx_ring->tx_buffer_info = vmalloc(size); 3373 if (!tx_ring->tx_buffer_info) 3374 goto err; 3375 3376 u64_stats_init(&tx_ring->syncp); 3377 3378 /* round up to nearest 4K */ 3379 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 3380 tx_ring->size = ALIGN(tx_ring->size, 4096); 3381 3382 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, 3383 &tx_ring->dma, GFP_KERNEL); 3384 if (!tx_ring->desc) 3385 goto err; 3386 3387 return 0; 3388 3389 err: 3390 vfree(tx_ring->tx_buffer_info); 3391 tx_ring->tx_buffer_info = NULL; 3392 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit descriptor ring\n"); 3393 return -ENOMEM; 3394 } 3395 3396 /** 3397 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 3398 * @adapter: board private structure 3399 * 3400 * If this function returns with an error, then it's possible one or 3401 * more of the rings is populated (while the rest are not). It is the 3402 * callers duty to clean those orphaned rings. 3403 * 3404 * Return 0 on success, negative on failure 3405 **/ 3406 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 3407 { 3408 int i, j = 0, err = 0; 3409 3410 for (i = 0; i < adapter->num_tx_queues; i++) { 3411 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]); 3412 if (!err) 3413 continue; 3414 hw_dbg(&adapter->hw, "Allocation for Tx Queue %u failed\n", i); 3415 goto err_setup_tx; 3416 } 3417 3418 for (j = 0; j < adapter->num_xdp_queues; j++) { 3419 err = ixgbevf_setup_tx_resources(adapter->xdp_ring[j]); 3420 if (!err) 3421 continue; 3422 hw_dbg(&adapter->hw, "Allocation for XDP Queue %u failed\n", j); 3423 goto err_setup_tx; 3424 } 3425 3426 return 0; 3427 err_setup_tx: 3428 /* rewind the index freeing the rings as we go */ 3429 while (j--) 3430 ixgbevf_free_tx_resources(adapter->xdp_ring[j]); 3431 while (i--) 3432 ixgbevf_free_tx_resources(adapter->tx_ring[i]); 3433 3434 return err; 3435 } 3436 3437 /** 3438 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 3439 * @adapter: board private structure 3440 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 3441 * 3442 * Returns 0 on success, negative on failure 3443 **/ 3444 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, 3445 struct ixgbevf_ring *rx_ring) 3446 { 3447 int size; 3448 3449 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 3450 rx_ring->rx_buffer_info = vmalloc(size); 3451 if (!rx_ring->rx_buffer_info) 3452 goto err; 3453 3454 u64_stats_init(&rx_ring->syncp); 3455 3456 /* Round up to nearest 4K */ 3457 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 3458 rx_ring->size = ALIGN(rx_ring->size, 4096); 3459 3460 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, 3461 &rx_ring->dma, GFP_KERNEL); 3462 3463 if (!rx_ring->desc) 3464 goto err; 3465 3466 /* XDP RX-queue info */ 3467 if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, 3468 rx_ring->queue_index) < 0) 3469 goto err; 3470 3471 rx_ring->xdp_prog = adapter->xdp_prog; 3472 3473 return 0; 3474 err: 3475 vfree(rx_ring->rx_buffer_info); 3476 rx_ring->rx_buffer_info = NULL; 3477 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); 3478 return -ENOMEM; 3479 } 3480 3481 /** 3482 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 3483 * @adapter: board private structure 3484 * 3485 * If this function returns with an error, then it's possible one or 3486 * more of the rings is populated (while the rest are not). It is the 3487 * callers duty to clean those orphaned rings. 3488 * 3489 * Return 0 on success, negative on failure 3490 **/ 3491 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 3492 { 3493 int i, err = 0; 3494 3495 for (i = 0; i < adapter->num_rx_queues; i++) { 3496 err = ixgbevf_setup_rx_resources(adapter, adapter->rx_ring[i]); 3497 if (!err) 3498 continue; 3499 hw_dbg(&adapter->hw, "Allocation for Rx Queue %u failed\n", i); 3500 goto err_setup_rx; 3501 } 3502 3503 return 0; 3504 err_setup_rx: 3505 /* rewind the index freeing the rings as we go */ 3506 while (i--) 3507 ixgbevf_free_rx_resources(adapter->rx_ring[i]); 3508 return err; 3509 } 3510 3511 /** 3512 * ixgbevf_free_rx_resources - Free Rx Resources 3513 * @rx_ring: ring to clean the resources from 3514 * 3515 * Free all receive software resources 3516 **/ 3517 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) 3518 { 3519 ixgbevf_clean_rx_ring(rx_ring); 3520 3521 rx_ring->xdp_prog = NULL; 3522 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 3523 vfree(rx_ring->rx_buffer_info); 3524 rx_ring->rx_buffer_info = NULL; 3525 3526 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, 3527 rx_ring->dma); 3528 3529 rx_ring->desc = NULL; 3530 } 3531 3532 /** 3533 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 3534 * @adapter: board private structure 3535 * 3536 * Free all receive software resources 3537 **/ 3538 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 3539 { 3540 int i; 3541 3542 for (i = 0; i < adapter->num_rx_queues; i++) 3543 if (adapter->rx_ring[i]->desc) 3544 ixgbevf_free_rx_resources(adapter->rx_ring[i]); 3545 } 3546 3547 /** 3548 * ixgbevf_open - Called when a network interface is made active 3549 * @netdev: network interface device structure 3550 * 3551 * Returns 0 on success, negative value on failure 3552 * 3553 * The open entry point is called when a network interface is made 3554 * active by the system (IFF_UP). At this point all resources needed 3555 * for transmit and receive operations are allocated, the interrupt 3556 * handler is registered with the OS, the watchdog timer is started, 3557 * and the stack is notified that the interface is ready. 3558 **/ 3559 int ixgbevf_open(struct net_device *netdev) 3560 { 3561 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3562 struct ixgbe_hw *hw = &adapter->hw; 3563 int err; 3564 3565 /* A previous failure to open the device because of a lack of 3566 * available MSIX vector resources may have reset the number 3567 * of msix vectors variable to zero. The only way to recover 3568 * is to unload/reload the driver and hope that the system has 3569 * been able to recover some MSIX vector resources. 3570 */ 3571 if (!adapter->num_msix_vectors) 3572 return -ENOMEM; 3573 3574 if (hw->adapter_stopped) { 3575 ixgbevf_reset(adapter); 3576 /* if adapter is still stopped then PF isn't up and 3577 * the VF can't start. 3578 */ 3579 if (hw->adapter_stopped) { 3580 err = IXGBE_ERR_MBX; 3581 pr_err("Unable to start - perhaps the PF Driver isn't up yet\n"); 3582 goto err_setup_reset; 3583 } 3584 } 3585 3586 /* disallow open during test */ 3587 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 3588 return -EBUSY; 3589 3590 netif_carrier_off(netdev); 3591 3592 /* allocate transmit descriptors */ 3593 err = ixgbevf_setup_all_tx_resources(adapter); 3594 if (err) 3595 goto err_setup_tx; 3596 3597 /* allocate receive descriptors */ 3598 err = ixgbevf_setup_all_rx_resources(adapter); 3599 if (err) 3600 goto err_setup_rx; 3601 3602 ixgbevf_configure(adapter); 3603 3604 err = ixgbevf_request_irq(adapter); 3605 if (err) 3606 goto err_req_irq; 3607 3608 /* Notify the stack of the actual queue counts. */ 3609 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 3610 if (err) 3611 goto err_set_queues; 3612 3613 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 3614 if (err) 3615 goto err_set_queues; 3616 3617 ixgbevf_up_complete(adapter); 3618 3619 return 0; 3620 3621 err_set_queues: 3622 ixgbevf_free_irq(adapter); 3623 err_req_irq: 3624 ixgbevf_free_all_rx_resources(adapter); 3625 err_setup_rx: 3626 ixgbevf_free_all_tx_resources(adapter); 3627 err_setup_tx: 3628 ixgbevf_reset(adapter); 3629 err_setup_reset: 3630 3631 return err; 3632 } 3633 3634 /** 3635 * ixgbevf_close_suspend - actions necessary to both suspend and close flows 3636 * @adapter: the private adapter struct 3637 * 3638 * This function should contain the necessary work common to both suspending 3639 * and closing of the device. 3640 */ 3641 static void ixgbevf_close_suspend(struct ixgbevf_adapter *adapter) 3642 { 3643 ixgbevf_down(adapter); 3644 ixgbevf_free_irq(adapter); 3645 ixgbevf_free_all_tx_resources(adapter); 3646 ixgbevf_free_all_rx_resources(adapter); 3647 } 3648 3649 /** 3650 * ixgbevf_close - Disables a network interface 3651 * @netdev: network interface device structure 3652 * 3653 * Returns 0, this is not allowed to fail 3654 * 3655 * The close entry point is called when an interface is de-activated 3656 * by the OS. The hardware is still under the drivers control, but 3657 * needs to be disabled. A global MAC reset is issued to stop the 3658 * hardware, and all transmit and receive resources are freed. 3659 **/ 3660 int ixgbevf_close(struct net_device *netdev) 3661 { 3662 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3663 3664 if (netif_device_present(netdev)) 3665 ixgbevf_close_suspend(adapter); 3666 3667 return 0; 3668 } 3669 3670 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter) 3671 { 3672 struct net_device *dev = adapter->netdev; 3673 3674 if (!test_and_clear_bit(__IXGBEVF_QUEUE_RESET_REQUESTED, 3675 &adapter->state)) 3676 return; 3677 3678 /* if interface is down do nothing */ 3679 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3680 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3681 return; 3682 3683 /* Hardware has to reinitialize queues and interrupts to 3684 * match packet buffer alignment. Unfortunately, the 3685 * hardware is not flexible enough to do this dynamically. 3686 */ 3687 rtnl_lock(); 3688 3689 if (netif_running(dev)) 3690 ixgbevf_close(dev); 3691 3692 ixgbevf_clear_interrupt_scheme(adapter); 3693 ixgbevf_init_interrupt_scheme(adapter); 3694 3695 if (netif_running(dev)) 3696 ixgbevf_open(dev); 3697 3698 rtnl_unlock(); 3699 } 3700 3701 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, 3702 u32 vlan_macip_lens, u32 type_tucmd, 3703 u32 mss_l4len_idx) 3704 { 3705 struct ixgbe_adv_tx_context_desc *context_desc; 3706 u16 i = tx_ring->next_to_use; 3707 3708 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i); 3709 3710 i++; 3711 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 3712 3713 /* set bits to identify this as an advanced context descriptor */ 3714 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3715 3716 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 3717 context_desc->seqnum_seed = 0; 3718 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 3719 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 3720 } 3721 3722 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, 3723 struct ixgbevf_tx_buffer *first, 3724 u8 *hdr_len) 3725 { 3726 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 3727 struct sk_buff *skb = first->skb; 3728 union { 3729 struct iphdr *v4; 3730 struct ipv6hdr *v6; 3731 unsigned char *hdr; 3732 } ip; 3733 union { 3734 struct tcphdr *tcp; 3735 unsigned char *hdr; 3736 } l4; 3737 u32 paylen, l4_offset; 3738 int err; 3739 3740 if (skb->ip_summed != CHECKSUM_PARTIAL) 3741 return 0; 3742 3743 if (!skb_is_gso(skb)) 3744 return 0; 3745 3746 err = skb_cow_head(skb, 0); 3747 if (err < 0) 3748 return err; 3749 3750 if (eth_p_mpls(first->protocol)) 3751 ip.hdr = skb_inner_network_header(skb); 3752 else 3753 ip.hdr = skb_network_header(skb); 3754 l4.hdr = skb_checksum_start(skb); 3755 3756 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 3757 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3758 3759 /* initialize outer IP header fields */ 3760 if (ip.v4->version == 4) { 3761 unsigned char *csum_start = skb_checksum_start(skb); 3762 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 3763 3764 /* IP header will have to cancel out any data that 3765 * is not a part of the outer IP header 3766 */ 3767 ip.v4->check = csum_fold(csum_partial(trans_start, 3768 csum_start - trans_start, 3769 0)); 3770 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3771 3772 ip.v4->tot_len = 0; 3773 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3774 IXGBE_TX_FLAGS_CSUM | 3775 IXGBE_TX_FLAGS_IPV4; 3776 } else { 3777 ip.v6->payload_len = 0; 3778 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3779 IXGBE_TX_FLAGS_CSUM; 3780 } 3781 3782 /* determine offset of inner transport header */ 3783 l4_offset = l4.hdr - skb->data; 3784 3785 /* compute length of segmentation header */ 3786 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 3787 3788 /* remove payload length from inner checksum */ 3789 paylen = skb->len - l4_offset; 3790 csum_replace_by_diff(&l4.tcp->check, htonl(paylen)); 3791 3792 /* update gso size and bytecount with header size */ 3793 first->gso_segs = skb_shinfo(skb)->gso_segs; 3794 first->bytecount += (first->gso_segs - 1) * *hdr_len; 3795 3796 /* mss_l4len_id: use 1 as index for TSO */ 3797 mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT; 3798 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 3799 mss_l4len_idx |= (1u << IXGBE_ADVTXD_IDX_SHIFT); 3800 3801 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 3802 vlan_macip_lens = l4.hdr - ip.hdr; 3803 vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT; 3804 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3805 3806 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3807 type_tucmd, mss_l4len_idx); 3808 3809 return 1; 3810 } 3811 3812 static inline bool ixgbevf_ipv6_csum_is_sctp(struct sk_buff *skb) 3813 { 3814 unsigned int offset = 0; 3815 3816 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 3817 3818 return offset == skb_checksum_start_offset(skb); 3819 } 3820 3821 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, 3822 struct ixgbevf_tx_buffer *first) 3823 { 3824 struct sk_buff *skb = first->skb; 3825 u32 vlan_macip_lens = 0; 3826 u32 type_tucmd = 0; 3827 3828 if (skb->ip_summed != CHECKSUM_PARTIAL) 3829 goto no_csum; 3830 3831 switch (skb->csum_offset) { 3832 case offsetof(struct tcphdr, check): 3833 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3834 /* fall through */ 3835 case offsetof(struct udphdr, check): 3836 break; 3837 case offsetof(struct sctphdr, checksum): 3838 /* validate that this is actually an SCTP request */ 3839 if (((first->protocol == htons(ETH_P_IP)) && 3840 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 3841 ((first->protocol == htons(ETH_P_IPV6)) && 3842 ixgbevf_ipv6_csum_is_sctp(skb))) { 3843 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_SCTP; 3844 break; 3845 } 3846 /* fall through */ 3847 default: 3848 skb_checksum_help(skb); 3849 goto no_csum; 3850 } 3851 /* update TX checksum flag */ 3852 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 3853 vlan_macip_lens = skb_checksum_start_offset(skb) - 3854 skb_network_offset(skb); 3855 no_csum: 3856 /* vlan_macip_lens: MACLEN, VLAN tag */ 3857 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3858 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3859 3860 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, 0); 3861 } 3862 3863 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags) 3864 { 3865 /* set type for advanced descriptor with frame checksum insertion */ 3866 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | 3867 IXGBE_ADVTXD_DCMD_IFCS | 3868 IXGBE_ADVTXD_DCMD_DEXT); 3869 3870 /* set HW VLAN bit if VLAN is present */ 3871 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 3872 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); 3873 3874 /* set segmentation enable bits for TSO/FSO */ 3875 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3876 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); 3877 3878 return cmd_type; 3879 } 3880 3881 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 3882 u32 tx_flags, unsigned int paylen) 3883 { 3884 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT); 3885 3886 /* enable L4 checksum for TSO and TX checksum offload */ 3887 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 3888 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM); 3889 3890 /* enble IPv4 checksum for TSO */ 3891 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 3892 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM); 3893 3894 /* use index 1 context for TSO/FSO/FCOE */ 3895 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3896 olinfo_status |= cpu_to_le32(1u << IXGBE_ADVTXD_IDX_SHIFT); 3897 3898 /* Check Context must be set if Tx switch is enabled, which it 3899 * always is for case where virtual functions are running 3900 */ 3901 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC); 3902 3903 tx_desc->read.olinfo_status = olinfo_status; 3904 } 3905 3906 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, 3907 struct ixgbevf_tx_buffer *first, 3908 const u8 hdr_len) 3909 { 3910 struct sk_buff *skb = first->skb; 3911 struct ixgbevf_tx_buffer *tx_buffer; 3912 union ixgbe_adv_tx_desc *tx_desc; 3913 struct skb_frag_struct *frag; 3914 dma_addr_t dma; 3915 unsigned int data_len, size; 3916 u32 tx_flags = first->tx_flags; 3917 __le32 cmd_type = ixgbevf_tx_cmd_type(tx_flags); 3918 u16 i = tx_ring->next_to_use; 3919 3920 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 3921 3922 ixgbevf_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len); 3923 3924 size = skb_headlen(skb); 3925 data_len = skb->data_len; 3926 3927 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 3928 3929 tx_buffer = first; 3930 3931 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 3932 if (dma_mapping_error(tx_ring->dev, dma)) 3933 goto dma_error; 3934 3935 /* record length, and DMA address */ 3936 dma_unmap_len_set(tx_buffer, len, size); 3937 dma_unmap_addr_set(tx_buffer, dma, dma); 3938 3939 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3940 3941 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 3942 tx_desc->read.cmd_type_len = 3943 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); 3944 3945 i++; 3946 tx_desc++; 3947 if (i == tx_ring->count) { 3948 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3949 i = 0; 3950 } 3951 tx_desc->read.olinfo_status = 0; 3952 3953 dma += IXGBE_MAX_DATA_PER_TXD; 3954 size -= IXGBE_MAX_DATA_PER_TXD; 3955 3956 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3957 } 3958 3959 if (likely(!data_len)) 3960 break; 3961 3962 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); 3963 3964 i++; 3965 tx_desc++; 3966 if (i == tx_ring->count) { 3967 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3968 i = 0; 3969 } 3970 tx_desc->read.olinfo_status = 0; 3971 3972 size = skb_frag_size(frag); 3973 data_len -= size; 3974 3975 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 3976 DMA_TO_DEVICE); 3977 3978 tx_buffer = &tx_ring->tx_buffer_info[i]; 3979 } 3980 3981 /* write last descriptor with RS and EOP bits */ 3982 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); 3983 tx_desc->read.cmd_type_len = cmd_type; 3984 3985 /* set the timestamp */ 3986 first->time_stamp = jiffies; 3987 3988 /* Force memory writes to complete before letting h/w know there 3989 * are new descriptors to fetch. (Only applicable for weak-ordered 3990 * memory model archs, such as IA-64). 3991 * 3992 * We also need this memory barrier (wmb) to make certain all of the 3993 * status bits have been updated before next_to_watch is written. 3994 */ 3995 wmb(); 3996 3997 /* set next_to_watch value indicating a packet is present */ 3998 first->next_to_watch = tx_desc; 3999 4000 i++; 4001 if (i == tx_ring->count) 4002 i = 0; 4003 4004 tx_ring->next_to_use = i; 4005 4006 /* notify HW of packet */ 4007 ixgbevf_write_tail(tx_ring, i); 4008 4009 return; 4010 dma_error: 4011 dev_err(tx_ring->dev, "TX DMA map failed\n"); 4012 tx_buffer = &tx_ring->tx_buffer_info[i]; 4013 4014 /* clear dma mappings for failed tx_buffer_info map */ 4015 while (tx_buffer != first) { 4016 if (dma_unmap_len(tx_buffer, len)) 4017 dma_unmap_page(tx_ring->dev, 4018 dma_unmap_addr(tx_buffer, dma), 4019 dma_unmap_len(tx_buffer, len), 4020 DMA_TO_DEVICE); 4021 dma_unmap_len_set(tx_buffer, len, 0); 4022 4023 if (i-- == 0) 4024 i += tx_ring->count; 4025 tx_buffer = &tx_ring->tx_buffer_info[i]; 4026 } 4027 4028 if (dma_unmap_len(tx_buffer, len)) 4029 dma_unmap_single(tx_ring->dev, 4030 dma_unmap_addr(tx_buffer, dma), 4031 dma_unmap_len(tx_buffer, len), 4032 DMA_TO_DEVICE); 4033 dma_unmap_len_set(tx_buffer, len, 0); 4034 4035 dev_kfree_skb_any(tx_buffer->skb); 4036 tx_buffer->skb = NULL; 4037 4038 tx_ring->next_to_use = i; 4039 } 4040 4041 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 4042 { 4043 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 4044 /* Herbert's original patch had: 4045 * smp_mb__after_netif_stop_queue(); 4046 * but since that doesn't exist yet, just open code it. 4047 */ 4048 smp_mb(); 4049 4050 /* We need to check again in a case another CPU has just 4051 * made room available. 4052 */ 4053 if (likely(ixgbevf_desc_unused(tx_ring) < size)) 4054 return -EBUSY; 4055 4056 /* A reprieve! - use start_queue because it doesn't call schedule */ 4057 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 4058 ++tx_ring->tx_stats.restart_queue; 4059 4060 return 0; 4061 } 4062 4063 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 4064 { 4065 if (likely(ixgbevf_desc_unused(tx_ring) >= size)) 4066 return 0; 4067 return __ixgbevf_maybe_stop_tx(tx_ring, size); 4068 } 4069 4070 static int ixgbevf_xmit_frame_ring(struct sk_buff *skb, 4071 struct ixgbevf_ring *tx_ring) 4072 { 4073 struct ixgbevf_tx_buffer *first; 4074 int tso; 4075 u32 tx_flags = 0; 4076 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 4077 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 4078 unsigned short f; 4079 #endif 4080 u8 hdr_len = 0; 4081 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL); 4082 4083 if (!dst_mac || is_link_local_ether_addr(dst_mac)) { 4084 dev_kfree_skb_any(skb); 4085 return NETDEV_TX_OK; 4086 } 4087 4088 /* need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 4089 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 4090 * + 2 desc gap to keep tail from touching head, 4091 * + 1 desc for context descriptor, 4092 * otherwise try next time 4093 */ 4094 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 4095 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 4096 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 4097 #else 4098 count += skb_shinfo(skb)->nr_frags; 4099 #endif 4100 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { 4101 tx_ring->tx_stats.tx_busy++; 4102 return NETDEV_TX_BUSY; 4103 } 4104 4105 /* record the location of the first descriptor for this packet */ 4106 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 4107 first->skb = skb; 4108 first->bytecount = skb->len; 4109 first->gso_segs = 1; 4110 4111 if (skb_vlan_tag_present(skb)) { 4112 tx_flags |= skb_vlan_tag_get(skb); 4113 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 4114 tx_flags |= IXGBE_TX_FLAGS_VLAN; 4115 } 4116 4117 /* record initial flags and protocol */ 4118 first->tx_flags = tx_flags; 4119 first->protocol = vlan_get_protocol(skb); 4120 4121 tso = ixgbevf_tso(tx_ring, first, &hdr_len); 4122 if (tso < 0) 4123 goto out_drop; 4124 else if (!tso) 4125 ixgbevf_tx_csum(tx_ring, first); 4126 4127 ixgbevf_tx_map(tx_ring, first, hdr_len); 4128 4129 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); 4130 4131 return NETDEV_TX_OK; 4132 4133 out_drop: 4134 dev_kfree_skb_any(first->skb); 4135 first->skb = NULL; 4136 4137 return NETDEV_TX_OK; 4138 } 4139 4140 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 4141 { 4142 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4143 struct ixgbevf_ring *tx_ring; 4144 4145 if (skb->len <= 0) { 4146 dev_kfree_skb_any(skb); 4147 return NETDEV_TX_OK; 4148 } 4149 4150 /* The minimum packet size for olinfo paylen is 17 so pad the skb 4151 * in order to meet this minimum size requirement. 4152 */ 4153 if (skb->len < 17) { 4154 if (skb_padto(skb, 17)) 4155 return NETDEV_TX_OK; 4156 skb->len = 17; 4157 } 4158 4159 tx_ring = adapter->tx_ring[skb->queue_mapping]; 4160 return ixgbevf_xmit_frame_ring(skb, tx_ring); 4161 } 4162 4163 /** 4164 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 4165 * @netdev: network interface device structure 4166 * @p: pointer to an address structure 4167 * 4168 * Returns 0 on success, negative on failure 4169 **/ 4170 static int ixgbevf_set_mac(struct net_device *netdev, void *p) 4171 { 4172 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4173 struct ixgbe_hw *hw = &adapter->hw; 4174 struct sockaddr *addr = p; 4175 int err; 4176 4177 if (!is_valid_ether_addr(addr->sa_data)) 4178 return -EADDRNOTAVAIL; 4179 4180 spin_lock_bh(&adapter->mbx_lock); 4181 4182 err = hw->mac.ops.set_rar(hw, 0, addr->sa_data, 0); 4183 4184 spin_unlock_bh(&adapter->mbx_lock); 4185 4186 if (err) 4187 return -EPERM; 4188 4189 ether_addr_copy(hw->mac.addr, addr->sa_data); 4190 ether_addr_copy(netdev->dev_addr, addr->sa_data); 4191 4192 return 0; 4193 } 4194 4195 /** 4196 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 4197 * @netdev: network interface device structure 4198 * @new_mtu: new value for maximum frame size 4199 * 4200 * Returns 0 on success, negative on failure 4201 **/ 4202 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 4203 { 4204 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4205 struct ixgbe_hw *hw = &adapter->hw; 4206 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 4207 int ret; 4208 4209 /* prevent MTU being changed to a size unsupported by XDP */ 4210 if (adapter->xdp_prog) { 4211 dev_warn(&adapter->pdev->dev, "MTU cannot be changed while XDP program is loaded\n"); 4212 return -EPERM; 4213 } 4214 4215 spin_lock_bh(&adapter->mbx_lock); 4216 /* notify the PF of our intent to use this size of frame */ 4217 ret = hw->mac.ops.set_rlpml(hw, max_frame); 4218 spin_unlock_bh(&adapter->mbx_lock); 4219 if (ret) 4220 return -EINVAL; 4221 4222 hw_dbg(hw, "changing MTU from %d to %d\n", 4223 netdev->mtu, new_mtu); 4224 4225 /* must set new MTU before calling down or up */ 4226 netdev->mtu = new_mtu; 4227 4228 if (netif_running(netdev)) 4229 ixgbevf_reinit_locked(adapter); 4230 4231 return 0; 4232 } 4233 4234 #ifdef CONFIG_NET_POLL_CONTROLLER 4235 /* Polling 'interrupt' - used by things like netconsole to send skbs 4236 * without having to re-enable interrupts. It's not called while 4237 * the interrupt routine is executing. 4238 */ 4239 static void ixgbevf_netpoll(struct net_device *netdev) 4240 { 4241 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4242 int i; 4243 4244 /* if interface is down do nothing */ 4245 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 4246 return; 4247 for (i = 0; i < adapter->num_rx_queues; i++) 4248 ixgbevf_msix_clean_rings(0, adapter->q_vector[i]); 4249 } 4250 #endif /* CONFIG_NET_POLL_CONTROLLER */ 4251 4252 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) 4253 { 4254 struct net_device *netdev = pci_get_drvdata(pdev); 4255 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4256 #ifdef CONFIG_PM 4257 int retval = 0; 4258 #endif 4259 4260 rtnl_lock(); 4261 netif_device_detach(netdev); 4262 4263 if (netif_running(netdev)) 4264 ixgbevf_close_suspend(adapter); 4265 4266 ixgbevf_clear_interrupt_scheme(adapter); 4267 rtnl_unlock(); 4268 4269 #ifdef CONFIG_PM 4270 retval = pci_save_state(pdev); 4271 if (retval) 4272 return retval; 4273 4274 #endif 4275 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 4276 pci_disable_device(pdev); 4277 4278 return 0; 4279 } 4280 4281 #ifdef CONFIG_PM 4282 static int ixgbevf_resume(struct pci_dev *pdev) 4283 { 4284 struct net_device *netdev = pci_get_drvdata(pdev); 4285 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4286 u32 err; 4287 4288 pci_restore_state(pdev); 4289 /* pci_restore_state clears dev->state_saved so call 4290 * pci_save_state to restore it. 4291 */ 4292 pci_save_state(pdev); 4293 4294 err = pci_enable_device_mem(pdev); 4295 if (err) { 4296 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 4297 return err; 4298 } 4299 4300 adapter->hw.hw_addr = adapter->io_addr; 4301 smp_mb__before_atomic(); 4302 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 4303 pci_set_master(pdev); 4304 4305 ixgbevf_reset(adapter); 4306 4307 rtnl_lock(); 4308 err = ixgbevf_init_interrupt_scheme(adapter); 4309 if (!err && netif_running(netdev)) 4310 err = ixgbevf_open(netdev); 4311 rtnl_unlock(); 4312 if (err) 4313 return err; 4314 4315 netif_device_attach(netdev); 4316 4317 return err; 4318 } 4319 4320 #endif /* CONFIG_PM */ 4321 static void ixgbevf_shutdown(struct pci_dev *pdev) 4322 { 4323 ixgbevf_suspend(pdev, PMSG_SUSPEND); 4324 } 4325 4326 static void ixgbevf_get_tx_ring_stats(struct rtnl_link_stats64 *stats, 4327 const struct ixgbevf_ring *ring) 4328 { 4329 u64 bytes, packets; 4330 unsigned int start; 4331 4332 if (ring) { 4333 do { 4334 start = u64_stats_fetch_begin_irq(&ring->syncp); 4335 bytes = ring->stats.bytes; 4336 packets = ring->stats.packets; 4337 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 4338 stats->tx_bytes += bytes; 4339 stats->tx_packets += packets; 4340 } 4341 } 4342 4343 static void ixgbevf_get_stats(struct net_device *netdev, 4344 struct rtnl_link_stats64 *stats) 4345 { 4346 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4347 unsigned int start; 4348 u64 bytes, packets; 4349 const struct ixgbevf_ring *ring; 4350 int i; 4351 4352 ixgbevf_update_stats(adapter); 4353 4354 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 4355 4356 rcu_read_lock(); 4357 for (i = 0; i < adapter->num_rx_queues; i++) { 4358 ring = adapter->rx_ring[i]; 4359 do { 4360 start = u64_stats_fetch_begin_irq(&ring->syncp); 4361 bytes = ring->stats.bytes; 4362 packets = ring->stats.packets; 4363 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 4364 stats->rx_bytes += bytes; 4365 stats->rx_packets += packets; 4366 } 4367 4368 for (i = 0; i < adapter->num_tx_queues; i++) { 4369 ring = adapter->tx_ring[i]; 4370 ixgbevf_get_tx_ring_stats(stats, ring); 4371 } 4372 4373 for (i = 0; i < adapter->num_xdp_queues; i++) { 4374 ring = adapter->xdp_ring[i]; 4375 ixgbevf_get_tx_ring_stats(stats, ring); 4376 } 4377 rcu_read_unlock(); 4378 } 4379 4380 #define IXGBEVF_MAX_MAC_HDR_LEN 127 4381 #define IXGBEVF_MAX_NETWORK_HDR_LEN 511 4382 4383 static netdev_features_t 4384 ixgbevf_features_check(struct sk_buff *skb, struct net_device *dev, 4385 netdev_features_t features) 4386 { 4387 unsigned int network_hdr_len, mac_hdr_len; 4388 4389 /* Make certain the headers can be described by a context descriptor */ 4390 mac_hdr_len = skb_network_header(skb) - skb->data; 4391 if (unlikely(mac_hdr_len > IXGBEVF_MAX_MAC_HDR_LEN)) 4392 return features & ~(NETIF_F_HW_CSUM | 4393 NETIF_F_SCTP_CRC | 4394 NETIF_F_HW_VLAN_CTAG_TX | 4395 NETIF_F_TSO | 4396 NETIF_F_TSO6); 4397 4398 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 4399 if (unlikely(network_hdr_len > IXGBEVF_MAX_NETWORK_HDR_LEN)) 4400 return features & ~(NETIF_F_HW_CSUM | 4401 NETIF_F_SCTP_CRC | 4402 NETIF_F_TSO | 4403 NETIF_F_TSO6); 4404 4405 /* We can only support IPV4 TSO in tunnels if we can mangle the 4406 * inner IP ID field, so strip TSO if MANGLEID is not supported. 4407 */ 4408 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 4409 features &= ~NETIF_F_TSO; 4410 4411 return features; 4412 } 4413 4414 static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) 4415 { 4416 int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 4417 struct ixgbevf_adapter *adapter = netdev_priv(dev); 4418 struct bpf_prog *old_prog; 4419 4420 /* verify ixgbevf ring attributes are sufficient for XDP */ 4421 for (i = 0; i < adapter->num_rx_queues; i++) { 4422 struct ixgbevf_ring *ring = adapter->rx_ring[i]; 4423 4424 if (frame_size > ixgbevf_rx_bufsz(ring)) 4425 return -EINVAL; 4426 } 4427 4428 old_prog = xchg(&adapter->xdp_prog, prog); 4429 4430 /* If transitioning XDP modes reconfigure rings */ 4431 if (!!prog != !!old_prog) { 4432 /* Hardware has to reinitialize queues and interrupts to 4433 * match packet buffer alignment. Unfortunately, the 4434 * hardware is not flexible enough to do this dynamically. 4435 */ 4436 if (netif_running(dev)) 4437 ixgbevf_close(dev); 4438 4439 ixgbevf_clear_interrupt_scheme(adapter); 4440 ixgbevf_init_interrupt_scheme(adapter); 4441 4442 if (netif_running(dev)) 4443 ixgbevf_open(dev); 4444 } else { 4445 for (i = 0; i < adapter->num_rx_queues; i++) 4446 xchg(&adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog); 4447 } 4448 4449 if (old_prog) 4450 bpf_prog_put(old_prog); 4451 4452 return 0; 4453 } 4454 4455 static int ixgbevf_xdp(struct net_device *dev, struct netdev_bpf *xdp) 4456 { 4457 struct ixgbevf_adapter *adapter = netdev_priv(dev); 4458 4459 switch (xdp->command) { 4460 case XDP_SETUP_PROG: 4461 return ixgbevf_xdp_setup(dev, xdp->prog); 4462 case XDP_QUERY_PROG: 4463 xdp->prog_attached = !!(adapter->xdp_prog); 4464 xdp->prog_id = adapter->xdp_prog ? 4465 adapter->xdp_prog->aux->id : 0; 4466 return 0; 4467 default: 4468 return -EINVAL; 4469 } 4470 } 4471 4472 static const struct net_device_ops ixgbevf_netdev_ops = { 4473 .ndo_open = ixgbevf_open, 4474 .ndo_stop = ixgbevf_close, 4475 .ndo_start_xmit = ixgbevf_xmit_frame, 4476 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 4477 .ndo_get_stats64 = ixgbevf_get_stats, 4478 .ndo_validate_addr = eth_validate_addr, 4479 .ndo_set_mac_address = ixgbevf_set_mac, 4480 .ndo_change_mtu = ixgbevf_change_mtu, 4481 .ndo_tx_timeout = ixgbevf_tx_timeout, 4482 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 4483 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 4484 #ifdef CONFIG_NET_POLL_CONTROLLER 4485 .ndo_poll_controller = ixgbevf_netpoll, 4486 #endif 4487 .ndo_features_check = ixgbevf_features_check, 4488 .ndo_bpf = ixgbevf_xdp, 4489 }; 4490 4491 static void ixgbevf_assign_netdev_ops(struct net_device *dev) 4492 { 4493 dev->netdev_ops = &ixgbevf_netdev_ops; 4494 ixgbevf_set_ethtool_ops(dev); 4495 dev->watchdog_timeo = 5 * HZ; 4496 } 4497 4498 /** 4499 * ixgbevf_probe - Device Initialization Routine 4500 * @pdev: PCI device information struct 4501 * @ent: entry in ixgbevf_pci_tbl 4502 * 4503 * Returns 0 on success, negative on failure 4504 * 4505 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 4506 * The OS initialization, configuring of the adapter private structure, 4507 * and a hardware reset occur. 4508 **/ 4509 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 4510 { 4511 struct net_device *netdev; 4512 struct ixgbevf_adapter *adapter = NULL; 4513 struct ixgbe_hw *hw = NULL; 4514 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 4515 int err, pci_using_dac; 4516 bool disable_dev = false; 4517 4518 err = pci_enable_device(pdev); 4519 if (err) 4520 return err; 4521 4522 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 4523 pci_using_dac = 1; 4524 } else { 4525 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 4526 if (err) { 4527 dev_err(&pdev->dev, "No usable DMA configuration, aborting\n"); 4528 goto err_dma; 4529 } 4530 pci_using_dac = 0; 4531 } 4532 4533 err = pci_request_regions(pdev, ixgbevf_driver_name); 4534 if (err) { 4535 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 4536 goto err_pci_reg; 4537 } 4538 4539 pci_set_master(pdev); 4540 4541 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 4542 MAX_TX_QUEUES); 4543 if (!netdev) { 4544 err = -ENOMEM; 4545 goto err_alloc_etherdev; 4546 } 4547 4548 SET_NETDEV_DEV(netdev, &pdev->dev); 4549 4550 adapter = netdev_priv(netdev); 4551 4552 adapter->netdev = netdev; 4553 adapter->pdev = pdev; 4554 hw = &adapter->hw; 4555 hw->back = adapter; 4556 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 4557 4558 /* call save state here in standalone driver because it relies on 4559 * adapter struct to exist, and needs to call netdev_priv 4560 */ 4561 pci_save_state(pdev); 4562 4563 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 4564 pci_resource_len(pdev, 0)); 4565 adapter->io_addr = hw->hw_addr; 4566 if (!hw->hw_addr) { 4567 err = -EIO; 4568 goto err_ioremap; 4569 } 4570 4571 ixgbevf_assign_netdev_ops(netdev); 4572 4573 /* Setup HW API */ 4574 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 4575 hw->mac.type = ii->mac; 4576 4577 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 4578 sizeof(struct ixgbe_mbx_operations)); 4579 4580 /* setup the private structure */ 4581 err = ixgbevf_sw_init(adapter); 4582 if (err) 4583 goto err_sw_init; 4584 4585 /* The HW MAC address was set and/or determined in sw_init */ 4586 if (!is_valid_ether_addr(netdev->dev_addr)) { 4587 pr_err("invalid MAC address\n"); 4588 err = -EIO; 4589 goto err_sw_init; 4590 } 4591 4592 netdev->hw_features = NETIF_F_SG | 4593 NETIF_F_TSO | 4594 NETIF_F_TSO6 | 4595 NETIF_F_RXCSUM | 4596 NETIF_F_HW_CSUM | 4597 NETIF_F_SCTP_CRC; 4598 4599 #define IXGBEVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 4600 NETIF_F_GSO_GRE_CSUM | \ 4601 NETIF_F_GSO_IPXIP4 | \ 4602 NETIF_F_GSO_IPXIP6 | \ 4603 NETIF_F_GSO_UDP_TUNNEL | \ 4604 NETIF_F_GSO_UDP_TUNNEL_CSUM) 4605 4606 netdev->gso_partial_features = IXGBEVF_GSO_PARTIAL_FEATURES; 4607 netdev->hw_features |= NETIF_F_GSO_PARTIAL | 4608 IXGBEVF_GSO_PARTIAL_FEATURES; 4609 4610 netdev->features = netdev->hw_features; 4611 4612 if (pci_using_dac) 4613 netdev->features |= NETIF_F_HIGHDMA; 4614 4615 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; 4616 netdev->mpls_features |= NETIF_F_SG | 4617 NETIF_F_TSO | 4618 NETIF_F_TSO6 | 4619 NETIF_F_HW_CSUM; 4620 netdev->mpls_features |= IXGBEVF_GSO_PARTIAL_FEATURES; 4621 netdev->hw_enc_features |= netdev->vlan_features; 4622 4623 /* set this bit last since it cannot be part of vlan_features */ 4624 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | 4625 NETIF_F_HW_VLAN_CTAG_RX | 4626 NETIF_F_HW_VLAN_CTAG_TX; 4627 4628 netdev->priv_flags |= IFF_UNICAST_FLT; 4629 4630 /* MTU range: 68 - 1504 or 9710 */ 4631 netdev->min_mtu = ETH_MIN_MTU; 4632 switch (adapter->hw.api_version) { 4633 case ixgbe_mbox_api_11: 4634 case ixgbe_mbox_api_12: 4635 case ixgbe_mbox_api_13: 4636 netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - 4637 (ETH_HLEN + ETH_FCS_LEN); 4638 break; 4639 default: 4640 if (adapter->hw.mac.type != ixgbe_mac_82599_vf) 4641 netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - 4642 (ETH_HLEN + ETH_FCS_LEN); 4643 else 4644 netdev->max_mtu = ETH_DATA_LEN + ETH_FCS_LEN; 4645 break; 4646 } 4647 4648 if (IXGBE_REMOVED(hw->hw_addr)) { 4649 err = -EIO; 4650 goto err_sw_init; 4651 } 4652 4653 timer_setup(&adapter->service_timer, ixgbevf_service_timer, 0); 4654 4655 INIT_WORK(&adapter->service_task, ixgbevf_service_task); 4656 set_bit(__IXGBEVF_SERVICE_INITED, &adapter->state); 4657 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 4658 4659 err = ixgbevf_init_interrupt_scheme(adapter); 4660 if (err) 4661 goto err_sw_init; 4662 4663 strcpy(netdev->name, "eth%d"); 4664 4665 err = register_netdev(netdev); 4666 if (err) 4667 goto err_register; 4668 4669 pci_set_drvdata(pdev, netdev); 4670 netif_carrier_off(netdev); 4671 4672 ixgbevf_init_last_counter_stats(adapter); 4673 4674 /* print the VF info */ 4675 dev_info(&pdev->dev, "%pM\n", netdev->dev_addr); 4676 dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type); 4677 4678 switch (hw->mac.type) { 4679 case ixgbe_mac_X550_vf: 4680 dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n"); 4681 break; 4682 case ixgbe_mac_X540_vf: 4683 dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n"); 4684 break; 4685 case ixgbe_mac_82599_vf: 4686 default: 4687 dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n"); 4688 break; 4689 } 4690 4691 return 0; 4692 4693 err_register: 4694 ixgbevf_clear_interrupt_scheme(adapter); 4695 err_sw_init: 4696 ixgbevf_reset_interrupt_capability(adapter); 4697 iounmap(adapter->io_addr); 4698 kfree(adapter->rss_key); 4699 err_ioremap: 4700 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4701 free_netdev(netdev); 4702 err_alloc_etherdev: 4703 pci_release_regions(pdev); 4704 err_pci_reg: 4705 err_dma: 4706 if (!adapter || disable_dev) 4707 pci_disable_device(pdev); 4708 return err; 4709 } 4710 4711 /** 4712 * ixgbevf_remove - Device Removal Routine 4713 * @pdev: PCI device information struct 4714 * 4715 * ixgbevf_remove is called by the PCI subsystem to alert the driver 4716 * that it should release a PCI device. The could be caused by a 4717 * Hot-Plug event, or because the driver is going to be removed from 4718 * memory. 4719 **/ 4720 static void ixgbevf_remove(struct pci_dev *pdev) 4721 { 4722 struct net_device *netdev = pci_get_drvdata(pdev); 4723 struct ixgbevf_adapter *adapter; 4724 bool disable_dev; 4725 4726 if (!netdev) 4727 return; 4728 4729 adapter = netdev_priv(netdev); 4730 4731 set_bit(__IXGBEVF_REMOVING, &adapter->state); 4732 cancel_work_sync(&adapter->service_task); 4733 4734 if (netdev->reg_state == NETREG_REGISTERED) 4735 unregister_netdev(netdev); 4736 4737 ixgbevf_clear_interrupt_scheme(adapter); 4738 ixgbevf_reset_interrupt_capability(adapter); 4739 4740 iounmap(adapter->io_addr); 4741 pci_release_regions(pdev); 4742 4743 hw_dbg(&adapter->hw, "Remove complete\n"); 4744 4745 kfree(adapter->rss_key); 4746 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4747 free_netdev(netdev); 4748 4749 if (disable_dev) 4750 pci_disable_device(pdev); 4751 } 4752 4753 /** 4754 * ixgbevf_io_error_detected - called when PCI error is detected 4755 * @pdev: Pointer to PCI device 4756 * @state: The current pci connection state 4757 * 4758 * This function is called after a PCI bus error affecting 4759 * this device has been detected. 4760 **/ 4761 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, 4762 pci_channel_state_t state) 4763 { 4764 struct net_device *netdev = pci_get_drvdata(pdev); 4765 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4766 4767 if (!test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 4768 return PCI_ERS_RESULT_DISCONNECT; 4769 4770 rtnl_lock(); 4771 netif_device_detach(netdev); 4772 4773 if (state == pci_channel_io_perm_failure) { 4774 rtnl_unlock(); 4775 return PCI_ERS_RESULT_DISCONNECT; 4776 } 4777 4778 if (netif_running(netdev)) 4779 ixgbevf_close_suspend(adapter); 4780 4781 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 4782 pci_disable_device(pdev); 4783 rtnl_unlock(); 4784 4785 /* Request a slot slot reset. */ 4786 return PCI_ERS_RESULT_NEED_RESET; 4787 } 4788 4789 /** 4790 * ixgbevf_io_slot_reset - called after the pci bus has been reset. 4791 * @pdev: Pointer to PCI device 4792 * 4793 * Restart the card from scratch, as if from a cold-boot. Implementation 4794 * resembles the first-half of the ixgbevf_resume routine. 4795 **/ 4796 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) 4797 { 4798 struct net_device *netdev = pci_get_drvdata(pdev); 4799 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4800 4801 if (pci_enable_device_mem(pdev)) { 4802 dev_err(&pdev->dev, 4803 "Cannot re-enable PCI device after reset.\n"); 4804 return PCI_ERS_RESULT_DISCONNECT; 4805 } 4806 4807 adapter->hw.hw_addr = adapter->io_addr; 4808 smp_mb__before_atomic(); 4809 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 4810 pci_set_master(pdev); 4811 4812 ixgbevf_reset(adapter); 4813 4814 return PCI_ERS_RESULT_RECOVERED; 4815 } 4816 4817 /** 4818 * ixgbevf_io_resume - called when traffic can start flowing again. 4819 * @pdev: Pointer to PCI device 4820 * 4821 * This callback is called when the error recovery driver tells us that 4822 * its OK to resume normal operation. Implementation resembles the 4823 * second-half of the ixgbevf_resume routine. 4824 **/ 4825 static void ixgbevf_io_resume(struct pci_dev *pdev) 4826 { 4827 struct net_device *netdev = pci_get_drvdata(pdev); 4828 4829 rtnl_lock(); 4830 if (netif_running(netdev)) 4831 ixgbevf_open(netdev); 4832 4833 netif_device_attach(netdev); 4834 rtnl_unlock(); 4835 } 4836 4837 /* PCI Error Recovery (ERS) */ 4838 static const struct pci_error_handlers ixgbevf_err_handler = { 4839 .error_detected = ixgbevf_io_error_detected, 4840 .slot_reset = ixgbevf_io_slot_reset, 4841 .resume = ixgbevf_io_resume, 4842 }; 4843 4844 static struct pci_driver ixgbevf_driver = { 4845 .name = ixgbevf_driver_name, 4846 .id_table = ixgbevf_pci_tbl, 4847 .probe = ixgbevf_probe, 4848 .remove = ixgbevf_remove, 4849 #ifdef CONFIG_PM 4850 /* Power Management Hooks */ 4851 .suspend = ixgbevf_suspend, 4852 .resume = ixgbevf_resume, 4853 #endif 4854 .shutdown = ixgbevf_shutdown, 4855 .err_handler = &ixgbevf_err_handler 4856 }; 4857 4858 /** 4859 * ixgbevf_init_module - Driver Registration Routine 4860 * 4861 * ixgbevf_init_module is the first routine called when the driver is 4862 * loaded. All it does is register with the PCI subsystem. 4863 **/ 4864 static int __init ixgbevf_init_module(void) 4865 { 4866 pr_info("%s - version %s\n", ixgbevf_driver_string, 4867 ixgbevf_driver_version); 4868 4869 pr_info("%s\n", ixgbevf_copyright); 4870 ixgbevf_wq = create_singlethread_workqueue(ixgbevf_driver_name); 4871 if (!ixgbevf_wq) { 4872 pr_err("%s: Failed to create workqueue\n", ixgbevf_driver_name); 4873 return -ENOMEM; 4874 } 4875 4876 return pci_register_driver(&ixgbevf_driver); 4877 } 4878 4879 module_init(ixgbevf_init_module); 4880 4881 /** 4882 * ixgbevf_exit_module - Driver Exit Cleanup Routine 4883 * 4884 * ixgbevf_exit_module is called just before the driver is removed 4885 * from memory. 4886 **/ 4887 static void __exit ixgbevf_exit_module(void) 4888 { 4889 pci_unregister_driver(&ixgbevf_driver); 4890 if (ixgbevf_wq) { 4891 destroy_workqueue(ixgbevf_wq); 4892 ixgbevf_wq = NULL; 4893 } 4894 } 4895 4896 #ifdef DEBUG 4897 /** 4898 * ixgbevf_get_hw_dev_name - return device name string 4899 * used by hardware layer to print debugging information 4900 * @hw: pointer to private hardware struct 4901 **/ 4902 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 4903 { 4904 struct ixgbevf_adapter *adapter = hw->back; 4905 4906 return adapter->netdev->name; 4907 } 4908 4909 #endif 4910 module_exit(ixgbevf_exit_module); 4911 4912 /* ixgbevf_main.c */ 4913