1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 1999 - 2018 Intel Corporation. */ 3 4 /****************************************************************************** 5 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 6 ******************************************************************************/ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 10 #include <linux/types.h> 11 #include <linux/bitops.h> 12 #include <linux/module.h> 13 #include <linux/pci.h> 14 #include <linux/netdevice.h> 15 #include <linux/vmalloc.h> 16 #include <linux/string.h> 17 #include <linux/in.h> 18 #include <linux/ip.h> 19 #include <linux/tcp.h> 20 #include <linux/sctp.h> 21 #include <linux/ipv6.h> 22 #include <linux/slab.h> 23 #include <net/checksum.h> 24 #include <net/ip6_checksum.h> 25 #include <linux/ethtool.h> 26 #include <linux/if.h> 27 #include <linux/if_vlan.h> 28 #include <linux/prefetch.h> 29 #include <net/mpls.h> 30 #include <linux/bpf.h> 31 #include <linux/bpf_trace.h> 32 #include <linux/atomic.h> 33 34 #include "ixgbevf.h" 35 36 const char ixgbevf_driver_name[] = "ixgbevf"; 37 static const char ixgbevf_driver_string[] = 38 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 39 40 #define DRV_VERSION "4.1.0-k" 41 const char ixgbevf_driver_version[] = DRV_VERSION; 42 static char ixgbevf_copyright[] = 43 "Copyright (c) 2009 - 2018 Intel Corporation."; 44 45 static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 46 [board_82599_vf] = &ixgbevf_82599_vf_info, 47 [board_82599_vf_hv] = &ixgbevf_82599_vf_hv_info, 48 [board_X540_vf] = &ixgbevf_X540_vf_info, 49 [board_X540_vf_hv] = &ixgbevf_X540_vf_hv_info, 50 [board_X550_vf] = &ixgbevf_X550_vf_info, 51 [board_X550_vf_hv] = &ixgbevf_X550_vf_hv_info, 52 [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info, 53 [board_X550EM_x_vf_hv] = &ixgbevf_X550EM_x_vf_hv_info, 54 [board_x550em_a_vf] = &ixgbevf_x550em_a_vf_info, 55 }; 56 57 /* ixgbevf_pci_tbl - PCI Device ID Table 58 * 59 * Wildcard entries (PCI_ANY_ID) should come last 60 * Last entry must be all 0s 61 * 62 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 63 * Class, Class Mask, private data (not used) } 64 */ 65 static const struct pci_device_id ixgbevf_pci_tbl[] = { 66 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf }, 67 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF_HV), board_82599_vf_hv }, 68 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf }, 69 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF_HV), board_X540_vf_hv }, 70 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf }, 71 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF_HV), board_X550_vf_hv }, 72 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf }, 73 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF_HV), board_X550EM_x_vf_hv}, 74 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_VF), board_x550em_a_vf }, 75 /* required last entry */ 76 {0, } 77 }; 78 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 79 80 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 81 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); 82 MODULE_LICENSE("GPL v2"); 83 MODULE_VERSION(DRV_VERSION); 84 85 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 86 static int debug = -1; 87 module_param(debug, int, 0); 88 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 89 90 static struct workqueue_struct *ixgbevf_wq; 91 92 static void ixgbevf_service_event_schedule(struct ixgbevf_adapter *adapter) 93 { 94 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 95 !test_bit(__IXGBEVF_REMOVING, &adapter->state) && 96 !test_and_set_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)) 97 queue_work(ixgbevf_wq, &adapter->service_task); 98 } 99 100 static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter) 101 { 102 BUG_ON(!test_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)); 103 104 /* flush memory to make sure state is correct before next watchdog */ 105 smp_mb__before_atomic(); 106 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 107 } 108 109 /* forward decls */ 110 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); 111 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); 112 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); 113 static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer); 114 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, 115 struct ixgbevf_rx_buffer *old_buff); 116 117 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) 118 { 119 struct ixgbevf_adapter *adapter = hw->back; 120 121 if (!hw->hw_addr) 122 return; 123 hw->hw_addr = NULL; 124 dev_err(&adapter->pdev->dev, "Adapter removed\n"); 125 if (test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 126 ixgbevf_service_event_schedule(adapter); 127 } 128 129 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg) 130 { 131 u32 value; 132 133 /* The following check not only optimizes a bit by not 134 * performing a read on the status register when the 135 * register just read was a status register read that 136 * returned IXGBE_FAILED_READ_REG. It also blocks any 137 * potential recursion. 138 */ 139 if (reg == IXGBE_VFSTATUS) { 140 ixgbevf_remove_adapter(hw); 141 return; 142 } 143 value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS); 144 if (value == IXGBE_FAILED_READ_REG) 145 ixgbevf_remove_adapter(hw); 146 } 147 148 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg) 149 { 150 u8 __iomem *reg_addr = READ_ONCE(hw->hw_addr); 151 u32 value; 152 153 if (IXGBE_REMOVED(reg_addr)) 154 return IXGBE_FAILED_READ_REG; 155 value = readl(reg_addr + reg); 156 if (unlikely(value == IXGBE_FAILED_READ_REG)) 157 ixgbevf_check_remove(hw, reg); 158 return value; 159 } 160 161 /** 162 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 163 * @adapter: pointer to adapter struct 164 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 165 * @queue: queue to map the corresponding interrupt to 166 * @msix_vector: the vector to map to the corresponding queue 167 **/ 168 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 169 u8 queue, u8 msix_vector) 170 { 171 u32 ivar, index; 172 struct ixgbe_hw *hw = &adapter->hw; 173 174 if (direction == -1) { 175 /* other causes */ 176 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 177 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 178 ivar &= ~0xFF; 179 ivar |= msix_vector; 180 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 181 } else { 182 /* Tx or Rx causes */ 183 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 184 index = ((16 * (queue & 1)) + (8 * direction)); 185 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 186 ivar &= ~(0xFF << index); 187 ivar |= (msix_vector << index); 188 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 189 } 190 } 191 192 static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring) 193 { 194 return ring->stats.packets; 195 } 196 197 static u32 ixgbevf_get_tx_pending(struct ixgbevf_ring *ring) 198 { 199 struct ixgbevf_adapter *adapter = netdev_priv(ring->netdev); 200 struct ixgbe_hw *hw = &adapter->hw; 201 202 u32 head = IXGBE_READ_REG(hw, IXGBE_VFTDH(ring->reg_idx)); 203 u32 tail = IXGBE_READ_REG(hw, IXGBE_VFTDT(ring->reg_idx)); 204 205 if (head != tail) 206 return (head < tail) ? 207 tail - head : (tail + ring->count - head); 208 209 return 0; 210 } 211 212 static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring) 213 { 214 u32 tx_done = ixgbevf_get_tx_completed(tx_ring); 215 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 216 u32 tx_pending = ixgbevf_get_tx_pending(tx_ring); 217 218 clear_check_for_tx_hang(tx_ring); 219 220 /* Check for a hung queue, but be thorough. This verifies 221 * that a transmit has been completed since the previous 222 * check AND there is at least one packet pending. The 223 * ARMED bit is set to indicate a potential hang. 224 */ 225 if ((tx_done_old == tx_done) && tx_pending) { 226 /* make sure it is true for two checks in a row */ 227 return test_and_set_bit(__IXGBEVF_HANG_CHECK_ARMED, 228 &tx_ring->state); 229 } 230 /* reset the countdown */ 231 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &tx_ring->state); 232 233 /* update completed stats and continue */ 234 tx_ring->tx_stats.tx_done_old = tx_done; 235 236 return false; 237 } 238 239 static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter) 240 { 241 /* Do the reset outside of interrupt context */ 242 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 243 set_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state); 244 ixgbevf_service_event_schedule(adapter); 245 } 246 } 247 248 /** 249 * ixgbevf_tx_timeout - Respond to a Tx Hang 250 * @netdev: network interface device structure 251 **/ 252 static void ixgbevf_tx_timeout(struct net_device *netdev) 253 { 254 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 255 256 ixgbevf_tx_timeout_reset(adapter); 257 } 258 259 /** 260 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 261 * @q_vector: board private structure 262 * @tx_ring: tx ring to clean 263 * @napi_budget: Used to determine if we are in netpoll 264 **/ 265 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, 266 struct ixgbevf_ring *tx_ring, int napi_budget) 267 { 268 struct ixgbevf_adapter *adapter = q_vector->adapter; 269 struct ixgbevf_tx_buffer *tx_buffer; 270 union ixgbe_adv_tx_desc *tx_desc; 271 unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0; 272 unsigned int budget = tx_ring->count / 2; 273 unsigned int i = tx_ring->next_to_clean; 274 275 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 276 return true; 277 278 tx_buffer = &tx_ring->tx_buffer_info[i]; 279 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 280 i -= tx_ring->count; 281 282 do { 283 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 284 285 /* if next_to_watch is not set then there is no work pending */ 286 if (!eop_desc) 287 break; 288 289 /* prevent any other reads prior to eop_desc */ 290 smp_rmb(); 291 292 /* if DD is not set pending work has not been completed */ 293 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 294 break; 295 296 /* clear next_to_watch to prevent false hangs */ 297 tx_buffer->next_to_watch = NULL; 298 299 /* update the statistics for this packet */ 300 total_bytes += tx_buffer->bytecount; 301 total_packets += tx_buffer->gso_segs; 302 if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC) 303 total_ipsec++; 304 305 /* free the skb */ 306 if (ring_is_xdp(tx_ring)) 307 page_frag_free(tx_buffer->data); 308 else 309 napi_consume_skb(tx_buffer->skb, napi_budget); 310 311 /* unmap skb header data */ 312 dma_unmap_single(tx_ring->dev, 313 dma_unmap_addr(tx_buffer, dma), 314 dma_unmap_len(tx_buffer, len), 315 DMA_TO_DEVICE); 316 317 /* clear tx_buffer data */ 318 dma_unmap_len_set(tx_buffer, len, 0); 319 320 /* unmap remaining buffers */ 321 while (tx_desc != eop_desc) { 322 tx_buffer++; 323 tx_desc++; 324 i++; 325 if (unlikely(!i)) { 326 i -= tx_ring->count; 327 tx_buffer = tx_ring->tx_buffer_info; 328 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 329 } 330 331 /* unmap any remaining paged data */ 332 if (dma_unmap_len(tx_buffer, len)) { 333 dma_unmap_page(tx_ring->dev, 334 dma_unmap_addr(tx_buffer, dma), 335 dma_unmap_len(tx_buffer, len), 336 DMA_TO_DEVICE); 337 dma_unmap_len_set(tx_buffer, len, 0); 338 } 339 } 340 341 /* move us one more past the eop_desc for start of next pkt */ 342 tx_buffer++; 343 tx_desc++; 344 i++; 345 if (unlikely(!i)) { 346 i -= tx_ring->count; 347 tx_buffer = tx_ring->tx_buffer_info; 348 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 349 } 350 351 /* issue prefetch for next Tx descriptor */ 352 prefetch(tx_desc); 353 354 /* update budget accounting */ 355 budget--; 356 } while (likely(budget)); 357 358 i += tx_ring->count; 359 tx_ring->next_to_clean = i; 360 u64_stats_update_begin(&tx_ring->syncp); 361 tx_ring->stats.bytes += total_bytes; 362 tx_ring->stats.packets += total_packets; 363 u64_stats_update_end(&tx_ring->syncp); 364 q_vector->tx.total_bytes += total_bytes; 365 q_vector->tx.total_packets += total_packets; 366 adapter->tx_ipsec += total_ipsec; 367 368 if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) { 369 struct ixgbe_hw *hw = &adapter->hw; 370 union ixgbe_adv_tx_desc *eop_desc; 371 372 eop_desc = tx_ring->tx_buffer_info[i].next_to_watch; 373 374 pr_err("Detected Tx Unit Hang%s\n" 375 " Tx Queue <%d>\n" 376 " TDH, TDT <%x>, <%x>\n" 377 " next_to_use <%x>\n" 378 " next_to_clean <%x>\n" 379 "tx_buffer_info[next_to_clean]\n" 380 " next_to_watch <%p>\n" 381 " eop_desc->wb.status <%x>\n" 382 " time_stamp <%lx>\n" 383 " jiffies <%lx>\n", 384 ring_is_xdp(tx_ring) ? " XDP" : "", 385 tx_ring->queue_index, 386 IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)), 387 IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)), 388 tx_ring->next_to_use, i, 389 eop_desc, (eop_desc ? eop_desc->wb.status : 0), 390 tx_ring->tx_buffer_info[i].time_stamp, jiffies); 391 392 if (!ring_is_xdp(tx_ring)) 393 netif_stop_subqueue(tx_ring->netdev, 394 tx_ring->queue_index); 395 396 /* schedule immediate reset if we believe we hung */ 397 ixgbevf_tx_timeout_reset(adapter); 398 399 return true; 400 } 401 402 if (ring_is_xdp(tx_ring)) 403 return !!budget; 404 405 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 406 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 407 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 408 /* Make sure that anybody stopping the queue after this 409 * sees the new next_to_clean. 410 */ 411 smp_mb(); 412 413 if (__netif_subqueue_stopped(tx_ring->netdev, 414 tx_ring->queue_index) && 415 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 416 netif_wake_subqueue(tx_ring->netdev, 417 tx_ring->queue_index); 418 ++tx_ring->tx_stats.restart_queue; 419 } 420 } 421 422 return !!budget; 423 } 424 425 /** 426 * ixgbevf_rx_skb - Helper function to determine proper Rx method 427 * @q_vector: structure containing interrupt and ring information 428 * @skb: packet to send up 429 **/ 430 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, 431 struct sk_buff *skb) 432 { 433 napi_gro_receive(&q_vector->napi, skb); 434 } 435 436 #define IXGBE_RSS_L4_TYPES_MASK \ 437 ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ 438 (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ 439 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ 440 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) 441 442 static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring, 443 union ixgbe_adv_rx_desc *rx_desc, 444 struct sk_buff *skb) 445 { 446 u16 rss_type; 447 448 if (!(ring->netdev->features & NETIF_F_RXHASH)) 449 return; 450 451 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & 452 IXGBE_RXDADV_RSSTYPE_MASK; 453 454 if (!rss_type) 455 return; 456 457 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 458 (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? 459 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); 460 } 461 462 /** 463 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 464 * @ring: structure containig ring specific data 465 * @rx_desc: current Rx descriptor being processed 466 * @skb: skb currently being received and modified 467 **/ 468 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, 469 union ixgbe_adv_rx_desc *rx_desc, 470 struct sk_buff *skb) 471 { 472 skb_checksum_none_assert(skb); 473 474 /* Rx csum disabled */ 475 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 476 return; 477 478 /* if IP and error */ 479 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 480 ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 481 ring->rx_stats.csum_err++; 482 return; 483 } 484 485 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 486 return; 487 488 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 489 ring->rx_stats.csum_err++; 490 return; 491 } 492 493 /* It must be a TCP or UDP packet with a valid checksum */ 494 skb->ip_summed = CHECKSUM_UNNECESSARY; 495 } 496 497 /** 498 * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor 499 * @rx_ring: rx descriptor ring packet is being transacted on 500 * @rx_desc: pointer to the EOP Rx descriptor 501 * @skb: pointer to current skb being populated 502 * 503 * This function checks the ring, descriptor, and packet information in 504 * order to populate the checksum, VLAN, protocol, and other fields within 505 * the skb. 506 **/ 507 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, 508 union ixgbe_adv_rx_desc *rx_desc, 509 struct sk_buff *skb) 510 { 511 ixgbevf_rx_hash(rx_ring, rx_desc, skb); 512 ixgbevf_rx_checksum(rx_ring, rx_desc, skb); 513 514 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 515 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 516 unsigned long *active_vlans = netdev_priv(rx_ring->netdev); 517 518 if (test_bit(vid & VLAN_VID_MASK, active_vlans)) 519 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 520 } 521 522 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP)) 523 ixgbevf_ipsec_rx(rx_ring, rx_desc, skb); 524 525 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 526 } 527 528 static 529 struct ixgbevf_rx_buffer *ixgbevf_get_rx_buffer(struct ixgbevf_ring *rx_ring, 530 const unsigned int size) 531 { 532 struct ixgbevf_rx_buffer *rx_buffer; 533 534 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 535 prefetchw(rx_buffer->page); 536 537 /* we are reusing so sync this buffer for CPU use */ 538 dma_sync_single_range_for_cpu(rx_ring->dev, 539 rx_buffer->dma, 540 rx_buffer->page_offset, 541 size, 542 DMA_FROM_DEVICE); 543 544 rx_buffer->pagecnt_bias--; 545 546 return rx_buffer; 547 } 548 549 static void ixgbevf_put_rx_buffer(struct ixgbevf_ring *rx_ring, 550 struct ixgbevf_rx_buffer *rx_buffer, 551 struct sk_buff *skb) 552 { 553 if (ixgbevf_can_reuse_rx_page(rx_buffer)) { 554 /* hand second half of page back to the ring */ 555 ixgbevf_reuse_rx_page(rx_ring, rx_buffer); 556 } else { 557 if (IS_ERR(skb)) 558 /* We are not reusing the buffer so unmap it and free 559 * any references we are holding to it 560 */ 561 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 562 ixgbevf_rx_pg_size(rx_ring), 563 DMA_FROM_DEVICE, 564 IXGBEVF_RX_DMA_ATTR); 565 __page_frag_cache_drain(rx_buffer->page, 566 rx_buffer->pagecnt_bias); 567 } 568 569 /* clear contents of rx_buffer */ 570 rx_buffer->page = NULL; 571 } 572 573 /** 574 * ixgbevf_is_non_eop - process handling of non-EOP buffers 575 * @rx_ring: Rx ring being processed 576 * @rx_desc: Rx descriptor for current buffer 577 * 578 * This function updates next to clean. If the buffer is an EOP buffer 579 * this function exits returning false, otherwise it will place the 580 * sk_buff in the next buffer to be chained and return true indicating 581 * that this is in fact a non-EOP buffer. 582 **/ 583 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, 584 union ixgbe_adv_rx_desc *rx_desc) 585 { 586 u32 ntc = rx_ring->next_to_clean + 1; 587 588 /* fetch, update, and store next to clean */ 589 ntc = (ntc < rx_ring->count) ? ntc : 0; 590 rx_ring->next_to_clean = ntc; 591 592 prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); 593 594 if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 595 return false; 596 597 return true; 598 } 599 600 static inline unsigned int ixgbevf_rx_offset(struct ixgbevf_ring *rx_ring) 601 { 602 return ring_uses_build_skb(rx_ring) ? IXGBEVF_SKB_PAD : 0; 603 } 604 605 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, 606 struct ixgbevf_rx_buffer *bi) 607 { 608 struct page *page = bi->page; 609 dma_addr_t dma; 610 611 /* since we are recycling buffers we should seldom need to alloc */ 612 if (likely(page)) 613 return true; 614 615 /* alloc new page for storage */ 616 page = dev_alloc_pages(ixgbevf_rx_pg_order(rx_ring)); 617 if (unlikely(!page)) { 618 rx_ring->rx_stats.alloc_rx_page_failed++; 619 return false; 620 } 621 622 /* map page for use */ 623 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 624 ixgbevf_rx_pg_size(rx_ring), 625 DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR); 626 627 /* if mapping failed free memory back to system since 628 * there isn't much point in holding memory we can't use 629 */ 630 if (dma_mapping_error(rx_ring->dev, dma)) { 631 __free_pages(page, ixgbevf_rx_pg_order(rx_ring)); 632 633 rx_ring->rx_stats.alloc_rx_page_failed++; 634 return false; 635 } 636 637 bi->dma = dma; 638 bi->page = page; 639 bi->page_offset = ixgbevf_rx_offset(rx_ring); 640 bi->pagecnt_bias = 1; 641 rx_ring->rx_stats.alloc_rx_page++; 642 643 return true; 644 } 645 646 /** 647 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 648 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on 649 * @cleaned_count: number of buffers to replace 650 **/ 651 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, 652 u16 cleaned_count) 653 { 654 union ixgbe_adv_rx_desc *rx_desc; 655 struct ixgbevf_rx_buffer *bi; 656 unsigned int i = rx_ring->next_to_use; 657 658 /* nothing to do or no valid netdev defined */ 659 if (!cleaned_count || !rx_ring->netdev) 660 return; 661 662 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 663 bi = &rx_ring->rx_buffer_info[i]; 664 i -= rx_ring->count; 665 666 do { 667 if (!ixgbevf_alloc_mapped_page(rx_ring, bi)) 668 break; 669 670 /* sync the buffer for use by the device */ 671 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 672 bi->page_offset, 673 ixgbevf_rx_bufsz(rx_ring), 674 DMA_FROM_DEVICE); 675 676 /* Refresh the desc even if pkt_addr didn't change 677 * because each write-back erases this info. 678 */ 679 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 680 681 rx_desc++; 682 bi++; 683 i++; 684 if (unlikely(!i)) { 685 rx_desc = IXGBEVF_RX_DESC(rx_ring, 0); 686 bi = rx_ring->rx_buffer_info; 687 i -= rx_ring->count; 688 } 689 690 /* clear the length for the next_to_use descriptor */ 691 rx_desc->wb.upper.length = 0; 692 693 cleaned_count--; 694 } while (cleaned_count); 695 696 i += rx_ring->count; 697 698 if (rx_ring->next_to_use != i) { 699 /* record the next descriptor to use */ 700 rx_ring->next_to_use = i; 701 702 /* update next to alloc since we have filled the ring */ 703 rx_ring->next_to_alloc = i; 704 705 /* Force memory writes to complete before letting h/w 706 * know there are new descriptors to fetch. (Only 707 * applicable for weak-ordered memory model archs, 708 * such as IA-64). 709 */ 710 wmb(); 711 ixgbevf_write_tail(rx_ring, i); 712 } 713 } 714 715 /** 716 * ixgbevf_cleanup_headers - Correct corrupted or empty headers 717 * @rx_ring: rx descriptor ring packet is being transacted on 718 * @rx_desc: pointer to the EOP Rx descriptor 719 * @skb: pointer to current skb being fixed 720 * 721 * Check for corrupted packet headers caused by senders on the local L2 722 * embedded NIC switch not setting up their Tx Descriptors right. These 723 * should be very rare. 724 * 725 * Also address the case where we are pulling data in on pages only 726 * and as such no data is present in the skb header. 727 * 728 * In addition if skb is not at least 60 bytes we need to pad it so that 729 * it is large enough to qualify as a valid Ethernet frame. 730 * 731 * Returns true if an error was encountered and skb was freed. 732 **/ 733 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, 734 union ixgbe_adv_rx_desc *rx_desc, 735 struct sk_buff *skb) 736 { 737 /* XDP packets use error pointer so abort at this point */ 738 if (IS_ERR(skb)) 739 return true; 740 741 /* verify that the packet does not have any known errors */ 742 if (unlikely(ixgbevf_test_staterr(rx_desc, 743 IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { 744 struct net_device *netdev = rx_ring->netdev; 745 746 if (!(netdev->features & NETIF_F_RXALL)) { 747 dev_kfree_skb_any(skb); 748 return true; 749 } 750 } 751 752 /* if eth_skb_pad returns an error the skb was freed */ 753 if (eth_skb_pad(skb)) 754 return true; 755 756 return false; 757 } 758 759 /** 760 * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring 761 * @rx_ring: rx descriptor ring to store buffers on 762 * @old_buff: donor buffer to have page reused 763 * 764 * Synchronizes page for reuse by the adapter 765 **/ 766 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, 767 struct ixgbevf_rx_buffer *old_buff) 768 { 769 struct ixgbevf_rx_buffer *new_buff; 770 u16 nta = rx_ring->next_to_alloc; 771 772 new_buff = &rx_ring->rx_buffer_info[nta]; 773 774 /* update, and store next to alloc */ 775 nta++; 776 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 777 778 /* transfer page from old buffer to new buffer */ 779 new_buff->page = old_buff->page; 780 new_buff->dma = old_buff->dma; 781 new_buff->page_offset = old_buff->page_offset; 782 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 783 } 784 785 static inline bool ixgbevf_page_is_reserved(struct page *page) 786 { 787 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 788 } 789 790 static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer) 791 { 792 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 793 struct page *page = rx_buffer->page; 794 795 /* avoid re-using remote pages */ 796 if (unlikely(ixgbevf_page_is_reserved(page))) 797 return false; 798 799 #if (PAGE_SIZE < 8192) 800 /* if we are only owner of page we can reuse it */ 801 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 802 return false; 803 #else 804 #define IXGBEVF_LAST_OFFSET \ 805 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBEVF_RXBUFFER_2048) 806 807 if (rx_buffer->page_offset > IXGBEVF_LAST_OFFSET) 808 return false; 809 810 #endif 811 812 /* If we have drained the page fragment pool we need to update 813 * the pagecnt_bias and page count so that we fully restock the 814 * number of references the driver holds. 815 */ 816 if (unlikely(!pagecnt_bias)) { 817 page_ref_add(page, USHRT_MAX); 818 rx_buffer->pagecnt_bias = USHRT_MAX; 819 } 820 821 return true; 822 } 823 824 /** 825 * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff 826 * @rx_ring: rx descriptor ring to transact packets on 827 * @rx_buffer: buffer containing page to add 828 * @skb: sk_buff to place the data into 829 * @size: size of buffer to be added 830 * 831 * This function will add the data contained in rx_buffer->page to the skb. 832 **/ 833 static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, 834 struct ixgbevf_rx_buffer *rx_buffer, 835 struct sk_buff *skb, 836 unsigned int size) 837 { 838 #if (PAGE_SIZE < 8192) 839 unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; 840 #else 841 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 842 SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) : 843 SKB_DATA_ALIGN(size); 844 #endif 845 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 846 rx_buffer->page_offset, size, truesize); 847 #if (PAGE_SIZE < 8192) 848 rx_buffer->page_offset ^= truesize; 849 #else 850 rx_buffer->page_offset += truesize; 851 #endif 852 } 853 854 static 855 struct sk_buff *ixgbevf_construct_skb(struct ixgbevf_ring *rx_ring, 856 struct ixgbevf_rx_buffer *rx_buffer, 857 struct xdp_buff *xdp, 858 union ixgbe_adv_rx_desc *rx_desc) 859 { 860 unsigned int size = xdp->data_end - xdp->data; 861 #if (PAGE_SIZE < 8192) 862 unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; 863 #else 864 unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end - 865 xdp->data_hard_start); 866 #endif 867 unsigned int headlen; 868 struct sk_buff *skb; 869 870 /* prefetch first cache line of first page */ 871 prefetch(xdp->data); 872 #if L1_CACHE_BYTES < 128 873 prefetch(xdp->data + L1_CACHE_BYTES); 874 #endif 875 /* Note, we get here by enabling legacy-rx via: 876 * 877 * ethtool --set-priv-flags <dev> legacy-rx on 878 * 879 * In this mode, we currently get 0 extra XDP headroom as 880 * opposed to having legacy-rx off, where we process XDP 881 * packets going to stack via ixgbevf_build_skb(). 882 * 883 * For ixgbevf_construct_skb() mode it means that the 884 * xdp->data_meta will always point to xdp->data, since 885 * the helper cannot expand the head. Should this ever 886 * changed in future for legacy-rx mode on, then lets also 887 * add xdp->data_meta handling here. 888 */ 889 890 /* allocate a skb to store the frags */ 891 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBEVF_RX_HDR_SIZE); 892 if (unlikely(!skb)) 893 return NULL; 894 895 /* Determine available headroom for copy */ 896 headlen = size; 897 if (headlen > IXGBEVF_RX_HDR_SIZE) 898 headlen = eth_get_headlen(skb->dev, xdp->data, 899 IXGBEVF_RX_HDR_SIZE); 900 901 /* align pull length to size of long to optimize memcpy performance */ 902 memcpy(__skb_put(skb, headlen), xdp->data, 903 ALIGN(headlen, sizeof(long))); 904 905 /* update all of the pointers */ 906 size -= headlen; 907 if (size) { 908 skb_add_rx_frag(skb, 0, rx_buffer->page, 909 (xdp->data + headlen) - 910 page_address(rx_buffer->page), 911 size, truesize); 912 #if (PAGE_SIZE < 8192) 913 rx_buffer->page_offset ^= truesize; 914 #else 915 rx_buffer->page_offset += truesize; 916 #endif 917 } else { 918 rx_buffer->pagecnt_bias++; 919 } 920 921 return skb; 922 } 923 924 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 925 u32 qmask) 926 { 927 struct ixgbe_hw *hw = &adapter->hw; 928 929 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); 930 } 931 932 static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring, 933 struct ixgbevf_rx_buffer *rx_buffer, 934 struct xdp_buff *xdp, 935 union ixgbe_adv_rx_desc *rx_desc) 936 { 937 unsigned int metasize = xdp->data - xdp->data_meta; 938 #if (PAGE_SIZE < 8192) 939 unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; 940 #else 941 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 942 SKB_DATA_ALIGN(xdp->data_end - 943 xdp->data_hard_start); 944 #endif 945 struct sk_buff *skb; 946 947 /* Prefetch first cache line of first page. If xdp->data_meta 948 * is unused, this points to xdp->data, otherwise, we likely 949 * have a consumer accessing first few bytes of meta data, 950 * and then actual data. 951 */ 952 prefetch(xdp->data_meta); 953 #if L1_CACHE_BYTES < 128 954 prefetch(xdp->data_meta + L1_CACHE_BYTES); 955 #endif 956 957 /* build an skb around the page buffer */ 958 skb = build_skb(xdp->data_hard_start, truesize); 959 if (unlikely(!skb)) 960 return NULL; 961 962 /* update pointers within the skb to store the data */ 963 skb_reserve(skb, xdp->data - xdp->data_hard_start); 964 __skb_put(skb, xdp->data_end - xdp->data); 965 if (metasize) 966 skb_metadata_set(skb, metasize); 967 968 /* update buffer offset */ 969 #if (PAGE_SIZE < 8192) 970 rx_buffer->page_offset ^= truesize; 971 #else 972 rx_buffer->page_offset += truesize; 973 #endif 974 975 return skb; 976 } 977 978 #define IXGBEVF_XDP_PASS 0 979 #define IXGBEVF_XDP_CONSUMED 1 980 #define IXGBEVF_XDP_TX 2 981 982 static int ixgbevf_xmit_xdp_ring(struct ixgbevf_ring *ring, 983 struct xdp_buff *xdp) 984 { 985 struct ixgbevf_tx_buffer *tx_buffer; 986 union ixgbe_adv_tx_desc *tx_desc; 987 u32 len, cmd_type; 988 dma_addr_t dma; 989 u16 i; 990 991 len = xdp->data_end - xdp->data; 992 993 if (unlikely(!ixgbevf_desc_unused(ring))) 994 return IXGBEVF_XDP_CONSUMED; 995 996 dma = dma_map_single(ring->dev, xdp->data, len, DMA_TO_DEVICE); 997 if (dma_mapping_error(ring->dev, dma)) 998 return IXGBEVF_XDP_CONSUMED; 999 1000 /* record the location of the first descriptor for this packet */ 1001 i = ring->next_to_use; 1002 tx_buffer = &ring->tx_buffer_info[i]; 1003 1004 dma_unmap_len_set(tx_buffer, len, len); 1005 dma_unmap_addr_set(tx_buffer, dma, dma); 1006 tx_buffer->data = xdp->data; 1007 tx_buffer->bytecount = len; 1008 tx_buffer->gso_segs = 1; 1009 tx_buffer->protocol = 0; 1010 1011 /* Populate minimal context descriptor that will provide for the 1012 * fact that we are expected to process Ethernet frames. 1013 */ 1014 if (!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state)) { 1015 struct ixgbe_adv_tx_context_desc *context_desc; 1016 1017 set_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); 1018 1019 context_desc = IXGBEVF_TX_CTXTDESC(ring, 0); 1020 context_desc->vlan_macip_lens = 1021 cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT); 1022 context_desc->fceof_saidx = 0; 1023 context_desc->type_tucmd_mlhl = 1024 cpu_to_le32(IXGBE_TXD_CMD_DEXT | 1025 IXGBE_ADVTXD_DTYP_CTXT); 1026 context_desc->mss_l4len_idx = 0; 1027 1028 i = 1; 1029 } 1030 1031 /* put descriptor type bits */ 1032 cmd_type = IXGBE_ADVTXD_DTYP_DATA | 1033 IXGBE_ADVTXD_DCMD_DEXT | 1034 IXGBE_ADVTXD_DCMD_IFCS; 1035 cmd_type |= len | IXGBE_TXD_CMD; 1036 1037 tx_desc = IXGBEVF_TX_DESC(ring, i); 1038 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1039 1040 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1041 tx_desc->read.olinfo_status = 1042 cpu_to_le32((len << IXGBE_ADVTXD_PAYLEN_SHIFT) | 1043 IXGBE_ADVTXD_CC); 1044 1045 /* Avoid any potential race with cleanup */ 1046 smp_wmb(); 1047 1048 /* set next_to_watch value indicating a packet is present */ 1049 i++; 1050 if (i == ring->count) 1051 i = 0; 1052 1053 tx_buffer->next_to_watch = tx_desc; 1054 ring->next_to_use = i; 1055 1056 return IXGBEVF_XDP_TX; 1057 } 1058 1059 static struct sk_buff *ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, 1060 struct ixgbevf_ring *rx_ring, 1061 struct xdp_buff *xdp) 1062 { 1063 int result = IXGBEVF_XDP_PASS; 1064 struct ixgbevf_ring *xdp_ring; 1065 struct bpf_prog *xdp_prog; 1066 u32 act; 1067 1068 rcu_read_lock(); 1069 xdp_prog = READ_ONCE(rx_ring->xdp_prog); 1070 1071 if (!xdp_prog) 1072 goto xdp_out; 1073 1074 act = bpf_prog_run_xdp(xdp_prog, xdp); 1075 switch (act) { 1076 case XDP_PASS: 1077 break; 1078 case XDP_TX: 1079 xdp_ring = adapter->xdp_ring[rx_ring->queue_index]; 1080 result = ixgbevf_xmit_xdp_ring(xdp_ring, xdp); 1081 break; 1082 default: 1083 bpf_warn_invalid_xdp_action(act); 1084 /* fallthrough */ 1085 case XDP_ABORTED: 1086 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); 1087 /* fallthrough -- handle aborts by dropping packet */ 1088 case XDP_DROP: 1089 result = IXGBEVF_XDP_CONSUMED; 1090 break; 1091 } 1092 xdp_out: 1093 rcu_read_unlock(); 1094 return ERR_PTR(-result); 1095 } 1096 1097 static void ixgbevf_rx_buffer_flip(struct ixgbevf_ring *rx_ring, 1098 struct ixgbevf_rx_buffer *rx_buffer, 1099 unsigned int size) 1100 { 1101 #if (PAGE_SIZE < 8192) 1102 unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; 1103 1104 rx_buffer->page_offset ^= truesize; 1105 #else 1106 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1107 SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) : 1108 SKB_DATA_ALIGN(size); 1109 1110 rx_buffer->page_offset += truesize; 1111 #endif 1112 } 1113 1114 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 1115 struct ixgbevf_ring *rx_ring, 1116 int budget) 1117 { 1118 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1119 struct ixgbevf_adapter *adapter = q_vector->adapter; 1120 u16 cleaned_count = ixgbevf_desc_unused(rx_ring); 1121 struct sk_buff *skb = rx_ring->skb; 1122 bool xdp_xmit = false; 1123 struct xdp_buff xdp; 1124 1125 xdp.rxq = &rx_ring->xdp_rxq; 1126 1127 while (likely(total_rx_packets < budget)) { 1128 struct ixgbevf_rx_buffer *rx_buffer; 1129 union ixgbe_adv_rx_desc *rx_desc; 1130 unsigned int size; 1131 1132 /* return some buffers to hardware, one at a time is too slow */ 1133 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 1134 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count); 1135 cleaned_count = 0; 1136 } 1137 1138 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); 1139 size = le16_to_cpu(rx_desc->wb.upper.length); 1140 if (!size) 1141 break; 1142 1143 /* This memory barrier is needed to keep us from reading 1144 * any other fields out of the rx_desc until we know the 1145 * RXD_STAT_DD bit is set 1146 */ 1147 rmb(); 1148 1149 rx_buffer = ixgbevf_get_rx_buffer(rx_ring, size); 1150 1151 /* retrieve a buffer from the ring */ 1152 if (!skb) { 1153 xdp.data = page_address(rx_buffer->page) + 1154 rx_buffer->page_offset; 1155 xdp.data_meta = xdp.data; 1156 xdp.data_hard_start = xdp.data - 1157 ixgbevf_rx_offset(rx_ring); 1158 xdp.data_end = xdp.data + size; 1159 1160 skb = ixgbevf_run_xdp(adapter, rx_ring, &xdp); 1161 } 1162 1163 if (IS_ERR(skb)) { 1164 if (PTR_ERR(skb) == -IXGBEVF_XDP_TX) { 1165 xdp_xmit = true; 1166 ixgbevf_rx_buffer_flip(rx_ring, rx_buffer, 1167 size); 1168 } else { 1169 rx_buffer->pagecnt_bias++; 1170 } 1171 total_rx_packets++; 1172 total_rx_bytes += size; 1173 } else if (skb) { 1174 ixgbevf_add_rx_frag(rx_ring, rx_buffer, skb, size); 1175 } else if (ring_uses_build_skb(rx_ring)) { 1176 skb = ixgbevf_build_skb(rx_ring, rx_buffer, 1177 &xdp, rx_desc); 1178 } else { 1179 skb = ixgbevf_construct_skb(rx_ring, rx_buffer, 1180 &xdp, rx_desc); 1181 } 1182 1183 /* exit if we failed to retrieve a buffer */ 1184 if (!skb) { 1185 rx_ring->rx_stats.alloc_rx_buff_failed++; 1186 rx_buffer->pagecnt_bias++; 1187 break; 1188 } 1189 1190 ixgbevf_put_rx_buffer(rx_ring, rx_buffer, skb); 1191 cleaned_count++; 1192 1193 /* fetch next buffer in frame if non-eop */ 1194 if (ixgbevf_is_non_eop(rx_ring, rx_desc)) 1195 continue; 1196 1197 /* verify the packet layout is correct */ 1198 if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) { 1199 skb = NULL; 1200 continue; 1201 } 1202 1203 /* probably a little skewed due to removing CRC */ 1204 total_rx_bytes += skb->len; 1205 1206 /* Workaround hardware that can't do proper VEPA multicast 1207 * source pruning. 1208 */ 1209 if ((skb->pkt_type == PACKET_BROADCAST || 1210 skb->pkt_type == PACKET_MULTICAST) && 1211 ether_addr_equal(rx_ring->netdev->dev_addr, 1212 eth_hdr(skb)->h_source)) { 1213 dev_kfree_skb_irq(skb); 1214 continue; 1215 } 1216 1217 /* populate checksum, VLAN, and protocol */ 1218 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); 1219 1220 ixgbevf_rx_skb(q_vector, skb); 1221 1222 /* reset skb pointer */ 1223 skb = NULL; 1224 1225 /* update budget accounting */ 1226 total_rx_packets++; 1227 } 1228 1229 /* place incomplete frames back on ring for completion */ 1230 rx_ring->skb = skb; 1231 1232 if (xdp_xmit) { 1233 struct ixgbevf_ring *xdp_ring = 1234 adapter->xdp_ring[rx_ring->queue_index]; 1235 1236 /* Force memory writes to complete before letting h/w 1237 * know there are new descriptors to fetch. 1238 */ 1239 wmb(); 1240 ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); 1241 } 1242 1243 u64_stats_update_begin(&rx_ring->syncp); 1244 rx_ring->stats.packets += total_rx_packets; 1245 rx_ring->stats.bytes += total_rx_bytes; 1246 u64_stats_update_end(&rx_ring->syncp); 1247 q_vector->rx.total_packets += total_rx_packets; 1248 q_vector->rx.total_bytes += total_rx_bytes; 1249 1250 return total_rx_packets; 1251 } 1252 1253 /** 1254 * ixgbevf_poll - NAPI polling calback 1255 * @napi: napi struct with our devices info in it 1256 * @budget: amount of work driver is allowed to do this pass, in packets 1257 * 1258 * This function will clean more than one or more rings associated with a 1259 * q_vector. 1260 **/ 1261 static int ixgbevf_poll(struct napi_struct *napi, int budget) 1262 { 1263 struct ixgbevf_q_vector *q_vector = 1264 container_of(napi, struct ixgbevf_q_vector, napi); 1265 struct ixgbevf_adapter *adapter = q_vector->adapter; 1266 struct ixgbevf_ring *ring; 1267 int per_ring_budget, work_done = 0; 1268 bool clean_complete = true; 1269 1270 ixgbevf_for_each_ring(ring, q_vector->tx) { 1271 if (!ixgbevf_clean_tx_irq(q_vector, ring, budget)) 1272 clean_complete = false; 1273 } 1274 1275 if (budget <= 0) 1276 return budget; 1277 1278 /* attempt to distribute budget to each queue fairly, but don't allow 1279 * the budget to go below 1 because we'll exit polling 1280 */ 1281 if (q_vector->rx.count > 1) 1282 per_ring_budget = max(budget/q_vector->rx.count, 1); 1283 else 1284 per_ring_budget = budget; 1285 1286 ixgbevf_for_each_ring(ring, q_vector->rx) { 1287 int cleaned = ixgbevf_clean_rx_irq(q_vector, ring, 1288 per_ring_budget); 1289 work_done += cleaned; 1290 if (cleaned >= per_ring_budget) 1291 clean_complete = false; 1292 } 1293 1294 /* If all work not completed, return budget and keep polling */ 1295 if (!clean_complete) 1296 return budget; 1297 1298 /* Exit the polling mode, but don't re-enable interrupts if stack might 1299 * poll us due to busy-polling 1300 */ 1301 if (likely(napi_complete_done(napi, work_done))) { 1302 if (adapter->rx_itr_setting == 1) 1303 ixgbevf_set_itr(q_vector); 1304 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 1305 !test_bit(__IXGBEVF_REMOVING, &adapter->state)) 1306 ixgbevf_irq_enable_queues(adapter, 1307 BIT(q_vector->v_idx)); 1308 } 1309 1310 return min(work_done, budget - 1); 1311 } 1312 1313 /** 1314 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 1315 * @q_vector: structure containing interrupt and ring information 1316 **/ 1317 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) 1318 { 1319 struct ixgbevf_adapter *adapter = q_vector->adapter; 1320 struct ixgbe_hw *hw = &adapter->hw; 1321 int v_idx = q_vector->v_idx; 1322 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 1323 1324 /* set the WDIS bit to not clear the timer bits and cause an 1325 * immediate assertion of the interrupt 1326 */ 1327 itr_reg |= IXGBE_EITR_CNT_WDIS; 1328 1329 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 1330 } 1331 1332 /** 1333 * ixgbevf_configure_msix - Configure MSI-X hardware 1334 * @adapter: board private structure 1335 * 1336 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 1337 * interrupts. 1338 **/ 1339 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 1340 { 1341 struct ixgbevf_q_vector *q_vector; 1342 int q_vectors, v_idx; 1343 1344 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1345 adapter->eims_enable_mask = 0; 1346 1347 /* Populate the IVAR table and set the ITR values to the 1348 * corresponding register. 1349 */ 1350 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 1351 struct ixgbevf_ring *ring; 1352 1353 q_vector = adapter->q_vector[v_idx]; 1354 1355 ixgbevf_for_each_ring(ring, q_vector->rx) 1356 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx); 1357 1358 ixgbevf_for_each_ring(ring, q_vector->tx) 1359 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx); 1360 1361 if (q_vector->tx.ring && !q_vector->rx.ring) { 1362 /* Tx only vector */ 1363 if (adapter->tx_itr_setting == 1) 1364 q_vector->itr = IXGBE_12K_ITR; 1365 else 1366 q_vector->itr = adapter->tx_itr_setting; 1367 } else { 1368 /* Rx or Rx/Tx vector */ 1369 if (adapter->rx_itr_setting == 1) 1370 q_vector->itr = IXGBE_20K_ITR; 1371 else 1372 q_vector->itr = adapter->rx_itr_setting; 1373 } 1374 1375 /* add q_vector eims value to global eims_enable_mask */ 1376 adapter->eims_enable_mask |= BIT(v_idx); 1377 1378 ixgbevf_write_eitr(q_vector); 1379 } 1380 1381 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 1382 /* setup eims_other and add value to global eims_enable_mask */ 1383 adapter->eims_other = BIT(v_idx); 1384 adapter->eims_enable_mask |= adapter->eims_other; 1385 } 1386 1387 enum latency_range { 1388 lowest_latency = 0, 1389 low_latency = 1, 1390 bulk_latency = 2, 1391 latency_invalid = 255 1392 }; 1393 1394 /** 1395 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 1396 * @q_vector: structure containing interrupt and ring information 1397 * @ring_container: structure containing ring performance data 1398 * 1399 * Stores a new ITR value based on packets and byte 1400 * counts during the last interrupt. The advantage of per interrupt 1401 * computation is faster updates and more accurate ITR for the current 1402 * traffic pattern. Constants in this function were computed 1403 * based on theoretical maximum wire speed and thresholds were set based 1404 * on testing data as well as attempting to minimize response time 1405 * while increasing bulk throughput. 1406 **/ 1407 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector, 1408 struct ixgbevf_ring_container *ring_container) 1409 { 1410 int bytes = ring_container->total_bytes; 1411 int packets = ring_container->total_packets; 1412 u32 timepassed_us; 1413 u64 bytes_perint; 1414 u8 itr_setting = ring_container->itr; 1415 1416 if (packets == 0) 1417 return; 1418 1419 /* simple throttle rate management 1420 * 0-20MB/s lowest (100000 ints/s) 1421 * 20-100MB/s low (20000 ints/s) 1422 * 100-1249MB/s bulk (12000 ints/s) 1423 */ 1424 /* what was last interrupt timeslice? */ 1425 timepassed_us = q_vector->itr >> 2; 1426 if (timepassed_us == 0) 1427 return; 1428 1429 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 1430 1431 switch (itr_setting) { 1432 case lowest_latency: 1433 if (bytes_perint > 10) 1434 itr_setting = low_latency; 1435 break; 1436 case low_latency: 1437 if (bytes_perint > 20) 1438 itr_setting = bulk_latency; 1439 else if (bytes_perint <= 10) 1440 itr_setting = lowest_latency; 1441 break; 1442 case bulk_latency: 1443 if (bytes_perint <= 20) 1444 itr_setting = low_latency; 1445 break; 1446 } 1447 1448 /* clear work counters since we have the values we need */ 1449 ring_container->total_bytes = 0; 1450 ring_container->total_packets = 0; 1451 1452 /* write updated itr to ring container */ 1453 ring_container->itr = itr_setting; 1454 } 1455 1456 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector) 1457 { 1458 u32 new_itr = q_vector->itr; 1459 u8 current_itr; 1460 1461 ixgbevf_update_itr(q_vector, &q_vector->tx); 1462 ixgbevf_update_itr(q_vector, &q_vector->rx); 1463 1464 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 1465 1466 switch (current_itr) { 1467 /* counts and packets in update_itr are dependent on these numbers */ 1468 case lowest_latency: 1469 new_itr = IXGBE_100K_ITR; 1470 break; 1471 case low_latency: 1472 new_itr = IXGBE_20K_ITR; 1473 break; 1474 case bulk_latency: 1475 new_itr = IXGBE_12K_ITR; 1476 break; 1477 default: 1478 break; 1479 } 1480 1481 if (new_itr != q_vector->itr) { 1482 /* do an exponential smoothing */ 1483 new_itr = (10 * new_itr * q_vector->itr) / 1484 ((9 * new_itr) + q_vector->itr); 1485 1486 /* save the algorithm value here */ 1487 q_vector->itr = new_itr; 1488 1489 ixgbevf_write_eitr(q_vector); 1490 } 1491 } 1492 1493 static irqreturn_t ixgbevf_msix_other(int irq, void *data) 1494 { 1495 struct ixgbevf_adapter *adapter = data; 1496 struct ixgbe_hw *hw = &adapter->hw; 1497 1498 hw->mac.get_link_status = 1; 1499 1500 ixgbevf_service_event_schedule(adapter); 1501 1502 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); 1503 1504 return IRQ_HANDLED; 1505 } 1506 1507 /** 1508 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) 1509 * @irq: unused 1510 * @data: pointer to our q_vector struct for this interrupt vector 1511 **/ 1512 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data) 1513 { 1514 struct ixgbevf_q_vector *q_vector = data; 1515 1516 /* EIAM disabled interrupts (on this vector) for us */ 1517 if (q_vector->rx.ring || q_vector->tx.ring) 1518 napi_schedule_irqoff(&q_vector->napi); 1519 1520 return IRQ_HANDLED; 1521 } 1522 1523 /** 1524 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 1525 * @adapter: board private structure 1526 * 1527 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 1528 * interrupts from the kernel. 1529 **/ 1530 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 1531 { 1532 struct net_device *netdev = adapter->netdev; 1533 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1534 unsigned int ri = 0, ti = 0; 1535 int vector, err; 1536 1537 for (vector = 0; vector < q_vectors; vector++) { 1538 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector]; 1539 struct msix_entry *entry = &adapter->msix_entries[vector]; 1540 1541 if (q_vector->tx.ring && q_vector->rx.ring) { 1542 snprintf(q_vector->name, sizeof(q_vector->name), 1543 "%s-TxRx-%u", netdev->name, ri++); 1544 ti++; 1545 } else if (q_vector->rx.ring) { 1546 snprintf(q_vector->name, sizeof(q_vector->name), 1547 "%s-rx-%u", netdev->name, ri++); 1548 } else if (q_vector->tx.ring) { 1549 snprintf(q_vector->name, sizeof(q_vector->name), 1550 "%s-tx-%u", netdev->name, ti++); 1551 } else { 1552 /* skip this unused q_vector */ 1553 continue; 1554 } 1555 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0, 1556 q_vector->name, q_vector); 1557 if (err) { 1558 hw_dbg(&adapter->hw, 1559 "request_irq failed for MSIX interrupt Error: %d\n", 1560 err); 1561 goto free_queue_irqs; 1562 } 1563 } 1564 1565 err = request_irq(adapter->msix_entries[vector].vector, 1566 &ixgbevf_msix_other, 0, netdev->name, adapter); 1567 if (err) { 1568 hw_dbg(&adapter->hw, "request_irq for msix_other failed: %d\n", 1569 err); 1570 goto free_queue_irqs; 1571 } 1572 1573 return 0; 1574 1575 free_queue_irqs: 1576 while (vector) { 1577 vector--; 1578 free_irq(adapter->msix_entries[vector].vector, 1579 adapter->q_vector[vector]); 1580 } 1581 /* This failure is non-recoverable - it indicates the system is 1582 * out of MSIX vector resources and the VF driver cannot run 1583 * without them. Set the number of msix vectors to zero 1584 * indicating that not enough can be allocated. The error 1585 * will be returned to the user indicating device open failed. 1586 * Any further attempts to force the driver to open will also 1587 * fail. The only way to recover is to unload the driver and 1588 * reload it again. If the system has recovered some MSIX 1589 * vectors then it may succeed. 1590 */ 1591 adapter->num_msix_vectors = 0; 1592 return err; 1593 } 1594 1595 /** 1596 * ixgbevf_request_irq - initialize interrupts 1597 * @adapter: board private structure 1598 * 1599 * Attempts to configure interrupts using the best available 1600 * capabilities of the hardware and kernel. 1601 **/ 1602 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 1603 { 1604 int err = ixgbevf_request_msix_irqs(adapter); 1605 1606 if (err) 1607 hw_dbg(&adapter->hw, "request_irq failed, Error %d\n", err); 1608 1609 return err; 1610 } 1611 1612 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 1613 { 1614 int i, q_vectors; 1615 1616 if (!adapter->msix_entries) 1617 return; 1618 1619 q_vectors = adapter->num_msix_vectors; 1620 i = q_vectors - 1; 1621 1622 free_irq(adapter->msix_entries[i].vector, adapter); 1623 i--; 1624 1625 for (; i >= 0; i--) { 1626 /* free only the irqs that were actually requested */ 1627 if (!adapter->q_vector[i]->rx.ring && 1628 !adapter->q_vector[i]->tx.ring) 1629 continue; 1630 1631 free_irq(adapter->msix_entries[i].vector, 1632 adapter->q_vector[i]); 1633 } 1634 } 1635 1636 /** 1637 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1638 * @adapter: board private structure 1639 **/ 1640 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1641 { 1642 struct ixgbe_hw *hw = &adapter->hw; 1643 int i; 1644 1645 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0); 1646 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1647 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0); 1648 1649 IXGBE_WRITE_FLUSH(hw); 1650 1651 for (i = 0; i < adapter->num_msix_vectors; i++) 1652 synchronize_irq(adapter->msix_entries[i].vector); 1653 } 1654 1655 /** 1656 * ixgbevf_irq_enable - Enable default interrupt generation settings 1657 * @adapter: board private structure 1658 **/ 1659 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) 1660 { 1661 struct ixgbe_hw *hw = &adapter->hw; 1662 1663 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask); 1664 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask); 1665 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); 1666 } 1667 1668 /** 1669 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset 1670 * @adapter: board private structure 1671 * @ring: structure containing ring specific data 1672 * 1673 * Configure the Tx descriptor ring after a reset. 1674 **/ 1675 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 1676 struct ixgbevf_ring *ring) 1677 { 1678 struct ixgbe_hw *hw = &adapter->hw; 1679 u64 tdba = ring->dma; 1680 int wait_loop = 10; 1681 u32 txdctl = IXGBE_TXDCTL_ENABLE; 1682 u8 reg_idx = ring->reg_idx; 1683 1684 /* disable queue to avoid issues while updating state */ 1685 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 1686 IXGBE_WRITE_FLUSH(hw); 1687 1688 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 1689 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32); 1690 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx), 1691 ring->count * sizeof(union ixgbe_adv_tx_desc)); 1692 1693 /* disable head writeback */ 1694 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0); 1695 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0); 1696 1697 /* enable relaxed ordering */ 1698 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx), 1699 (IXGBE_DCA_TXCTRL_DESC_RRO_EN | 1700 IXGBE_DCA_TXCTRL_DATA_RRO_EN)); 1701 1702 /* reset head and tail pointers */ 1703 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0); 1704 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0); 1705 ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx); 1706 1707 /* reset ntu and ntc to place SW in sync with hardwdare */ 1708 ring->next_to_clean = 0; 1709 ring->next_to_use = 0; 1710 1711 /* In order to avoid issues WTHRESH + PTHRESH should always be equal 1712 * to or less than the number of on chip descriptors, which is 1713 * currently 40. 1714 */ 1715 txdctl |= (8 << 16); /* WTHRESH = 8 */ 1716 1717 /* Setting PTHRESH to 32 both improves performance */ 1718 txdctl |= (1u << 8) | /* HTHRESH = 1 */ 1719 32; /* PTHRESH = 32 */ 1720 1721 /* reinitialize tx_buffer_info */ 1722 memset(ring->tx_buffer_info, 0, 1723 sizeof(struct ixgbevf_tx_buffer) * ring->count); 1724 1725 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); 1726 clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); 1727 1728 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl); 1729 1730 /* poll to verify queue is enabled */ 1731 do { 1732 usleep_range(1000, 2000); 1733 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx)); 1734 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 1735 if (!wait_loop) 1736 hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx); 1737 } 1738 1739 /** 1740 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1741 * @adapter: board private structure 1742 * 1743 * Configure the Tx unit of the MAC after a reset. 1744 **/ 1745 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1746 { 1747 u32 i; 1748 1749 /* Setup the HW Tx Head and Tail descriptor pointers */ 1750 for (i = 0; i < adapter->num_tx_queues; i++) 1751 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]); 1752 for (i = 0; i < adapter->num_xdp_queues; i++) 1753 ixgbevf_configure_tx_ring(adapter, adapter->xdp_ring[i]); 1754 } 1755 1756 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1757 1758 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, 1759 struct ixgbevf_ring *ring, int index) 1760 { 1761 struct ixgbe_hw *hw = &adapter->hw; 1762 u32 srrctl; 1763 1764 srrctl = IXGBE_SRRCTL_DROP_EN; 1765 1766 srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; 1767 if (ring_uses_large_buffer(ring)) 1768 srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1769 else 1770 srrctl |= IXGBEVF_RXBUFFER_2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1771 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1772 1773 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1774 } 1775 1776 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) 1777 { 1778 struct ixgbe_hw *hw = &adapter->hw; 1779 1780 /* PSRTYPE must be initialized in 82599 */ 1781 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 1782 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR | 1783 IXGBE_PSRTYPE_L2HDR; 1784 1785 if (adapter->num_rx_queues > 1) 1786 psrtype |= BIT(29); 1787 1788 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1789 } 1790 1791 #define IXGBEVF_MAX_RX_DESC_POLL 10 1792 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, 1793 struct ixgbevf_ring *ring) 1794 { 1795 struct ixgbe_hw *hw = &adapter->hw; 1796 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1797 u32 rxdctl; 1798 u8 reg_idx = ring->reg_idx; 1799 1800 if (IXGBE_REMOVED(hw->hw_addr)) 1801 return; 1802 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1803 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1804 1805 /* write value back with RXDCTL.ENABLE bit cleared */ 1806 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1807 1808 /* the hardware may take up to 100us to really disable the Rx queue */ 1809 do { 1810 udelay(10); 1811 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1812 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 1813 1814 if (!wait_loop) 1815 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", 1816 reg_idx); 1817 } 1818 1819 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1820 struct ixgbevf_ring *ring) 1821 { 1822 struct ixgbe_hw *hw = &adapter->hw; 1823 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1824 u32 rxdctl; 1825 u8 reg_idx = ring->reg_idx; 1826 1827 if (IXGBE_REMOVED(hw->hw_addr)) 1828 return; 1829 do { 1830 usleep_range(1000, 2000); 1831 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1832 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 1833 1834 if (!wait_loop) 1835 pr_err("RXDCTL.ENABLE queue %d not set while polling\n", 1836 reg_idx); 1837 } 1838 1839 /** 1840 * ixgbevf_init_rss_key - Initialize adapter RSS key 1841 * @adapter: device handle 1842 * 1843 * Allocates and initializes the RSS key if it is not allocated. 1844 **/ 1845 static inline int ixgbevf_init_rss_key(struct ixgbevf_adapter *adapter) 1846 { 1847 u32 *rss_key; 1848 1849 if (!adapter->rss_key) { 1850 rss_key = kzalloc(IXGBEVF_RSS_HASH_KEY_SIZE, GFP_KERNEL); 1851 if (unlikely(!rss_key)) 1852 return -ENOMEM; 1853 1854 netdev_rss_key_fill(rss_key, IXGBEVF_RSS_HASH_KEY_SIZE); 1855 adapter->rss_key = rss_key; 1856 } 1857 1858 return 0; 1859 } 1860 1861 static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) 1862 { 1863 struct ixgbe_hw *hw = &adapter->hw; 1864 u32 vfmrqc = 0, vfreta = 0; 1865 u16 rss_i = adapter->num_rx_queues; 1866 u8 i, j; 1867 1868 /* Fill out hash function seeds */ 1869 for (i = 0; i < IXGBEVF_VFRSSRK_REGS; i++) 1870 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), *(adapter->rss_key + i)); 1871 1872 for (i = 0, j = 0; i < IXGBEVF_X550_VFRETA_SIZE; i++, j++) { 1873 if (j == rss_i) 1874 j = 0; 1875 1876 adapter->rss_indir_tbl[i] = j; 1877 1878 vfreta |= j << (i & 0x3) * 8; 1879 if ((i & 3) == 3) { 1880 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), vfreta); 1881 vfreta = 0; 1882 } 1883 } 1884 1885 /* Perform hash on these packet types */ 1886 vfmrqc |= IXGBE_VFMRQC_RSS_FIELD_IPV4 | 1887 IXGBE_VFMRQC_RSS_FIELD_IPV4_TCP | 1888 IXGBE_VFMRQC_RSS_FIELD_IPV6 | 1889 IXGBE_VFMRQC_RSS_FIELD_IPV6_TCP; 1890 1891 vfmrqc |= IXGBE_VFMRQC_RSSEN; 1892 1893 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); 1894 } 1895 1896 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, 1897 struct ixgbevf_ring *ring) 1898 { 1899 struct ixgbe_hw *hw = &adapter->hw; 1900 union ixgbe_adv_rx_desc *rx_desc; 1901 u64 rdba = ring->dma; 1902 u32 rxdctl; 1903 u8 reg_idx = ring->reg_idx; 1904 1905 /* disable queue to avoid issues while updating state */ 1906 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1907 ixgbevf_disable_rx_queue(adapter, ring); 1908 1909 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 1910 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32); 1911 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx), 1912 ring->count * sizeof(union ixgbe_adv_rx_desc)); 1913 1914 #ifndef CONFIG_SPARC 1915 /* enable relaxed ordering */ 1916 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1917 IXGBE_DCA_RXCTRL_DESC_RRO_EN); 1918 #else 1919 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1920 IXGBE_DCA_RXCTRL_DESC_RRO_EN | 1921 IXGBE_DCA_RXCTRL_DATA_WRO_EN); 1922 #endif 1923 1924 /* reset head and tail pointers */ 1925 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0); 1926 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); 1927 ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); 1928 1929 /* initialize rx_buffer_info */ 1930 memset(ring->rx_buffer_info, 0, 1931 sizeof(struct ixgbevf_rx_buffer) * ring->count); 1932 1933 /* initialize Rx descriptor 0 */ 1934 rx_desc = IXGBEVF_RX_DESC(ring, 0); 1935 rx_desc->wb.upper.length = 0; 1936 1937 /* reset ntu and ntc to place SW in sync with hardwdare */ 1938 ring->next_to_clean = 0; 1939 ring->next_to_use = 0; 1940 ring->next_to_alloc = 0; 1941 1942 ixgbevf_configure_srrctl(adapter, ring, reg_idx); 1943 1944 /* RXDCTL.RLPML does not work on 82599 */ 1945 if (adapter->hw.mac.type != ixgbe_mac_82599_vf) { 1946 rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | 1947 IXGBE_RXDCTL_RLPML_EN); 1948 1949 #if (PAGE_SIZE < 8192) 1950 /* Limit the maximum frame size so we don't overrun the skb */ 1951 if (ring_uses_build_skb(ring) && 1952 !ring_uses_large_buffer(ring)) 1953 rxdctl |= IXGBEVF_MAX_FRAME_BUILD_SKB | 1954 IXGBE_RXDCTL_RLPML_EN; 1955 #endif 1956 } 1957 1958 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1959 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1960 1961 ixgbevf_rx_desc_queue_enable(adapter, ring); 1962 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); 1963 } 1964 1965 static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter, 1966 struct ixgbevf_ring *rx_ring) 1967 { 1968 struct net_device *netdev = adapter->netdev; 1969 unsigned int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 1970 1971 /* set build_skb and buffer size flags */ 1972 clear_ring_build_skb_enabled(rx_ring); 1973 clear_ring_uses_large_buffer(rx_ring); 1974 1975 if (adapter->flags & IXGBEVF_FLAGS_LEGACY_RX) 1976 return; 1977 1978 set_ring_build_skb_enabled(rx_ring); 1979 1980 if (PAGE_SIZE < 8192) { 1981 if (max_frame <= IXGBEVF_MAX_FRAME_BUILD_SKB) 1982 return; 1983 1984 set_ring_uses_large_buffer(rx_ring); 1985 } 1986 } 1987 1988 /** 1989 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1990 * @adapter: board private structure 1991 * 1992 * Configure the Rx unit of the MAC after a reset. 1993 **/ 1994 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1995 { 1996 struct ixgbe_hw *hw = &adapter->hw; 1997 struct net_device *netdev = adapter->netdev; 1998 int i, ret; 1999 2000 ixgbevf_setup_psrtype(adapter); 2001 if (hw->mac.type >= ixgbe_mac_X550_vf) 2002 ixgbevf_setup_vfmrqc(adapter); 2003 2004 spin_lock_bh(&adapter->mbx_lock); 2005 /* notify the PF of our intent to use this size of frame */ 2006 ret = hw->mac.ops.set_rlpml(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN); 2007 spin_unlock_bh(&adapter->mbx_lock); 2008 if (ret) 2009 dev_err(&adapter->pdev->dev, 2010 "Failed to set MTU at %d\n", netdev->mtu); 2011 2012 /* Setup the HW Rx Head and Tail Descriptor Pointers and 2013 * the Base and Length of the Rx Descriptor Ring 2014 */ 2015 for (i = 0; i < adapter->num_rx_queues; i++) { 2016 struct ixgbevf_ring *rx_ring = adapter->rx_ring[i]; 2017 2018 ixgbevf_set_rx_buffer_len(adapter, rx_ring); 2019 ixgbevf_configure_rx_ring(adapter, rx_ring); 2020 } 2021 } 2022 2023 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, 2024 __be16 proto, u16 vid) 2025 { 2026 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2027 struct ixgbe_hw *hw = &adapter->hw; 2028 int err; 2029 2030 spin_lock_bh(&adapter->mbx_lock); 2031 2032 /* add VID to filter table */ 2033 err = hw->mac.ops.set_vfta(hw, vid, 0, true); 2034 2035 spin_unlock_bh(&adapter->mbx_lock); 2036 2037 /* translate error return types so error makes sense */ 2038 if (err == IXGBE_ERR_MBX) 2039 return -EIO; 2040 2041 if (err == IXGBE_ERR_INVALID_ARGUMENT) 2042 return -EACCES; 2043 2044 set_bit(vid, adapter->active_vlans); 2045 2046 return err; 2047 } 2048 2049 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, 2050 __be16 proto, u16 vid) 2051 { 2052 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2053 struct ixgbe_hw *hw = &adapter->hw; 2054 int err; 2055 2056 spin_lock_bh(&adapter->mbx_lock); 2057 2058 /* remove VID from filter table */ 2059 err = hw->mac.ops.set_vfta(hw, vid, 0, false); 2060 2061 spin_unlock_bh(&adapter->mbx_lock); 2062 2063 clear_bit(vid, adapter->active_vlans); 2064 2065 return err; 2066 } 2067 2068 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 2069 { 2070 u16 vid; 2071 2072 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2073 ixgbevf_vlan_rx_add_vid(adapter->netdev, 2074 htons(ETH_P_8021Q), vid); 2075 } 2076 2077 static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 2078 { 2079 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2080 struct ixgbe_hw *hw = &adapter->hw; 2081 int count = 0; 2082 2083 if ((netdev_uc_count(netdev)) > 10) { 2084 pr_err("Too many unicast filters - No Space\n"); 2085 return -ENOSPC; 2086 } 2087 2088 if (!netdev_uc_empty(netdev)) { 2089 struct netdev_hw_addr *ha; 2090 2091 netdev_for_each_uc_addr(ha, netdev) { 2092 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 2093 udelay(200); 2094 } 2095 } else { 2096 /* If the list is empty then send message to PF driver to 2097 * clear all MAC VLANs on this VF. 2098 */ 2099 hw->mac.ops.set_uc_addr(hw, 0, NULL); 2100 } 2101 2102 return count; 2103 } 2104 2105 /** 2106 * ixgbevf_set_rx_mode - Multicast and unicast set 2107 * @netdev: network interface device structure 2108 * 2109 * The set_rx_method entry point is called whenever the multicast address 2110 * list, unicast address list or the network interface flags are updated. 2111 * This routine is responsible for configuring the hardware for proper 2112 * multicast mode and configuring requested unicast filters. 2113 **/ 2114 static void ixgbevf_set_rx_mode(struct net_device *netdev) 2115 { 2116 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2117 struct ixgbe_hw *hw = &adapter->hw; 2118 unsigned int flags = netdev->flags; 2119 int xcast_mode; 2120 2121 /* request the most inclusive mode we need */ 2122 if (flags & IFF_PROMISC) 2123 xcast_mode = IXGBEVF_XCAST_MODE_PROMISC; 2124 else if (flags & IFF_ALLMULTI) 2125 xcast_mode = IXGBEVF_XCAST_MODE_ALLMULTI; 2126 else if (flags & (IFF_BROADCAST | IFF_MULTICAST)) 2127 xcast_mode = IXGBEVF_XCAST_MODE_MULTI; 2128 else 2129 xcast_mode = IXGBEVF_XCAST_MODE_NONE; 2130 2131 spin_lock_bh(&adapter->mbx_lock); 2132 2133 hw->mac.ops.update_xcast_mode(hw, xcast_mode); 2134 2135 /* reprogram multicast list */ 2136 hw->mac.ops.update_mc_addr_list(hw, netdev); 2137 2138 ixgbevf_write_uc_addr_list(netdev); 2139 2140 spin_unlock_bh(&adapter->mbx_lock); 2141 } 2142 2143 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 2144 { 2145 int q_idx; 2146 struct ixgbevf_q_vector *q_vector; 2147 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2148 2149 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 2150 q_vector = adapter->q_vector[q_idx]; 2151 napi_enable(&q_vector->napi); 2152 } 2153 } 2154 2155 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 2156 { 2157 int q_idx; 2158 struct ixgbevf_q_vector *q_vector; 2159 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2160 2161 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 2162 q_vector = adapter->q_vector[q_idx]; 2163 napi_disable(&q_vector->napi); 2164 } 2165 } 2166 2167 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter) 2168 { 2169 struct ixgbe_hw *hw = &adapter->hw; 2170 unsigned int def_q = 0; 2171 unsigned int num_tcs = 0; 2172 unsigned int num_rx_queues = adapter->num_rx_queues; 2173 unsigned int num_tx_queues = adapter->num_tx_queues; 2174 int err; 2175 2176 spin_lock_bh(&adapter->mbx_lock); 2177 2178 /* fetch queue configuration from the PF */ 2179 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2180 2181 spin_unlock_bh(&adapter->mbx_lock); 2182 2183 if (err) 2184 return err; 2185 2186 if (num_tcs > 1) { 2187 /* we need only one Tx queue */ 2188 num_tx_queues = 1; 2189 2190 /* update default Tx ring register index */ 2191 adapter->tx_ring[0]->reg_idx = def_q; 2192 2193 /* we need as many queues as traffic classes */ 2194 num_rx_queues = num_tcs; 2195 } 2196 2197 /* if we have a bad config abort request queue reset */ 2198 if ((adapter->num_rx_queues != num_rx_queues) || 2199 (adapter->num_tx_queues != num_tx_queues)) { 2200 /* force mailbox timeout to prevent further messages */ 2201 hw->mbx.timeout = 0; 2202 2203 /* wait for watchdog to come around and bail us out */ 2204 set_bit(__IXGBEVF_QUEUE_RESET_REQUESTED, &adapter->state); 2205 } 2206 2207 return 0; 2208 } 2209 2210 static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 2211 { 2212 ixgbevf_configure_dcb(adapter); 2213 2214 ixgbevf_set_rx_mode(adapter->netdev); 2215 2216 ixgbevf_restore_vlan(adapter); 2217 ixgbevf_ipsec_restore(adapter); 2218 2219 ixgbevf_configure_tx(adapter); 2220 ixgbevf_configure_rx(adapter); 2221 } 2222 2223 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 2224 { 2225 /* Only save pre-reset stats if there are some */ 2226 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 2227 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 2228 adapter->stats.base_vfgprc; 2229 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 2230 adapter->stats.base_vfgptc; 2231 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 2232 adapter->stats.base_vfgorc; 2233 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 2234 adapter->stats.base_vfgotc; 2235 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 2236 adapter->stats.base_vfmprc; 2237 } 2238 } 2239 2240 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 2241 { 2242 struct ixgbe_hw *hw = &adapter->hw; 2243 2244 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2245 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2246 adapter->stats.last_vfgorc |= 2247 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2248 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2249 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2250 adapter->stats.last_vfgotc |= 2251 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2252 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2253 2254 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 2255 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 2256 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 2257 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 2258 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 2259 } 2260 2261 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) 2262 { 2263 struct ixgbe_hw *hw = &adapter->hw; 2264 int api[] = { ixgbe_mbox_api_14, 2265 ixgbe_mbox_api_13, 2266 ixgbe_mbox_api_12, 2267 ixgbe_mbox_api_11, 2268 ixgbe_mbox_api_10, 2269 ixgbe_mbox_api_unknown }; 2270 int err, idx = 0; 2271 2272 spin_lock_bh(&adapter->mbx_lock); 2273 2274 while (api[idx] != ixgbe_mbox_api_unknown) { 2275 err = hw->mac.ops.negotiate_api_version(hw, api[idx]); 2276 if (!err) 2277 break; 2278 idx++; 2279 } 2280 2281 spin_unlock_bh(&adapter->mbx_lock); 2282 } 2283 2284 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 2285 { 2286 struct net_device *netdev = adapter->netdev; 2287 struct ixgbe_hw *hw = &adapter->hw; 2288 2289 ixgbevf_configure_msix(adapter); 2290 2291 spin_lock_bh(&adapter->mbx_lock); 2292 2293 if (is_valid_ether_addr(hw->mac.addr)) 2294 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 2295 else 2296 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 2297 2298 spin_unlock_bh(&adapter->mbx_lock); 2299 2300 smp_mb__before_atomic(); 2301 clear_bit(__IXGBEVF_DOWN, &adapter->state); 2302 ixgbevf_napi_enable_all(adapter); 2303 2304 /* clear any pending interrupts, may auto mask */ 2305 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2306 ixgbevf_irq_enable(adapter); 2307 2308 /* enable transmits */ 2309 netif_tx_start_all_queues(netdev); 2310 2311 ixgbevf_save_reset_stats(adapter); 2312 ixgbevf_init_last_counter_stats(adapter); 2313 2314 hw->mac.get_link_status = 1; 2315 mod_timer(&adapter->service_timer, jiffies); 2316 } 2317 2318 void ixgbevf_up(struct ixgbevf_adapter *adapter) 2319 { 2320 ixgbevf_configure(adapter); 2321 2322 ixgbevf_up_complete(adapter); 2323 } 2324 2325 /** 2326 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 2327 * @rx_ring: ring to free buffers from 2328 **/ 2329 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) 2330 { 2331 u16 i = rx_ring->next_to_clean; 2332 2333 /* Free Rx ring sk_buff */ 2334 if (rx_ring->skb) { 2335 dev_kfree_skb(rx_ring->skb); 2336 rx_ring->skb = NULL; 2337 } 2338 2339 /* Free all the Rx ring pages */ 2340 while (i != rx_ring->next_to_alloc) { 2341 struct ixgbevf_rx_buffer *rx_buffer; 2342 2343 rx_buffer = &rx_ring->rx_buffer_info[i]; 2344 2345 /* Invalidate cache lines that may have been written to by 2346 * device so that we avoid corrupting memory. 2347 */ 2348 dma_sync_single_range_for_cpu(rx_ring->dev, 2349 rx_buffer->dma, 2350 rx_buffer->page_offset, 2351 ixgbevf_rx_bufsz(rx_ring), 2352 DMA_FROM_DEVICE); 2353 2354 /* free resources associated with mapping */ 2355 dma_unmap_page_attrs(rx_ring->dev, 2356 rx_buffer->dma, 2357 ixgbevf_rx_pg_size(rx_ring), 2358 DMA_FROM_DEVICE, 2359 IXGBEVF_RX_DMA_ATTR); 2360 2361 __page_frag_cache_drain(rx_buffer->page, 2362 rx_buffer->pagecnt_bias); 2363 2364 i++; 2365 if (i == rx_ring->count) 2366 i = 0; 2367 } 2368 2369 rx_ring->next_to_alloc = 0; 2370 rx_ring->next_to_clean = 0; 2371 rx_ring->next_to_use = 0; 2372 } 2373 2374 /** 2375 * ixgbevf_clean_tx_ring - Free Tx Buffers 2376 * @tx_ring: ring to be cleaned 2377 **/ 2378 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) 2379 { 2380 u16 i = tx_ring->next_to_clean; 2381 struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 2382 2383 while (i != tx_ring->next_to_use) { 2384 union ixgbe_adv_tx_desc *eop_desc, *tx_desc; 2385 2386 /* Free all the Tx ring sk_buffs */ 2387 if (ring_is_xdp(tx_ring)) 2388 page_frag_free(tx_buffer->data); 2389 else 2390 dev_kfree_skb_any(tx_buffer->skb); 2391 2392 /* unmap skb header data */ 2393 dma_unmap_single(tx_ring->dev, 2394 dma_unmap_addr(tx_buffer, dma), 2395 dma_unmap_len(tx_buffer, len), 2396 DMA_TO_DEVICE); 2397 2398 /* check for eop_desc to determine the end of the packet */ 2399 eop_desc = tx_buffer->next_to_watch; 2400 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 2401 2402 /* unmap remaining buffers */ 2403 while (tx_desc != eop_desc) { 2404 tx_buffer++; 2405 tx_desc++; 2406 i++; 2407 if (unlikely(i == tx_ring->count)) { 2408 i = 0; 2409 tx_buffer = tx_ring->tx_buffer_info; 2410 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 2411 } 2412 2413 /* unmap any remaining paged data */ 2414 if (dma_unmap_len(tx_buffer, len)) 2415 dma_unmap_page(tx_ring->dev, 2416 dma_unmap_addr(tx_buffer, dma), 2417 dma_unmap_len(tx_buffer, len), 2418 DMA_TO_DEVICE); 2419 } 2420 2421 /* move us one more past the eop_desc for start of next pkt */ 2422 tx_buffer++; 2423 i++; 2424 if (unlikely(i == tx_ring->count)) { 2425 i = 0; 2426 tx_buffer = tx_ring->tx_buffer_info; 2427 } 2428 } 2429 2430 /* reset next_to_use and next_to_clean */ 2431 tx_ring->next_to_use = 0; 2432 tx_ring->next_to_clean = 0; 2433 2434 } 2435 2436 /** 2437 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 2438 * @adapter: board private structure 2439 **/ 2440 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 2441 { 2442 int i; 2443 2444 for (i = 0; i < adapter->num_rx_queues; i++) 2445 ixgbevf_clean_rx_ring(adapter->rx_ring[i]); 2446 } 2447 2448 /** 2449 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 2450 * @adapter: board private structure 2451 **/ 2452 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 2453 { 2454 int i; 2455 2456 for (i = 0; i < adapter->num_tx_queues; i++) 2457 ixgbevf_clean_tx_ring(adapter->tx_ring[i]); 2458 for (i = 0; i < adapter->num_xdp_queues; i++) 2459 ixgbevf_clean_tx_ring(adapter->xdp_ring[i]); 2460 } 2461 2462 void ixgbevf_down(struct ixgbevf_adapter *adapter) 2463 { 2464 struct net_device *netdev = adapter->netdev; 2465 struct ixgbe_hw *hw = &adapter->hw; 2466 int i; 2467 2468 /* signal that we are down to the interrupt handler */ 2469 if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state)) 2470 return; /* do nothing if already down */ 2471 2472 /* disable all enabled Rx queues */ 2473 for (i = 0; i < adapter->num_rx_queues; i++) 2474 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]); 2475 2476 usleep_range(10000, 20000); 2477 2478 netif_tx_stop_all_queues(netdev); 2479 2480 /* call carrier off first to avoid false dev_watchdog timeouts */ 2481 netif_carrier_off(netdev); 2482 netif_tx_disable(netdev); 2483 2484 ixgbevf_irq_disable(adapter); 2485 2486 ixgbevf_napi_disable_all(adapter); 2487 2488 del_timer_sync(&adapter->service_timer); 2489 2490 /* disable transmits in the hardware now that interrupts are off */ 2491 for (i = 0; i < adapter->num_tx_queues; i++) { 2492 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 2493 2494 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), 2495 IXGBE_TXDCTL_SWFLSH); 2496 } 2497 2498 for (i = 0; i < adapter->num_xdp_queues; i++) { 2499 u8 reg_idx = adapter->xdp_ring[i]->reg_idx; 2500 2501 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), 2502 IXGBE_TXDCTL_SWFLSH); 2503 } 2504 2505 if (!pci_channel_offline(adapter->pdev)) 2506 ixgbevf_reset(adapter); 2507 2508 ixgbevf_clean_all_tx_rings(adapter); 2509 ixgbevf_clean_all_rx_rings(adapter); 2510 } 2511 2512 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 2513 { 2514 WARN_ON(in_interrupt()); 2515 2516 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 2517 msleep(1); 2518 2519 ixgbevf_down(adapter); 2520 ixgbevf_up(adapter); 2521 2522 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 2523 } 2524 2525 void ixgbevf_reset(struct ixgbevf_adapter *adapter) 2526 { 2527 struct ixgbe_hw *hw = &adapter->hw; 2528 struct net_device *netdev = adapter->netdev; 2529 2530 if (hw->mac.ops.reset_hw(hw)) { 2531 hw_dbg(hw, "PF still resetting\n"); 2532 } else { 2533 hw->mac.ops.init_hw(hw); 2534 ixgbevf_negotiate_api(adapter); 2535 } 2536 2537 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 2538 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr); 2539 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr); 2540 } 2541 2542 adapter->last_reset = jiffies; 2543 } 2544 2545 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 2546 int vectors) 2547 { 2548 int vector_threshold; 2549 2550 /* We'll want at least 2 (vector_threshold): 2551 * 1) TxQ[0] + RxQ[0] handler 2552 * 2) Other (Link Status Change, etc.) 2553 */ 2554 vector_threshold = MIN_MSIX_COUNT; 2555 2556 /* The more we get, the more we will assign to Tx/Rx Cleanup 2557 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 2558 * Right now, we simply care about how many we'll get; we'll 2559 * set them up later while requesting irq's. 2560 */ 2561 vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 2562 vector_threshold, vectors); 2563 2564 if (vectors < 0) { 2565 dev_err(&adapter->pdev->dev, 2566 "Unable to allocate MSI-X interrupts\n"); 2567 kfree(adapter->msix_entries); 2568 adapter->msix_entries = NULL; 2569 return vectors; 2570 } 2571 2572 /* Adjust for only the vectors we'll use, which is minimum 2573 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 2574 * vectors we were allocated. 2575 */ 2576 adapter->num_msix_vectors = vectors; 2577 2578 return 0; 2579 } 2580 2581 /** 2582 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent 2583 * @adapter: board private structure to initialize 2584 * 2585 * This is the top level queue allocation routine. The order here is very 2586 * important, starting with the "most" number of features turned on at once, 2587 * and ending with the smallest set of features. This way large combinations 2588 * can be allocated if they're turned on, and smaller combinations are the 2589 * fallthrough conditions. 2590 * 2591 **/ 2592 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 2593 { 2594 struct ixgbe_hw *hw = &adapter->hw; 2595 unsigned int def_q = 0; 2596 unsigned int num_tcs = 0; 2597 int err; 2598 2599 /* Start with base case */ 2600 adapter->num_rx_queues = 1; 2601 adapter->num_tx_queues = 1; 2602 adapter->num_xdp_queues = 0; 2603 2604 spin_lock_bh(&adapter->mbx_lock); 2605 2606 /* fetch queue configuration from the PF */ 2607 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2608 2609 spin_unlock_bh(&adapter->mbx_lock); 2610 2611 if (err) 2612 return; 2613 2614 /* we need as many queues as traffic classes */ 2615 if (num_tcs > 1) { 2616 adapter->num_rx_queues = num_tcs; 2617 } else { 2618 u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES); 2619 2620 switch (hw->api_version) { 2621 case ixgbe_mbox_api_11: 2622 case ixgbe_mbox_api_12: 2623 case ixgbe_mbox_api_13: 2624 case ixgbe_mbox_api_14: 2625 if (adapter->xdp_prog && 2626 hw->mac.max_tx_queues == rss) 2627 rss = rss > 3 ? 2 : 1; 2628 2629 adapter->num_rx_queues = rss; 2630 adapter->num_tx_queues = rss; 2631 adapter->num_xdp_queues = adapter->xdp_prog ? rss : 0; 2632 default: 2633 break; 2634 } 2635 } 2636 } 2637 2638 /** 2639 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 2640 * @adapter: board private structure to initialize 2641 * 2642 * Attempt to configure the interrupts using the best available 2643 * capabilities of the hardware and the kernel. 2644 **/ 2645 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 2646 { 2647 int vector, v_budget; 2648 2649 /* It's easy to be greedy for MSI-X vectors, but it really 2650 * doesn't do us much good if we have a lot more vectors 2651 * than CPU's. So let's be conservative and only ask for 2652 * (roughly) the same number of vectors as there are CPU's. 2653 * The default is to use pairs of vectors. 2654 */ 2655 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); 2656 v_budget = min_t(int, v_budget, num_online_cpus()); 2657 v_budget += NON_Q_VECTORS; 2658 2659 adapter->msix_entries = kcalloc(v_budget, 2660 sizeof(struct msix_entry), GFP_KERNEL); 2661 if (!adapter->msix_entries) 2662 return -ENOMEM; 2663 2664 for (vector = 0; vector < v_budget; vector++) 2665 adapter->msix_entries[vector].entry = vector; 2666 2667 /* A failure in MSI-X entry allocation isn't fatal, but the VF driver 2668 * does not support any other modes, so we will simply fail here. Note 2669 * that we clean up the msix_entries pointer else-where. 2670 */ 2671 return ixgbevf_acquire_msix_vectors(adapter, v_budget); 2672 } 2673 2674 static void ixgbevf_add_ring(struct ixgbevf_ring *ring, 2675 struct ixgbevf_ring_container *head) 2676 { 2677 ring->next = head->ring; 2678 head->ring = ring; 2679 head->count++; 2680 } 2681 2682 /** 2683 * ixgbevf_alloc_q_vector - Allocate memory for a single interrupt vector 2684 * @adapter: board private structure to initialize 2685 * @v_idx: index of vector in adapter struct 2686 * @txr_count: number of Tx rings for q vector 2687 * @txr_idx: index of first Tx ring to assign 2688 * @xdp_count: total number of XDP rings to allocate 2689 * @xdp_idx: index of first XDP ring to allocate 2690 * @rxr_count: number of Rx rings for q vector 2691 * @rxr_idx: index of first Rx ring to assign 2692 * 2693 * We allocate one q_vector. If allocation fails we return -ENOMEM. 2694 **/ 2695 static int ixgbevf_alloc_q_vector(struct ixgbevf_adapter *adapter, int v_idx, 2696 int txr_count, int txr_idx, 2697 int xdp_count, int xdp_idx, 2698 int rxr_count, int rxr_idx) 2699 { 2700 struct ixgbevf_q_vector *q_vector; 2701 int reg_idx = txr_idx + xdp_idx; 2702 struct ixgbevf_ring *ring; 2703 int ring_count, size; 2704 2705 ring_count = txr_count + xdp_count + rxr_count; 2706 size = sizeof(*q_vector) + (sizeof(*ring) * ring_count); 2707 2708 /* allocate q_vector and rings */ 2709 q_vector = kzalloc(size, GFP_KERNEL); 2710 if (!q_vector) 2711 return -ENOMEM; 2712 2713 /* initialize NAPI */ 2714 netif_napi_add(adapter->netdev, &q_vector->napi, ixgbevf_poll, 64); 2715 2716 /* tie q_vector and adapter together */ 2717 adapter->q_vector[v_idx] = q_vector; 2718 q_vector->adapter = adapter; 2719 q_vector->v_idx = v_idx; 2720 2721 /* initialize pointer to rings */ 2722 ring = q_vector->ring; 2723 2724 while (txr_count) { 2725 /* assign generic ring traits */ 2726 ring->dev = &adapter->pdev->dev; 2727 ring->netdev = adapter->netdev; 2728 2729 /* configure backlink on ring */ 2730 ring->q_vector = q_vector; 2731 2732 /* update q_vector Tx values */ 2733 ixgbevf_add_ring(ring, &q_vector->tx); 2734 2735 /* apply Tx specific ring traits */ 2736 ring->count = adapter->tx_ring_count; 2737 ring->queue_index = txr_idx; 2738 ring->reg_idx = reg_idx; 2739 2740 /* assign ring to adapter */ 2741 adapter->tx_ring[txr_idx] = ring; 2742 2743 /* update count and index */ 2744 txr_count--; 2745 txr_idx++; 2746 reg_idx++; 2747 2748 /* push pointer to next ring */ 2749 ring++; 2750 } 2751 2752 while (xdp_count) { 2753 /* assign generic ring traits */ 2754 ring->dev = &adapter->pdev->dev; 2755 ring->netdev = adapter->netdev; 2756 2757 /* configure backlink on ring */ 2758 ring->q_vector = q_vector; 2759 2760 /* update q_vector Tx values */ 2761 ixgbevf_add_ring(ring, &q_vector->tx); 2762 2763 /* apply Tx specific ring traits */ 2764 ring->count = adapter->tx_ring_count; 2765 ring->queue_index = xdp_idx; 2766 ring->reg_idx = reg_idx; 2767 set_ring_xdp(ring); 2768 2769 /* assign ring to adapter */ 2770 adapter->xdp_ring[xdp_idx] = ring; 2771 2772 /* update count and index */ 2773 xdp_count--; 2774 xdp_idx++; 2775 reg_idx++; 2776 2777 /* push pointer to next ring */ 2778 ring++; 2779 } 2780 2781 while (rxr_count) { 2782 /* assign generic ring traits */ 2783 ring->dev = &adapter->pdev->dev; 2784 ring->netdev = adapter->netdev; 2785 2786 /* configure backlink on ring */ 2787 ring->q_vector = q_vector; 2788 2789 /* update q_vector Rx values */ 2790 ixgbevf_add_ring(ring, &q_vector->rx); 2791 2792 /* apply Rx specific ring traits */ 2793 ring->count = adapter->rx_ring_count; 2794 ring->queue_index = rxr_idx; 2795 ring->reg_idx = rxr_idx; 2796 2797 /* assign ring to adapter */ 2798 adapter->rx_ring[rxr_idx] = ring; 2799 2800 /* update count and index */ 2801 rxr_count--; 2802 rxr_idx++; 2803 2804 /* push pointer to next ring */ 2805 ring++; 2806 } 2807 2808 return 0; 2809 } 2810 2811 /** 2812 * ixgbevf_free_q_vector - Free memory allocated for specific interrupt vector 2813 * @adapter: board private structure to initialize 2814 * @v_idx: index of vector in adapter struct 2815 * 2816 * This function frees the memory allocated to the q_vector. In addition if 2817 * NAPI is enabled it will delete any references to the NAPI struct prior 2818 * to freeing the q_vector. 2819 **/ 2820 static void ixgbevf_free_q_vector(struct ixgbevf_adapter *adapter, int v_idx) 2821 { 2822 struct ixgbevf_q_vector *q_vector = adapter->q_vector[v_idx]; 2823 struct ixgbevf_ring *ring; 2824 2825 ixgbevf_for_each_ring(ring, q_vector->tx) { 2826 if (ring_is_xdp(ring)) 2827 adapter->xdp_ring[ring->queue_index] = NULL; 2828 else 2829 adapter->tx_ring[ring->queue_index] = NULL; 2830 } 2831 2832 ixgbevf_for_each_ring(ring, q_vector->rx) 2833 adapter->rx_ring[ring->queue_index] = NULL; 2834 2835 adapter->q_vector[v_idx] = NULL; 2836 netif_napi_del(&q_vector->napi); 2837 2838 /* ixgbevf_get_stats() might access the rings on this vector, 2839 * we must wait a grace period before freeing it. 2840 */ 2841 kfree_rcu(q_vector, rcu); 2842 } 2843 2844 /** 2845 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 2846 * @adapter: board private structure to initialize 2847 * 2848 * We allocate one q_vector per queue interrupt. If allocation fails we 2849 * return -ENOMEM. 2850 **/ 2851 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 2852 { 2853 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2854 int rxr_remaining = adapter->num_rx_queues; 2855 int txr_remaining = adapter->num_tx_queues; 2856 int xdp_remaining = adapter->num_xdp_queues; 2857 int rxr_idx = 0, txr_idx = 0, xdp_idx = 0, v_idx = 0; 2858 int err; 2859 2860 if (q_vectors >= (rxr_remaining + txr_remaining + xdp_remaining)) { 2861 for (; rxr_remaining; v_idx++, q_vectors--) { 2862 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors); 2863 2864 err = ixgbevf_alloc_q_vector(adapter, v_idx, 2865 0, 0, 0, 0, rqpv, rxr_idx); 2866 if (err) 2867 goto err_out; 2868 2869 /* update counts and index */ 2870 rxr_remaining -= rqpv; 2871 rxr_idx += rqpv; 2872 } 2873 } 2874 2875 for (; q_vectors; v_idx++, q_vectors--) { 2876 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors); 2877 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors); 2878 int xqpv = DIV_ROUND_UP(xdp_remaining, q_vectors); 2879 2880 err = ixgbevf_alloc_q_vector(adapter, v_idx, 2881 tqpv, txr_idx, 2882 xqpv, xdp_idx, 2883 rqpv, rxr_idx); 2884 2885 if (err) 2886 goto err_out; 2887 2888 /* update counts and index */ 2889 rxr_remaining -= rqpv; 2890 rxr_idx += rqpv; 2891 txr_remaining -= tqpv; 2892 txr_idx += tqpv; 2893 xdp_remaining -= xqpv; 2894 xdp_idx += xqpv; 2895 } 2896 2897 return 0; 2898 2899 err_out: 2900 while (v_idx) { 2901 v_idx--; 2902 ixgbevf_free_q_vector(adapter, v_idx); 2903 } 2904 2905 return -ENOMEM; 2906 } 2907 2908 /** 2909 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 2910 * @adapter: board private structure to initialize 2911 * 2912 * This function frees the memory allocated to the q_vectors. In addition if 2913 * NAPI is enabled it will delete any references to the NAPI struct prior 2914 * to freeing the q_vector. 2915 **/ 2916 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 2917 { 2918 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2919 2920 while (q_vectors) { 2921 q_vectors--; 2922 ixgbevf_free_q_vector(adapter, q_vectors); 2923 } 2924 } 2925 2926 /** 2927 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 2928 * @adapter: board private structure 2929 * 2930 **/ 2931 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 2932 { 2933 if (!adapter->msix_entries) 2934 return; 2935 2936 pci_disable_msix(adapter->pdev); 2937 kfree(adapter->msix_entries); 2938 adapter->msix_entries = NULL; 2939 } 2940 2941 /** 2942 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 2943 * @adapter: board private structure to initialize 2944 * 2945 **/ 2946 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 2947 { 2948 int err; 2949 2950 /* Number of supported queues */ 2951 ixgbevf_set_num_queues(adapter); 2952 2953 err = ixgbevf_set_interrupt_capability(adapter); 2954 if (err) { 2955 hw_dbg(&adapter->hw, 2956 "Unable to setup interrupt capabilities\n"); 2957 goto err_set_interrupt; 2958 } 2959 2960 err = ixgbevf_alloc_q_vectors(adapter); 2961 if (err) { 2962 hw_dbg(&adapter->hw, "Unable to allocate memory for queue vectors\n"); 2963 goto err_alloc_q_vectors; 2964 } 2965 2966 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u XDP Queue count %u\n", 2967 (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled", 2968 adapter->num_rx_queues, adapter->num_tx_queues, 2969 adapter->num_xdp_queues); 2970 2971 set_bit(__IXGBEVF_DOWN, &adapter->state); 2972 2973 return 0; 2974 err_alloc_q_vectors: 2975 ixgbevf_reset_interrupt_capability(adapter); 2976 err_set_interrupt: 2977 return err; 2978 } 2979 2980 /** 2981 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings 2982 * @adapter: board private structure to clear interrupt scheme on 2983 * 2984 * We go through and clear interrupt specific resources and reset the structure 2985 * to pre-load conditions 2986 **/ 2987 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) 2988 { 2989 adapter->num_tx_queues = 0; 2990 adapter->num_xdp_queues = 0; 2991 adapter->num_rx_queues = 0; 2992 2993 ixgbevf_free_q_vectors(adapter); 2994 ixgbevf_reset_interrupt_capability(adapter); 2995 } 2996 2997 /** 2998 * ixgbevf_sw_init - Initialize general software structures 2999 * @adapter: board private structure to initialize 3000 * 3001 * ixgbevf_sw_init initializes the Adapter private data structure. 3002 * Fields are initialized based on PCI device information and 3003 * OS network device settings (MTU size). 3004 **/ 3005 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 3006 { 3007 struct ixgbe_hw *hw = &adapter->hw; 3008 struct pci_dev *pdev = adapter->pdev; 3009 struct net_device *netdev = adapter->netdev; 3010 int err; 3011 3012 /* PCI config space info */ 3013 hw->vendor_id = pdev->vendor; 3014 hw->device_id = pdev->device; 3015 hw->revision_id = pdev->revision; 3016 hw->subsystem_vendor_id = pdev->subsystem_vendor; 3017 hw->subsystem_device_id = pdev->subsystem_device; 3018 3019 hw->mbx.ops.init_params(hw); 3020 3021 if (hw->mac.type >= ixgbe_mac_X550_vf) { 3022 err = ixgbevf_init_rss_key(adapter); 3023 if (err) 3024 goto out; 3025 } 3026 3027 /* assume legacy case in which PF would only give VF 2 queues */ 3028 hw->mac.max_tx_queues = 2; 3029 hw->mac.max_rx_queues = 2; 3030 3031 /* lock to protect mailbox accesses */ 3032 spin_lock_init(&adapter->mbx_lock); 3033 3034 err = hw->mac.ops.reset_hw(hw); 3035 if (err) { 3036 dev_info(&pdev->dev, 3037 "PF still in reset state. Is the PF interface up?\n"); 3038 } else { 3039 err = hw->mac.ops.init_hw(hw); 3040 if (err) { 3041 pr_err("init_shared_code failed: %d\n", err); 3042 goto out; 3043 } 3044 ixgbevf_negotiate_api(adapter); 3045 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr); 3046 if (err) 3047 dev_info(&pdev->dev, "Error reading MAC address\n"); 3048 else if (is_zero_ether_addr(adapter->hw.mac.addr)) 3049 dev_info(&pdev->dev, 3050 "MAC address not assigned by administrator.\n"); 3051 ether_addr_copy(netdev->dev_addr, hw->mac.addr); 3052 } 3053 3054 if (!is_valid_ether_addr(netdev->dev_addr)) { 3055 dev_info(&pdev->dev, "Assigning random MAC address\n"); 3056 eth_hw_addr_random(netdev); 3057 ether_addr_copy(hw->mac.addr, netdev->dev_addr); 3058 ether_addr_copy(hw->mac.perm_addr, netdev->dev_addr); 3059 } 3060 3061 /* Enable dynamic interrupt throttling rates */ 3062 adapter->rx_itr_setting = 1; 3063 adapter->tx_itr_setting = 1; 3064 3065 /* set default ring sizes */ 3066 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 3067 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 3068 3069 set_bit(__IXGBEVF_DOWN, &adapter->state); 3070 return 0; 3071 3072 out: 3073 return err; 3074 } 3075 3076 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 3077 { \ 3078 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 3079 if (current_counter < last_counter) \ 3080 counter += 0x100000000LL; \ 3081 last_counter = current_counter; \ 3082 counter &= 0xFFFFFFFF00000000LL; \ 3083 counter |= current_counter; \ 3084 } 3085 3086 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 3087 { \ 3088 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 3089 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 3090 u64 current_counter = (current_counter_msb << 32) | \ 3091 current_counter_lsb; \ 3092 if (current_counter < last_counter) \ 3093 counter += 0x1000000000LL; \ 3094 last_counter = current_counter; \ 3095 counter &= 0xFFFFFFF000000000LL; \ 3096 counter |= current_counter; \ 3097 } 3098 /** 3099 * ixgbevf_update_stats - Update the board statistics counters. 3100 * @adapter: board private structure 3101 **/ 3102 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 3103 { 3104 struct ixgbe_hw *hw = &adapter->hw; 3105 u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0; 3106 u64 alloc_rx_page = 0, hw_csum_rx_error = 0; 3107 int i; 3108 3109 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3110 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3111 return; 3112 3113 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 3114 adapter->stats.vfgprc); 3115 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 3116 adapter->stats.vfgptc); 3117 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3118 adapter->stats.last_vfgorc, 3119 adapter->stats.vfgorc); 3120 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3121 adapter->stats.last_vfgotc, 3122 adapter->stats.vfgotc); 3123 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 3124 adapter->stats.vfmprc); 3125 3126 for (i = 0; i < adapter->num_rx_queues; i++) { 3127 struct ixgbevf_ring *rx_ring = adapter->rx_ring[i]; 3128 3129 hw_csum_rx_error += rx_ring->rx_stats.csum_err; 3130 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed; 3131 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed; 3132 alloc_rx_page += rx_ring->rx_stats.alloc_rx_page; 3133 } 3134 3135 adapter->hw_csum_rx_error = hw_csum_rx_error; 3136 adapter->alloc_rx_page_failed = alloc_rx_page_failed; 3137 adapter->alloc_rx_buff_failed = alloc_rx_buff_failed; 3138 adapter->alloc_rx_page = alloc_rx_page; 3139 } 3140 3141 /** 3142 * ixgbevf_service_timer - Timer Call-back 3143 * @t: pointer to timer_list struct 3144 **/ 3145 static void ixgbevf_service_timer(struct timer_list *t) 3146 { 3147 struct ixgbevf_adapter *adapter = from_timer(adapter, t, 3148 service_timer); 3149 3150 /* Reset the timer */ 3151 mod_timer(&adapter->service_timer, (HZ * 2) + jiffies); 3152 3153 ixgbevf_service_event_schedule(adapter); 3154 } 3155 3156 static void ixgbevf_reset_subtask(struct ixgbevf_adapter *adapter) 3157 { 3158 if (!test_and_clear_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state)) 3159 return; 3160 3161 rtnl_lock(); 3162 /* If we're already down or resetting, just bail */ 3163 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3164 test_bit(__IXGBEVF_REMOVING, &adapter->state) || 3165 test_bit(__IXGBEVF_RESETTING, &adapter->state)) { 3166 rtnl_unlock(); 3167 return; 3168 } 3169 3170 adapter->tx_timeout_count++; 3171 3172 ixgbevf_reinit_locked(adapter); 3173 rtnl_unlock(); 3174 } 3175 3176 /** 3177 * ixgbevf_check_hang_subtask - check for hung queues and dropped interrupts 3178 * @adapter: pointer to the device adapter structure 3179 * 3180 * This function serves two purposes. First it strobes the interrupt lines 3181 * in order to make certain interrupts are occurring. Secondly it sets the 3182 * bits needed to check for TX hangs. As a result we should immediately 3183 * determine if a hang has occurred. 3184 **/ 3185 static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter) 3186 { 3187 struct ixgbe_hw *hw = &adapter->hw; 3188 u32 eics = 0; 3189 int i; 3190 3191 /* If we're down or resetting, just bail */ 3192 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3193 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3194 return; 3195 3196 /* Force detection of hung controller */ 3197 if (netif_carrier_ok(adapter->netdev)) { 3198 for (i = 0; i < adapter->num_tx_queues; i++) 3199 set_check_for_tx_hang(adapter->tx_ring[i]); 3200 for (i = 0; i < adapter->num_xdp_queues; i++) 3201 set_check_for_tx_hang(adapter->xdp_ring[i]); 3202 } 3203 3204 /* get one bit for every active Tx/Rx interrupt vector */ 3205 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 3206 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 3207 3208 if (qv->rx.ring || qv->tx.ring) 3209 eics |= BIT(i); 3210 } 3211 3212 /* Cause software interrupt to ensure rings are cleaned */ 3213 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics); 3214 } 3215 3216 /** 3217 * ixgbevf_watchdog_update_link - update the link status 3218 * @adapter: pointer to the device adapter structure 3219 **/ 3220 static void ixgbevf_watchdog_update_link(struct ixgbevf_adapter *adapter) 3221 { 3222 struct ixgbe_hw *hw = &adapter->hw; 3223 u32 link_speed = adapter->link_speed; 3224 bool link_up = adapter->link_up; 3225 s32 err; 3226 3227 spin_lock_bh(&adapter->mbx_lock); 3228 3229 err = hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 3230 3231 spin_unlock_bh(&adapter->mbx_lock); 3232 3233 /* if check for link returns error we will need to reset */ 3234 if (err && time_after(jiffies, adapter->last_reset + (10 * HZ))) { 3235 set_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state); 3236 link_up = false; 3237 } 3238 3239 adapter->link_up = link_up; 3240 adapter->link_speed = link_speed; 3241 } 3242 3243 /** 3244 * ixgbevf_watchdog_link_is_up - update netif_carrier status and 3245 * print link up message 3246 * @adapter: pointer to the device adapter structure 3247 **/ 3248 static void ixgbevf_watchdog_link_is_up(struct ixgbevf_adapter *adapter) 3249 { 3250 struct net_device *netdev = adapter->netdev; 3251 3252 /* only continue if link was previously down */ 3253 if (netif_carrier_ok(netdev)) 3254 return; 3255 3256 dev_info(&adapter->pdev->dev, "NIC Link is Up %s\n", 3257 (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 3258 "10 Gbps" : 3259 (adapter->link_speed == IXGBE_LINK_SPEED_1GB_FULL) ? 3260 "1 Gbps" : 3261 (adapter->link_speed == IXGBE_LINK_SPEED_100_FULL) ? 3262 "100 Mbps" : 3263 "unknown speed"); 3264 3265 netif_carrier_on(netdev); 3266 } 3267 3268 /** 3269 * ixgbevf_watchdog_link_is_down - update netif_carrier status and 3270 * print link down message 3271 * @adapter: pointer to the adapter structure 3272 **/ 3273 static void ixgbevf_watchdog_link_is_down(struct ixgbevf_adapter *adapter) 3274 { 3275 struct net_device *netdev = adapter->netdev; 3276 3277 adapter->link_speed = 0; 3278 3279 /* only continue if link was up previously */ 3280 if (!netif_carrier_ok(netdev)) 3281 return; 3282 3283 dev_info(&adapter->pdev->dev, "NIC Link is Down\n"); 3284 3285 netif_carrier_off(netdev); 3286 } 3287 3288 /** 3289 * ixgbevf_watchdog_subtask - worker thread to bring link up 3290 * @adapter: board private structure 3291 **/ 3292 static void ixgbevf_watchdog_subtask(struct ixgbevf_adapter *adapter) 3293 { 3294 /* if interface is down do nothing */ 3295 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3296 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3297 return; 3298 3299 ixgbevf_watchdog_update_link(adapter); 3300 3301 if (adapter->link_up) 3302 ixgbevf_watchdog_link_is_up(adapter); 3303 else 3304 ixgbevf_watchdog_link_is_down(adapter); 3305 3306 ixgbevf_update_stats(adapter); 3307 } 3308 3309 /** 3310 * ixgbevf_service_task - manages and runs subtasks 3311 * @work: pointer to work_struct containing our data 3312 **/ 3313 static void ixgbevf_service_task(struct work_struct *work) 3314 { 3315 struct ixgbevf_adapter *adapter = container_of(work, 3316 struct ixgbevf_adapter, 3317 service_task); 3318 struct ixgbe_hw *hw = &adapter->hw; 3319 3320 if (IXGBE_REMOVED(hw->hw_addr)) { 3321 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 3322 rtnl_lock(); 3323 ixgbevf_down(adapter); 3324 rtnl_unlock(); 3325 } 3326 return; 3327 } 3328 3329 ixgbevf_queue_reset_subtask(adapter); 3330 ixgbevf_reset_subtask(adapter); 3331 ixgbevf_watchdog_subtask(adapter); 3332 ixgbevf_check_hang_subtask(adapter); 3333 3334 ixgbevf_service_event_complete(adapter); 3335 } 3336 3337 /** 3338 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 3339 * @tx_ring: Tx descriptor ring for a specific queue 3340 * 3341 * Free all transmit software resources 3342 **/ 3343 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) 3344 { 3345 ixgbevf_clean_tx_ring(tx_ring); 3346 3347 vfree(tx_ring->tx_buffer_info); 3348 tx_ring->tx_buffer_info = NULL; 3349 3350 /* if not set, then don't free */ 3351 if (!tx_ring->desc) 3352 return; 3353 3354 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, 3355 tx_ring->dma); 3356 3357 tx_ring->desc = NULL; 3358 } 3359 3360 /** 3361 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 3362 * @adapter: board private structure 3363 * 3364 * Free all transmit software resources 3365 **/ 3366 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 3367 { 3368 int i; 3369 3370 for (i = 0; i < adapter->num_tx_queues; i++) 3371 if (adapter->tx_ring[i]->desc) 3372 ixgbevf_free_tx_resources(adapter->tx_ring[i]); 3373 for (i = 0; i < adapter->num_xdp_queues; i++) 3374 if (adapter->xdp_ring[i]->desc) 3375 ixgbevf_free_tx_resources(adapter->xdp_ring[i]); 3376 } 3377 3378 /** 3379 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 3380 * @tx_ring: Tx descriptor ring (for a specific queue) to setup 3381 * 3382 * Return 0 on success, negative on failure 3383 **/ 3384 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) 3385 { 3386 struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); 3387 int size; 3388 3389 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 3390 tx_ring->tx_buffer_info = vmalloc(size); 3391 if (!tx_ring->tx_buffer_info) 3392 goto err; 3393 3394 u64_stats_init(&tx_ring->syncp); 3395 3396 /* round up to nearest 4K */ 3397 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 3398 tx_ring->size = ALIGN(tx_ring->size, 4096); 3399 3400 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, 3401 &tx_ring->dma, GFP_KERNEL); 3402 if (!tx_ring->desc) 3403 goto err; 3404 3405 return 0; 3406 3407 err: 3408 vfree(tx_ring->tx_buffer_info); 3409 tx_ring->tx_buffer_info = NULL; 3410 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit descriptor ring\n"); 3411 return -ENOMEM; 3412 } 3413 3414 /** 3415 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 3416 * @adapter: board private structure 3417 * 3418 * If this function returns with an error, then it's possible one or 3419 * more of the rings is populated (while the rest are not). It is the 3420 * callers duty to clean those orphaned rings. 3421 * 3422 * Return 0 on success, negative on failure 3423 **/ 3424 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 3425 { 3426 int i, j = 0, err = 0; 3427 3428 for (i = 0; i < adapter->num_tx_queues; i++) { 3429 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]); 3430 if (!err) 3431 continue; 3432 hw_dbg(&adapter->hw, "Allocation for Tx Queue %u failed\n", i); 3433 goto err_setup_tx; 3434 } 3435 3436 for (j = 0; j < adapter->num_xdp_queues; j++) { 3437 err = ixgbevf_setup_tx_resources(adapter->xdp_ring[j]); 3438 if (!err) 3439 continue; 3440 hw_dbg(&adapter->hw, "Allocation for XDP Queue %u failed\n", j); 3441 goto err_setup_tx; 3442 } 3443 3444 return 0; 3445 err_setup_tx: 3446 /* rewind the index freeing the rings as we go */ 3447 while (j--) 3448 ixgbevf_free_tx_resources(adapter->xdp_ring[j]); 3449 while (i--) 3450 ixgbevf_free_tx_resources(adapter->tx_ring[i]); 3451 3452 return err; 3453 } 3454 3455 /** 3456 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 3457 * @adapter: board private structure 3458 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 3459 * 3460 * Returns 0 on success, negative on failure 3461 **/ 3462 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, 3463 struct ixgbevf_ring *rx_ring) 3464 { 3465 int size; 3466 3467 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 3468 rx_ring->rx_buffer_info = vmalloc(size); 3469 if (!rx_ring->rx_buffer_info) 3470 goto err; 3471 3472 u64_stats_init(&rx_ring->syncp); 3473 3474 /* Round up to nearest 4K */ 3475 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 3476 rx_ring->size = ALIGN(rx_ring->size, 4096); 3477 3478 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, 3479 &rx_ring->dma, GFP_KERNEL); 3480 3481 if (!rx_ring->desc) 3482 goto err; 3483 3484 /* XDP RX-queue info */ 3485 if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, 3486 rx_ring->queue_index) < 0) 3487 goto err; 3488 3489 rx_ring->xdp_prog = adapter->xdp_prog; 3490 3491 return 0; 3492 err: 3493 vfree(rx_ring->rx_buffer_info); 3494 rx_ring->rx_buffer_info = NULL; 3495 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); 3496 return -ENOMEM; 3497 } 3498 3499 /** 3500 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 3501 * @adapter: board private structure 3502 * 3503 * If this function returns with an error, then it's possible one or 3504 * more of the rings is populated (while the rest are not). It is the 3505 * callers duty to clean those orphaned rings. 3506 * 3507 * Return 0 on success, negative on failure 3508 **/ 3509 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 3510 { 3511 int i, err = 0; 3512 3513 for (i = 0; i < adapter->num_rx_queues; i++) { 3514 err = ixgbevf_setup_rx_resources(adapter, adapter->rx_ring[i]); 3515 if (!err) 3516 continue; 3517 hw_dbg(&adapter->hw, "Allocation for Rx Queue %u failed\n", i); 3518 goto err_setup_rx; 3519 } 3520 3521 return 0; 3522 err_setup_rx: 3523 /* rewind the index freeing the rings as we go */ 3524 while (i--) 3525 ixgbevf_free_rx_resources(adapter->rx_ring[i]); 3526 return err; 3527 } 3528 3529 /** 3530 * ixgbevf_free_rx_resources - Free Rx Resources 3531 * @rx_ring: ring to clean the resources from 3532 * 3533 * Free all receive software resources 3534 **/ 3535 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) 3536 { 3537 ixgbevf_clean_rx_ring(rx_ring); 3538 3539 rx_ring->xdp_prog = NULL; 3540 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 3541 vfree(rx_ring->rx_buffer_info); 3542 rx_ring->rx_buffer_info = NULL; 3543 3544 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, 3545 rx_ring->dma); 3546 3547 rx_ring->desc = NULL; 3548 } 3549 3550 /** 3551 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 3552 * @adapter: board private structure 3553 * 3554 * Free all receive software resources 3555 **/ 3556 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 3557 { 3558 int i; 3559 3560 for (i = 0; i < adapter->num_rx_queues; i++) 3561 if (adapter->rx_ring[i]->desc) 3562 ixgbevf_free_rx_resources(adapter->rx_ring[i]); 3563 } 3564 3565 /** 3566 * ixgbevf_open - Called when a network interface is made active 3567 * @netdev: network interface device structure 3568 * 3569 * Returns 0 on success, negative value on failure 3570 * 3571 * The open entry point is called when a network interface is made 3572 * active by the system (IFF_UP). At this point all resources needed 3573 * for transmit and receive operations are allocated, the interrupt 3574 * handler is registered with the OS, the watchdog timer is started, 3575 * and the stack is notified that the interface is ready. 3576 **/ 3577 int ixgbevf_open(struct net_device *netdev) 3578 { 3579 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3580 struct ixgbe_hw *hw = &adapter->hw; 3581 int err; 3582 3583 /* A previous failure to open the device because of a lack of 3584 * available MSIX vector resources may have reset the number 3585 * of msix vectors variable to zero. The only way to recover 3586 * is to unload/reload the driver and hope that the system has 3587 * been able to recover some MSIX vector resources. 3588 */ 3589 if (!adapter->num_msix_vectors) 3590 return -ENOMEM; 3591 3592 if (hw->adapter_stopped) { 3593 ixgbevf_reset(adapter); 3594 /* if adapter is still stopped then PF isn't up and 3595 * the VF can't start. 3596 */ 3597 if (hw->adapter_stopped) { 3598 err = IXGBE_ERR_MBX; 3599 pr_err("Unable to start - perhaps the PF Driver isn't up yet\n"); 3600 goto err_setup_reset; 3601 } 3602 } 3603 3604 /* disallow open during test */ 3605 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 3606 return -EBUSY; 3607 3608 netif_carrier_off(netdev); 3609 3610 /* allocate transmit descriptors */ 3611 err = ixgbevf_setup_all_tx_resources(adapter); 3612 if (err) 3613 goto err_setup_tx; 3614 3615 /* allocate receive descriptors */ 3616 err = ixgbevf_setup_all_rx_resources(adapter); 3617 if (err) 3618 goto err_setup_rx; 3619 3620 ixgbevf_configure(adapter); 3621 3622 err = ixgbevf_request_irq(adapter); 3623 if (err) 3624 goto err_req_irq; 3625 3626 /* Notify the stack of the actual queue counts. */ 3627 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 3628 if (err) 3629 goto err_set_queues; 3630 3631 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 3632 if (err) 3633 goto err_set_queues; 3634 3635 ixgbevf_up_complete(adapter); 3636 3637 return 0; 3638 3639 err_set_queues: 3640 ixgbevf_free_irq(adapter); 3641 err_req_irq: 3642 ixgbevf_free_all_rx_resources(adapter); 3643 err_setup_rx: 3644 ixgbevf_free_all_tx_resources(adapter); 3645 err_setup_tx: 3646 ixgbevf_reset(adapter); 3647 err_setup_reset: 3648 3649 return err; 3650 } 3651 3652 /** 3653 * ixgbevf_close_suspend - actions necessary to both suspend and close flows 3654 * @adapter: the private adapter struct 3655 * 3656 * This function should contain the necessary work common to both suspending 3657 * and closing of the device. 3658 */ 3659 static void ixgbevf_close_suspend(struct ixgbevf_adapter *adapter) 3660 { 3661 ixgbevf_down(adapter); 3662 ixgbevf_free_irq(adapter); 3663 ixgbevf_free_all_tx_resources(adapter); 3664 ixgbevf_free_all_rx_resources(adapter); 3665 } 3666 3667 /** 3668 * ixgbevf_close - Disables a network interface 3669 * @netdev: network interface device structure 3670 * 3671 * Returns 0, this is not allowed to fail 3672 * 3673 * The close entry point is called when an interface is de-activated 3674 * by the OS. The hardware is still under the drivers control, but 3675 * needs to be disabled. A global MAC reset is issued to stop the 3676 * hardware, and all transmit and receive resources are freed. 3677 **/ 3678 int ixgbevf_close(struct net_device *netdev) 3679 { 3680 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3681 3682 if (netif_device_present(netdev)) 3683 ixgbevf_close_suspend(adapter); 3684 3685 return 0; 3686 } 3687 3688 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter) 3689 { 3690 struct net_device *dev = adapter->netdev; 3691 3692 if (!test_and_clear_bit(__IXGBEVF_QUEUE_RESET_REQUESTED, 3693 &adapter->state)) 3694 return; 3695 3696 /* if interface is down do nothing */ 3697 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3698 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3699 return; 3700 3701 /* Hardware has to reinitialize queues and interrupts to 3702 * match packet buffer alignment. Unfortunately, the 3703 * hardware is not flexible enough to do this dynamically. 3704 */ 3705 rtnl_lock(); 3706 3707 if (netif_running(dev)) 3708 ixgbevf_close(dev); 3709 3710 ixgbevf_clear_interrupt_scheme(adapter); 3711 ixgbevf_init_interrupt_scheme(adapter); 3712 3713 if (netif_running(dev)) 3714 ixgbevf_open(dev); 3715 3716 rtnl_unlock(); 3717 } 3718 3719 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, 3720 u32 vlan_macip_lens, u32 fceof_saidx, 3721 u32 type_tucmd, u32 mss_l4len_idx) 3722 { 3723 struct ixgbe_adv_tx_context_desc *context_desc; 3724 u16 i = tx_ring->next_to_use; 3725 3726 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i); 3727 3728 i++; 3729 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 3730 3731 /* set bits to identify this as an advanced context descriptor */ 3732 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3733 3734 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 3735 context_desc->fceof_saidx = cpu_to_le32(fceof_saidx); 3736 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 3737 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 3738 } 3739 3740 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, 3741 struct ixgbevf_tx_buffer *first, 3742 u8 *hdr_len, 3743 struct ixgbevf_ipsec_tx_data *itd) 3744 { 3745 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 3746 struct sk_buff *skb = first->skb; 3747 union { 3748 struct iphdr *v4; 3749 struct ipv6hdr *v6; 3750 unsigned char *hdr; 3751 } ip; 3752 union { 3753 struct tcphdr *tcp; 3754 unsigned char *hdr; 3755 } l4; 3756 u32 paylen, l4_offset; 3757 u32 fceof_saidx = 0; 3758 int err; 3759 3760 if (skb->ip_summed != CHECKSUM_PARTIAL) 3761 return 0; 3762 3763 if (!skb_is_gso(skb)) 3764 return 0; 3765 3766 err = skb_cow_head(skb, 0); 3767 if (err < 0) 3768 return err; 3769 3770 if (eth_p_mpls(first->protocol)) 3771 ip.hdr = skb_inner_network_header(skb); 3772 else 3773 ip.hdr = skb_network_header(skb); 3774 l4.hdr = skb_checksum_start(skb); 3775 3776 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 3777 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3778 3779 /* initialize outer IP header fields */ 3780 if (ip.v4->version == 4) { 3781 unsigned char *csum_start = skb_checksum_start(skb); 3782 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 3783 int len = csum_start - trans_start; 3784 3785 /* IP header will have to cancel out any data that 3786 * is not a part of the outer IP header, so set to 3787 * a reverse csum if needed, else init check to 0. 3788 */ 3789 ip.v4->check = (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) ? 3790 csum_fold(csum_partial(trans_start, 3791 len, 0)) : 0; 3792 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3793 3794 ip.v4->tot_len = 0; 3795 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3796 IXGBE_TX_FLAGS_CSUM | 3797 IXGBE_TX_FLAGS_IPV4; 3798 } else { 3799 ip.v6->payload_len = 0; 3800 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3801 IXGBE_TX_FLAGS_CSUM; 3802 } 3803 3804 /* determine offset of inner transport header */ 3805 l4_offset = l4.hdr - skb->data; 3806 3807 /* compute length of segmentation header */ 3808 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 3809 3810 /* remove payload length from inner checksum */ 3811 paylen = skb->len - l4_offset; 3812 csum_replace_by_diff(&l4.tcp->check, htonl(paylen)); 3813 3814 /* update gso size and bytecount with header size */ 3815 first->gso_segs = skb_shinfo(skb)->gso_segs; 3816 first->bytecount += (first->gso_segs - 1) * *hdr_len; 3817 3818 /* mss_l4len_id: use 1 as index for TSO */ 3819 mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT; 3820 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 3821 mss_l4len_idx |= (1u << IXGBE_ADVTXD_IDX_SHIFT); 3822 3823 fceof_saidx |= itd->pfsa; 3824 type_tucmd |= itd->flags | itd->trailer_len; 3825 3826 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 3827 vlan_macip_lens = l4.hdr - ip.hdr; 3828 vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT; 3829 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3830 3831 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd, 3832 mss_l4len_idx); 3833 3834 return 1; 3835 } 3836 3837 static inline bool ixgbevf_ipv6_csum_is_sctp(struct sk_buff *skb) 3838 { 3839 unsigned int offset = 0; 3840 3841 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 3842 3843 return offset == skb_checksum_start_offset(skb); 3844 } 3845 3846 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, 3847 struct ixgbevf_tx_buffer *first, 3848 struct ixgbevf_ipsec_tx_data *itd) 3849 { 3850 struct sk_buff *skb = first->skb; 3851 u32 vlan_macip_lens = 0; 3852 u32 fceof_saidx = 0; 3853 u32 type_tucmd = 0; 3854 3855 if (skb->ip_summed != CHECKSUM_PARTIAL) 3856 goto no_csum; 3857 3858 switch (skb->csum_offset) { 3859 case offsetof(struct tcphdr, check): 3860 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3861 /* fall through */ 3862 case offsetof(struct udphdr, check): 3863 break; 3864 case offsetof(struct sctphdr, checksum): 3865 /* validate that this is actually an SCTP request */ 3866 if (((first->protocol == htons(ETH_P_IP)) && 3867 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 3868 ((first->protocol == htons(ETH_P_IPV6)) && 3869 ixgbevf_ipv6_csum_is_sctp(skb))) { 3870 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_SCTP; 3871 break; 3872 } 3873 /* fall through */ 3874 default: 3875 skb_checksum_help(skb); 3876 goto no_csum; 3877 } 3878 3879 if (first->protocol == htons(ETH_P_IP)) 3880 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3881 3882 /* update TX checksum flag */ 3883 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 3884 vlan_macip_lens = skb_checksum_start_offset(skb) - 3885 skb_network_offset(skb); 3886 no_csum: 3887 /* vlan_macip_lens: MACLEN, VLAN tag */ 3888 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3889 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3890 3891 fceof_saidx |= itd->pfsa; 3892 type_tucmd |= itd->flags | itd->trailer_len; 3893 3894 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3895 fceof_saidx, type_tucmd, 0); 3896 } 3897 3898 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags) 3899 { 3900 /* set type for advanced descriptor with frame checksum insertion */ 3901 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | 3902 IXGBE_ADVTXD_DCMD_IFCS | 3903 IXGBE_ADVTXD_DCMD_DEXT); 3904 3905 /* set HW VLAN bit if VLAN is present */ 3906 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 3907 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); 3908 3909 /* set segmentation enable bits for TSO/FSO */ 3910 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3911 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); 3912 3913 return cmd_type; 3914 } 3915 3916 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 3917 u32 tx_flags, unsigned int paylen) 3918 { 3919 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT); 3920 3921 /* enable L4 checksum for TSO and TX checksum offload */ 3922 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 3923 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM); 3924 3925 /* enble IPv4 checksum for TSO */ 3926 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 3927 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM); 3928 3929 /* enable IPsec */ 3930 if (tx_flags & IXGBE_TX_FLAGS_IPSEC) 3931 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IPSEC); 3932 3933 /* use index 1 context for TSO/FSO/FCOE/IPSEC */ 3934 if (tx_flags & (IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_IPSEC)) 3935 olinfo_status |= cpu_to_le32(1u << IXGBE_ADVTXD_IDX_SHIFT); 3936 3937 /* Check Context must be set if Tx switch is enabled, which it 3938 * always is for case where virtual functions are running 3939 */ 3940 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC); 3941 3942 tx_desc->read.olinfo_status = olinfo_status; 3943 } 3944 3945 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, 3946 struct ixgbevf_tx_buffer *first, 3947 const u8 hdr_len) 3948 { 3949 struct sk_buff *skb = first->skb; 3950 struct ixgbevf_tx_buffer *tx_buffer; 3951 union ixgbe_adv_tx_desc *tx_desc; 3952 struct skb_frag_struct *frag; 3953 dma_addr_t dma; 3954 unsigned int data_len, size; 3955 u32 tx_flags = first->tx_flags; 3956 __le32 cmd_type = ixgbevf_tx_cmd_type(tx_flags); 3957 u16 i = tx_ring->next_to_use; 3958 3959 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 3960 3961 ixgbevf_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len); 3962 3963 size = skb_headlen(skb); 3964 data_len = skb->data_len; 3965 3966 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 3967 3968 tx_buffer = first; 3969 3970 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 3971 if (dma_mapping_error(tx_ring->dev, dma)) 3972 goto dma_error; 3973 3974 /* record length, and DMA address */ 3975 dma_unmap_len_set(tx_buffer, len, size); 3976 dma_unmap_addr_set(tx_buffer, dma, dma); 3977 3978 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3979 3980 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 3981 tx_desc->read.cmd_type_len = 3982 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); 3983 3984 i++; 3985 tx_desc++; 3986 if (i == tx_ring->count) { 3987 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3988 i = 0; 3989 } 3990 tx_desc->read.olinfo_status = 0; 3991 3992 dma += IXGBE_MAX_DATA_PER_TXD; 3993 size -= IXGBE_MAX_DATA_PER_TXD; 3994 3995 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3996 } 3997 3998 if (likely(!data_len)) 3999 break; 4000 4001 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); 4002 4003 i++; 4004 tx_desc++; 4005 if (i == tx_ring->count) { 4006 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 4007 i = 0; 4008 } 4009 tx_desc->read.olinfo_status = 0; 4010 4011 size = skb_frag_size(frag); 4012 data_len -= size; 4013 4014 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 4015 DMA_TO_DEVICE); 4016 4017 tx_buffer = &tx_ring->tx_buffer_info[i]; 4018 } 4019 4020 /* write last descriptor with RS and EOP bits */ 4021 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); 4022 tx_desc->read.cmd_type_len = cmd_type; 4023 4024 /* set the timestamp */ 4025 first->time_stamp = jiffies; 4026 4027 skb_tx_timestamp(skb); 4028 4029 /* Force memory writes to complete before letting h/w know there 4030 * are new descriptors to fetch. (Only applicable for weak-ordered 4031 * memory model archs, such as IA-64). 4032 * 4033 * We also need this memory barrier (wmb) to make certain all of the 4034 * status bits have been updated before next_to_watch is written. 4035 */ 4036 wmb(); 4037 4038 /* set next_to_watch value indicating a packet is present */ 4039 first->next_to_watch = tx_desc; 4040 4041 i++; 4042 if (i == tx_ring->count) 4043 i = 0; 4044 4045 tx_ring->next_to_use = i; 4046 4047 /* notify HW of packet */ 4048 ixgbevf_write_tail(tx_ring, i); 4049 4050 return; 4051 dma_error: 4052 dev_err(tx_ring->dev, "TX DMA map failed\n"); 4053 tx_buffer = &tx_ring->tx_buffer_info[i]; 4054 4055 /* clear dma mappings for failed tx_buffer_info map */ 4056 while (tx_buffer != first) { 4057 if (dma_unmap_len(tx_buffer, len)) 4058 dma_unmap_page(tx_ring->dev, 4059 dma_unmap_addr(tx_buffer, dma), 4060 dma_unmap_len(tx_buffer, len), 4061 DMA_TO_DEVICE); 4062 dma_unmap_len_set(tx_buffer, len, 0); 4063 4064 if (i-- == 0) 4065 i += tx_ring->count; 4066 tx_buffer = &tx_ring->tx_buffer_info[i]; 4067 } 4068 4069 if (dma_unmap_len(tx_buffer, len)) 4070 dma_unmap_single(tx_ring->dev, 4071 dma_unmap_addr(tx_buffer, dma), 4072 dma_unmap_len(tx_buffer, len), 4073 DMA_TO_DEVICE); 4074 dma_unmap_len_set(tx_buffer, len, 0); 4075 4076 dev_kfree_skb_any(tx_buffer->skb); 4077 tx_buffer->skb = NULL; 4078 4079 tx_ring->next_to_use = i; 4080 } 4081 4082 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 4083 { 4084 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 4085 /* Herbert's original patch had: 4086 * smp_mb__after_netif_stop_queue(); 4087 * but since that doesn't exist yet, just open code it. 4088 */ 4089 smp_mb(); 4090 4091 /* We need to check again in a case another CPU has just 4092 * made room available. 4093 */ 4094 if (likely(ixgbevf_desc_unused(tx_ring) < size)) 4095 return -EBUSY; 4096 4097 /* A reprieve! - use start_queue because it doesn't call schedule */ 4098 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 4099 ++tx_ring->tx_stats.restart_queue; 4100 4101 return 0; 4102 } 4103 4104 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 4105 { 4106 if (likely(ixgbevf_desc_unused(tx_ring) >= size)) 4107 return 0; 4108 return __ixgbevf_maybe_stop_tx(tx_ring, size); 4109 } 4110 4111 static int ixgbevf_xmit_frame_ring(struct sk_buff *skb, 4112 struct ixgbevf_ring *tx_ring) 4113 { 4114 struct ixgbevf_tx_buffer *first; 4115 int tso; 4116 u32 tx_flags = 0; 4117 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 4118 struct ixgbevf_ipsec_tx_data ipsec_tx = { 0 }; 4119 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 4120 unsigned short f; 4121 #endif 4122 u8 hdr_len = 0; 4123 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL); 4124 4125 if (!dst_mac || is_link_local_ether_addr(dst_mac)) { 4126 dev_kfree_skb_any(skb); 4127 return NETDEV_TX_OK; 4128 } 4129 4130 /* need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 4131 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 4132 * + 2 desc gap to keep tail from touching head, 4133 * + 1 desc for context descriptor, 4134 * otherwise try next time 4135 */ 4136 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 4137 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 4138 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 4139 #else 4140 count += skb_shinfo(skb)->nr_frags; 4141 #endif 4142 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { 4143 tx_ring->tx_stats.tx_busy++; 4144 return NETDEV_TX_BUSY; 4145 } 4146 4147 /* record the location of the first descriptor for this packet */ 4148 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 4149 first->skb = skb; 4150 first->bytecount = skb->len; 4151 first->gso_segs = 1; 4152 4153 if (skb_vlan_tag_present(skb)) { 4154 tx_flags |= skb_vlan_tag_get(skb); 4155 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 4156 tx_flags |= IXGBE_TX_FLAGS_VLAN; 4157 } 4158 4159 /* record initial flags and protocol */ 4160 first->tx_flags = tx_flags; 4161 first->protocol = vlan_get_protocol(skb); 4162 4163 #ifdef CONFIG_IXGBEVF_IPSEC 4164 if (secpath_exists(skb) && !ixgbevf_ipsec_tx(tx_ring, first, &ipsec_tx)) 4165 goto out_drop; 4166 #endif 4167 tso = ixgbevf_tso(tx_ring, first, &hdr_len, &ipsec_tx); 4168 if (tso < 0) 4169 goto out_drop; 4170 else if (!tso) 4171 ixgbevf_tx_csum(tx_ring, first, &ipsec_tx); 4172 4173 ixgbevf_tx_map(tx_ring, first, hdr_len); 4174 4175 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); 4176 4177 return NETDEV_TX_OK; 4178 4179 out_drop: 4180 dev_kfree_skb_any(first->skb); 4181 first->skb = NULL; 4182 4183 return NETDEV_TX_OK; 4184 } 4185 4186 static netdev_tx_t ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 4187 { 4188 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4189 struct ixgbevf_ring *tx_ring; 4190 4191 if (skb->len <= 0) { 4192 dev_kfree_skb_any(skb); 4193 return NETDEV_TX_OK; 4194 } 4195 4196 /* The minimum packet size for olinfo paylen is 17 so pad the skb 4197 * in order to meet this minimum size requirement. 4198 */ 4199 if (skb->len < 17) { 4200 if (skb_padto(skb, 17)) 4201 return NETDEV_TX_OK; 4202 skb->len = 17; 4203 } 4204 4205 tx_ring = adapter->tx_ring[skb->queue_mapping]; 4206 return ixgbevf_xmit_frame_ring(skb, tx_ring); 4207 } 4208 4209 /** 4210 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 4211 * @netdev: network interface device structure 4212 * @p: pointer to an address structure 4213 * 4214 * Returns 0 on success, negative on failure 4215 **/ 4216 static int ixgbevf_set_mac(struct net_device *netdev, void *p) 4217 { 4218 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4219 struct ixgbe_hw *hw = &adapter->hw; 4220 struct sockaddr *addr = p; 4221 int err; 4222 4223 if (!is_valid_ether_addr(addr->sa_data)) 4224 return -EADDRNOTAVAIL; 4225 4226 spin_lock_bh(&adapter->mbx_lock); 4227 4228 err = hw->mac.ops.set_rar(hw, 0, addr->sa_data, 0); 4229 4230 spin_unlock_bh(&adapter->mbx_lock); 4231 4232 if (err) 4233 return -EPERM; 4234 4235 ether_addr_copy(hw->mac.addr, addr->sa_data); 4236 ether_addr_copy(hw->mac.perm_addr, addr->sa_data); 4237 ether_addr_copy(netdev->dev_addr, addr->sa_data); 4238 4239 return 0; 4240 } 4241 4242 /** 4243 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 4244 * @netdev: network interface device structure 4245 * @new_mtu: new value for maximum frame size 4246 * 4247 * Returns 0 on success, negative on failure 4248 **/ 4249 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 4250 { 4251 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4252 struct ixgbe_hw *hw = &adapter->hw; 4253 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 4254 int ret; 4255 4256 /* prevent MTU being changed to a size unsupported by XDP */ 4257 if (adapter->xdp_prog) { 4258 dev_warn(&adapter->pdev->dev, "MTU cannot be changed while XDP program is loaded\n"); 4259 return -EPERM; 4260 } 4261 4262 spin_lock_bh(&adapter->mbx_lock); 4263 /* notify the PF of our intent to use this size of frame */ 4264 ret = hw->mac.ops.set_rlpml(hw, max_frame); 4265 spin_unlock_bh(&adapter->mbx_lock); 4266 if (ret) 4267 return -EINVAL; 4268 4269 hw_dbg(hw, "changing MTU from %d to %d\n", 4270 netdev->mtu, new_mtu); 4271 4272 /* must set new MTU before calling down or up */ 4273 netdev->mtu = new_mtu; 4274 4275 if (netif_running(netdev)) 4276 ixgbevf_reinit_locked(adapter); 4277 4278 return 0; 4279 } 4280 4281 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) 4282 { 4283 struct net_device *netdev = pci_get_drvdata(pdev); 4284 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4285 #ifdef CONFIG_PM 4286 int retval = 0; 4287 #endif 4288 4289 rtnl_lock(); 4290 netif_device_detach(netdev); 4291 4292 if (netif_running(netdev)) 4293 ixgbevf_close_suspend(adapter); 4294 4295 ixgbevf_clear_interrupt_scheme(adapter); 4296 rtnl_unlock(); 4297 4298 #ifdef CONFIG_PM 4299 retval = pci_save_state(pdev); 4300 if (retval) 4301 return retval; 4302 4303 #endif 4304 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 4305 pci_disable_device(pdev); 4306 4307 return 0; 4308 } 4309 4310 #ifdef CONFIG_PM 4311 static int ixgbevf_resume(struct pci_dev *pdev) 4312 { 4313 struct net_device *netdev = pci_get_drvdata(pdev); 4314 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4315 u32 err; 4316 4317 pci_restore_state(pdev); 4318 /* pci_restore_state clears dev->state_saved so call 4319 * pci_save_state to restore it. 4320 */ 4321 pci_save_state(pdev); 4322 4323 err = pci_enable_device_mem(pdev); 4324 if (err) { 4325 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 4326 return err; 4327 } 4328 4329 adapter->hw.hw_addr = adapter->io_addr; 4330 smp_mb__before_atomic(); 4331 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 4332 pci_set_master(pdev); 4333 4334 ixgbevf_reset(adapter); 4335 4336 rtnl_lock(); 4337 err = ixgbevf_init_interrupt_scheme(adapter); 4338 if (!err && netif_running(netdev)) 4339 err = ixgbevf_open(netdev); 4340 rtnl_unlock(); 4341 if (err) 4342 return err; 4343 4344 netif_device_attach(netdev); 4345 4346 return err; 4347 } 4348 4349 #endif /* CONFIG_PM */ 4350 static void ixgbevf_shutdown(struct pci_dev *pdev) 4351 { 4352 ixgbevf_suspend(pdev, PMSG_SUSPEND); 4353 } 4354 4355 static void ixgbevf_get_tx_ring_stats(struct rtnl_link_stats64 *stats, 4356 const struct ixgbevf_ring *ring) 4357 { 4358 u64 bytes, packets; 4359 unsigned int start; 4360 4361 if (ring) { 4362 do { 4363 start = u64_stats_fetch_begin_irq(&ring->syncp); 4364 bytes = ring->stats.bytes; 4365 packets = ring->stats.packets; 4366 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 4367 stats->tx_bytes += bytes; 4368 stats->tx_packets += packets; 4369 } 4370 } 4371 4372 static void ixgbevf_get_stats(struct net_device *netdev, 4373 struct rtnl_link_stats64 *stats) 4374 { 4375 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4376 unsigned int start; 4377 u64 bytes, packets; 4378 const struct ixgbevf_ring *ring; 4379 int i; 4380 4381 ixgbevf_update_stats(adapter); 4382 4383 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 4384 4385 rcu_read_lock(); 4386 for (i = 0; i < adapter->num_rx_queues; i++) { 4387 ring = adapter->rx_ring[i]; 4388 do { 4389 start = u64_stats_fetch_begin_irq(&ring->syncp); 4390 bytes = ring->stats.bytes; 4391 packets = ring->stats.packets; 4392 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 4393 stats->rx_bytes += bytes; 4394 stats->rx_packets += packets; 4395 } 4396 4397 for (i = 0; i < adapter->num_tx_queues; i++) { 4398 ring = adapter->tx_ring[i]; 4399 ixgbevf_get_tx_ring_stats(stats, ring); 4400 } 4401 4402 for (i = 0; i < adapter->num_xdp_queues; i++) { 4403 ring = adapter->xdp_ring[i]; 4404 ixgbevf_get_tx_ring_stats(stats, ring); 4405 } 4406 rcu_read_unlock(); 4407 } 4408 4409 #define IXGBEVF_MAX_MAC_HDR_LEN 127 4410 #define IXGBEVF_MAX_NETWORK_HDR_LEN 511 4411 4412 static netdev_features_t 4413 ixgbevf_features_check(struct sk_buff *skb, struct net_device *dev, 4414 netdev_features_t features) 4415 { 4416 unsigned int network_hdr_len, mac_hdr_len; 4417 4418 /* Make certain the headers can be described by a context descriptor */ 4419 mac_hdr_len = skb_network_header(skb) - skb->data; 4420 if (unlikely(mac_hdr_len > IXGBEVF_MAX_MAC_HDR_LEN)) 4421 return features & ~(NETIF_F_HW_CSUM | 4422 NETIF_F_SCTP_CRC | 4423 NETIF_F_HW_VLAN_CTAG_TX | 4424 NETIF_F_TSO | 4425 NETIF_F_TSO6); 4426 4427 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 4428 if (unlikely(network_hdr_len > IXGBEVF_MAX_NETWORK_HDR_LEN)) 4429 return features & ~(NETIF_F_HW_CSUM | 4430 NETIF_F_SCTP_CRC | 4431 NETIF_F_TSO | 4432 NETIF_F_TSO6); 4433 4434 /* We can only support IPV4 TSO in tunnels if we can mangle the 4435 * inner IP ID field, so strip TSO if MANGLEID is not supported. 4436 */ 4437 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 4438 features &= ~NETIF_F_TSO; 4439 4440 return features; 4441 } 4442 4443 static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) 4444 { 4445 int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 4446 struct ixgbevf_adapter *adapter = netdev_priv(dev); 4447 struct bpf_prog *old_prog; 4448 4449 /* verify ixgbevf ring attributes are sufficient for XDP */ 4450 for (i = 0; i < adapter->num_rx_queues; i++) { 4451 struct ixgbevf_ring *ring = adapter->rx_ring[i]; 4452 4453 if (frame_size > ixgbevf_rx_bufsz(ring)) 4454 return -EINVAL; 4455 } 4456 4457 old_prog = xchg(&adapter->xdp_prog, prog); 4458 4459 /* If transitioning XDP modes reconfigure rings */ 4460 if (!!prog != !!old_prog) { 4461 /* Hardware has to reinitialize queues and interrupts to 4462 * match packet buffer alignment. Unfortunately, the 4463 * hardware is not flexible enough to do this dynamically. 4464 */ 4465 if (netif_running(dev)) 4466 ixgbevf_close(dev); 4467 4468 ixgbevf_clear_interrupt_scheme(adapter); 4469 ixgbevf_init_interrupt_scheme(adapter); 4470 4471 if (netif_running(dev)) 4472 ixgbevf_open(dev); 4473 } else { 4474 for (i = 0; i < adapter->num_rx_queues; i++) 4475 xchg(&adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog); 4476 } 4477 4478 if (old_prog) 4479 bpf_prog_put(old_prog); 4480 4481 return 0; 4482 } 4483 4484 static int ixgbevf_xdp(struct net_device *dev, struct netdev_bpf *xdp) 4485 { 4486 struct ixgbevf_adapter *adapter = netdev_priv(dev); 4487 4488 switch (xdp->command) { 4489 case XDP_SETUP_PROG: 4490 return ixgbevf_xdp_setup(dev, xdp->prog); 4491 case XDP_QUERY_PROG: 4492 xdp->prog_id = adapter->xdp_prog ? 4493 adapter->xdp_prog->aux->id : 0; 4494 return 0; 4495 default: 4496 return -EINVAL; 4497 } 4498 } 4499 4500 static const struct net_device_ops ixgbevf_netdev_ops = { 4501 .ndo_open = ixgbevf_open, 4502 .ndo_stop = ixgbevf_close, 4503 .ndo_start_xmit = ixgbevf_xmit_frame, 4504 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 4505 .ndo_get_stats64 = ixgbevf_get_stats, 4506 .ndo_validate_addr = eth_validate_addr, 4507 .ndo_set_mac_address = ixgbevf_set_mac, 4508 .ndo_change_mtu = ixgbevf_change_mtu, 4509 .ndo_tx_timeout = ixgbevf_tx_timeout, 4510 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 4511 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 4512 .ndo_features_check = ixgbevf_features_check, 4513 .ndo_bpf = ixgbevf_xdp, 4514 }; 4515 4516 static void ixgbevf_assign_netdev_ops(struct net_device *dev) 4517 { 4518 dev->netdev_ops = &ixgbevf_netdev_ops; 4519 ixgbevf_set_ethtool_ops(dev); 4520 dev->watchdog_timeo = 5 * HZ; 4521 } 4522 4523 /** 4524 * ixgbevf_probe - Device Initialization Routine 4525 * @pdev: PCI device information struct 4526 * @ent: entry in ixgbevf_pci_tbl 4527 * 4528 * Returns 0 on success, negative on failure 4529 * 4530 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 4531 * The OS initialization, configuring of the adapter private structure, 4532 * and a hardware reset occur. 4533 **/ 4534 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 4535 { 4536 struct net_device *netdev; 4537 struct ixgbevf_adapter *adapter = NULL; 4538 struct ixgbe_hw *hw = NULL; 4539 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 4540 int err, pci_using_dac; 4541 bool disable_dev = false; 4542 4543 err = pci_enable_device(pdev); 4544 if (err) 4545 return err; 4546 4547 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 4548 pci_using_dac = 1; 4549 } else { 4550 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 4551 if (err) { 4552 dev_err(&pdev->dev, "No usable DMA configuration, aborting\n"); 4553 goto err_dma; 4554 } 4555 pci_using_dac = 0; 4556 } 4557 4558 err = pci_request_regions(pdev, ixgbevf_driver_name); 4559 if (err) { 4560 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 4561 goto err_pci_reg; 4562 } 4563 4564 pci_set_master(pdev); 4565 4566 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 4567 MAX_TX_QUEUES); 4568 if (!netdev) { 4569 err = -ENOMEM; 4570 goto err_alloc_etherdev; 4571 } 4572 4573 SET_NETDEV_DEV(netdev, &pdev->dev); 4574 4575 adapter = netdev_priv(netdev); 4576 4577 adapter->netdev = netdev; 4578 adapter->pdev = pdev; 4579 hw = &adapter->hw; 4580 hw->back = adapter; 4581 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 4582 4583 /* call save state here in standalone driver because it relies on 4584 * adapter struct to exist, and needs to call netdev_priv 4585 */ 4586 pci_save_state(pdev); 4587 4588 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 4589 pci_resource_len(pdev, 0)); 4590 adapter->io_addr = hw->hw_addr; 4591 if (!hw->hw_addr) { 4592 err = -EIO; 4593 goto err_ioremap; 4594 } 4595 4596 ixgbevf_assign_netdev_ops(netdev); 4597 4598 /* Setup HW API */ 4599 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 4600 hw->mac.type = ii->mac; 4601 4602 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 4603 sizeof(struct ixgbe_mbx_operations)); 4604 4605 /* setup the private structure */ 4606 err = ixgbevf_sw_init(adapter); 4607 if (err) 4608 goto err_sw_init; 4609 4610 /* The HW MAC address was set and/or determined in sw_init */ 4611 if (!is_valid_ether_addr(netdev->dev_addr)) { 4612 pr_err("invalid MAC address\n"); 4613 err = -EIO; 4614 goto err_sw_init; 4615 } 4616 4617 netdev->hw_features = NETIF_F_SG | 4618 NETIF_F_TSO | 4619 NETIF_F_TSO6 | 4620 NETIF_F_RXCSUM | 4621 NETIF_F_HW_CSUM | 4622 NETIF_F_SCTP_CRC; 4623 4624 #define IXGBEVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 4625 NETIF_F_GSO_GRE_CSUM | \ 4626 NETIF_F_GSO_IPXIP4 | \ 4627 NETIF_F_GSO_IPXIP6 | \ 4628 NETIF_F_GSO_UDP_TUNNEL | \ 4629 NETIF_F_GSO_UDP_TUNNEL_CSUM) 4630 4631 netdev->gso_partial_features = IXGBEVF_GSO_PARTIAL_FEATURES; 4632 netdev->hw_features |= NETIF_F_GSO_PARTIAL | 4633 IXGBEVF_GSO_PARTIAL_FEATURES; 4634 4635 netdev->features = netdev->hw_features; 4636 4637 if (pci_using_dac) 4638 netdev->features |= NETIF_F_HIGHDMA; 4639 4640 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; 4641 netdev->mpls_features |= NETIF_F_SG | 4642 NETIF_F_TSO | 4643 NETIF_F_TSO6 | 4644 NETIF_F_HW_CSUM; 4645 netdev->mpls_features |= IXGBEVF_GSO_PARTIAL_FEATURES; 4646 netdev->hw_enc_features |= netdev->vlan_features; 4647 4648 /* set this bit last since it cannot be part of vlan_features */ 4649 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | 4650 NETIF_F_HW_VLAN_CTAG_RX | 4651 NETIF_F_HW_VLAN_CTAG_TX; 4652 4653 netdev->priv_flags |= IFF_UNICAST_FLT; 4654 4655 /* MTU range: 68 - 1504 or 9710 */ 4656 netdev->min_mtu = ETH_MIN_MTU; 4657 switch (adapter->hw.api_version) { 4658 case ixgbe_mbox_api_11: 4659 case ixgbe_mbox_api_12: 4660 case ixgbe_mbox_api_13: 4661 case ixgbe_mbox_api_14: 4662 netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - 4663 (ETH_HLEN + ETH_FCS_LEN); 4664 break; 4665 default: 4666 if (adapter->hw.mac.type != ixgbe_mac_82599_vf) 4667 netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - 4668 (ETH_HLEN + ETH_FCS_LEN); 4669 else 4670 netdev->max_mtu = ETH_DATA_LEN + ETH_FCS_LEN; 4671 break; 4672 } 4673 4674 if (IXGBE_REMOVED(hw->hw_addr)) { 4675 err = -EIO; 4676 goto err_sw_init; 4677 } 4678 4679 timer_setup(&adapter->service_timer, ixgbevf_service_timer, 0); 4680 4681 INIT_WORK(&adapter->service_task, ixgbevf_service_task); 4682 set_bit(__IXGBEVF_SERVICE_INITED, &adapter->state); 4683 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 4684 4685 err = ixgbevf_init_interrupt_scheme(adapter); 4686 if (err) 4687 goto err_sw_init; 4688 4689 strcpy(netdev->name, "eth%d"); 4690 4691 err = register_netdev(netdev); 4692 if (err) 4693 goto err_register; 4694 4695 pci_set_drvdata(pdev, netdev); 4696 netif_carrier_off(netdev); 4697 ixgbevf_init_ipsec_offload(adapter); 4698 4699 ixgbevf_init_last_counter_stats(adapter); 4700 4701 /* print the VF info */ 4702 dev_info(&pdev->dev, "%pM\n", netdev->dev_addr); 4703 dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type); 4704 4705 switch (hw->mac.type) { 4706 case ixgbe_mac_X550_vf: 4707 dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n"); 4708 break; 4709 case ixgbe_mac_X540_vf: 4710 dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n"); 4711 break; 4712 case ixgbe_mac_82599_vf: 4713 default: 4714 dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n"); 4715 break; 4716 } 4717 4718 return 0; 4719 4720 err_register: 4721 ixgbevf_clear_interrupt_scheme(adapter); 4722 err_sw_init: 4723 ixgbevf_reset_interrupt_capability(adapter); 4724 iounmap(adapter->io_addr); 4725 kfree(adapter->rss_key); 4726 err_ioremap: 4727 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4728 free_netdev(netdev); 4729 err_alloc_etherdev: 4730 pci_release_regions(pdev); 4731 err_pci_reg: 4732 err_dma: 4733 if (!adapter || disable_dev) 4734 pci_disable_device(pdev); 4735 return err; 4736 } 4737 4738 /** 4739 * ixgbevf_remove - Device Removal Routine 4740 * @pdev: PCI device information struct 4741 * 4742 * ixgbevf_remove is called by the PCI subsystem to alert the driver 4743 * that it should release a PCI device. The could be caused by a 4744 * Hot-Plug event, or because the driver is going to be removed from 4745 * memory. 4746 **/ 4747 static void ixgbevf_remove(struct pci_dev *pdev) 4748 { 4749 struct net_device *netdev = pci_get_drvdata(pdev); 4750 struct ixgbevf_adapter *adapter; 4751 bool disable_dev; 4752 4753 if (!netdev) 4754 return; 4755 4756 adapter = netdev_priv(netdev); 4757 4758 set_bit(__IXGBEVF_REMOVING, &adapter->state); 4759 cancel_work_sync(&adapter->service_task); 4760 4761 if (netdev->reg_state == NETREG_REGISTERED) 4762 unregister_netdev(netdev); 4763 4764 ixgbevf_stop_ipsec_offload(adapter); 4765 ixgbevf_clear_interrupt_scheme(adapter); 4766 ixgbevf_reset_interrupt_capability(adapter); 4767 4768 iounmap(adapter->io_addr); 4769 pci_release_regions(pdev); 4770 4771 hw_dbg(&adapter->hw, "Remove complete\n"); 4772 4773 kfree(adapter->rss_key); 4774 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4775 free_netdev(netdev); 4776 4777 if (disable_dev) 4778 pci_disable_device(pdev); 4779 } 4780 4781 /** 4782 * ixgbevf_io_error_detected - called when PCI error is detected 4783 * @pdev: Pointer to PCI device 4784 * @state: The current pci connection state 4785 * 4786 * This function is called after a PCI bus error affecting 4787 * this device has been detected. 4788 **/ 4789 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, 4790 pci_channel_state_t state) 4791 { 4792 struct net_device *netdev = pci_get_drvdata(pdev); 4793 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4794 4795 if (!test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 4796 return PCI_ERS_RESULT_DISCONNECT; 4797 4798 rtnl_lock(); 4799 netif_device_detach(netdev); 4800 4801 if (netif_running(netdev)) 4802 ixgbevf_close_suspend(adapter); 4803 4804 if (state == pci_channel_io_perm_failure) { 4805 rtnl_unlock(); 4806 return PCI_ERS_RESULT_DISCONNECT; 4807 } 4808 4809 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 4810 pci_disable_device(pdev); 4811 rtnl_unlock(); 4812 4813 /* Request a slot slot reset. */ 4814 return PCI_ERS_RESULT_NEED_RESET; 4815 } 4816 4817 /** 4818 * ixgbevf_io_slot_reset - called after the pci bus has been reset. 4819 * @pdev: Pointer to PCI device 4820 * 4821 * Restart the card from scratch, as if from a cold-boot. Implementation 4822 * resembles the first-half of the ixgbevf_resume routine. 4823 **/ 4824 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) 4825 { 4826 struct net_device *netdev = pci_get_drvdata(pdev); 4827 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4828 4829 if (pci_enable_device_mem(pdev)) { 4830 dev_err(&pdev->dev, 4831 "Cannot re-enable PCI device after reset.\n"); 4832 return PCI_ERS_RESULT_DISCONNECT; 4833 } 4834 4835 adapter->hw.hw_addr = adapter->io_addr; 4836 smp_mb__before_atomic(); 4837 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 4838 pci_set_master(pdev); 4839 4840 ixgbevf_reset(adapter); 4841 4842 return PCI_ERS_RESULT_RECOVERED; 4843 } 4844 4845 /** 4846 * ixgbevf_io_resume - called when traffic can start flowing again. 4847 * @pdev: Pointer to PCI device 4848 * 4849 * This callback is called when the error recovery driver tells us that 4850 * its OK to resume normal operation. Implementation resembles the 4851 * second-half of the ixgbevf_resume routine. 4852 **/ 4853 static void ixgbevf_io_resume(struct pci_dev *pdev) 4854 { 4855 struct net_device *netdev = pci_get_drvdata(pdev); 4856 4857 rtnl_lock(); 4858 if (netif_running(netdev)) 4859 ixgbevf_open(netdev); 4860 4861 netif_device_attach(netdev); 4862 rtnl_unlock(); 4863 } 4864 4865 /* PCI Error Recovery (ERS) */ 4866 static const struct pci_error_handlers ixgbevf_err_handler = { 4867 .error_detected = ixgbevf_io_error_detected, 4868 .slot_reset = ixgbevf_io_slot_reset, 4869 .resume = ixgbevf_io_resume, 4870 }; 4871 4872 static struct pci_driver ixgbevf_driver = { 4873 .name = ixgbevf_driver_name, 4874 .id_table = ixgbevf_pci_tbl, 4875 .probe = ixgbevf_probe, 4876 .remove = ixgbevf_remove, 4877 #ifdef CONFIG_PM 4878 /* Power Management Hooks */ 4879 .suspend = ixgbevf_suspend, 4880 .resume = ixgbevf_resume, 4881 #endif 4882 .shutdown = ixgbevf_shutdown, 4883 .err_handler = &ixgbevf_err_handler 4884 }; 4885 4886 /** 4887 * ixgbevf_init_module - Driver Registration Routine 4888 * 4889 * ixgbevf_init_module is the first routine called when the driver is 4890 * loaded. All it does is register with the PCI subsystem. 4891 **/ 4892 static int __init ixgbevf_init_module(void) 4893 { 4894 pr_info("%s - version %s\n", ixgbevf_driver_string, 4895 ixgbevf_driver_version); 4896 4897 pr_info("%s\n", ixgbevf_copyright); 4898 ixgbevf_wq = create_singlethread_workqueue(ixgbevf_driver_name); 4899 if (!ixgbevf_wq) { 4900 pr_err("%s: Failed to create workqueue\n", ixgbevf_driver_name); 4901 return -ENOMEM; 4902 } 4903 4904 return pci_register_driver(&ixgbevf_driver); 4905 } 4906 4907 module_init(ixgbevf_init_module); 4908 4909 /** 4910 * ixgbevf_exit_module - Driver Exit Cleanup Routine 4911 * 4912 * ixgbevf_exit_module is called just before the driver is removed 4913 * from memory. 4914 **/ 4915 static void __exit ixgbevf_exit_module(void) 4916 { 4917 pci_unregister_driver(&ixgbevf_driver); 4918 if (ixgbevf_wq) { 4919 destroy_workqueue(ixgbevf_wq); 4920 ixgbevf_wq = NULL; 4921 } 4922 } 4923 4924 #ifdef DEBUG 4925 /** 4926 * ixgbevf_get_hw_dev_name - return device name string 4927 * used by hardware layer to print debugging information 4928 * @hw: pointer to private hardware struct 4929 **/ 4930 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 4931 { 4932 struct ixgbevf_adapter *adapter = hw->back; 4933 4934 return adapter->netdev->name; 4935 } 4936 4937 #endif 4938 module_exit(ixgbevf_exit_module); 4939 4940 /* ixgbevf_main.c */ 4941