1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 1999 - 2018 Intel Corporation. */ 3 4 /****************************************************************************** 5 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 6 ******************************************************************************/ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 10 #include <linux/types.h> 11 #include <linux/bitops.h> 12 #include <linux/module.h> 13 #include <linux/pci.h> 14 #include <linux/netdevice.h> 15 #include <linux/vmalloc.h> 16 #include <linux/string.h> 17 #include <linux/in.h> 18 #include <linux/ip.h> 19 #include <linux/tcp.h> 20 #include <linux/sctp.h> 21 #include <linux/ipv6.h> 22 #include <linux/slab.h> 23 #include <net/checksum.h> 24 #include <net/ip6_checksum.h> 25 #include <linux/ethtool.h> 26 #include <linux/if.h> 27 #include <linux/if_vlan.h> 28 #include <linux/prefetch.h> 29 #include <net/mpls.h> 30 #include <linux/bpf.h> 31 #include <linux/bpf_trace.h> 32 #include <linux/atomic.h> 33 34 #include "ixgbevf.h" 35 36 const char ixgbevf_driver_name[] = "ixgbevf"; 37 static const char ixgbevf_driver_string[] = 38 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 39 40 #define DRV_VERSION "4.1.0-k" 41 const char ixgbevf_driver_version[] = DRV_VERSION; 42 static char ixgbevf_copyright[] = 43 "Copyright (c) 2009 - 2018 Intel Corporation."; 44 45 static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 46 [board_82599_vf] = &ixgbevf_82599_vf_info, 47 [board_82599_vf_hv] = &ixgbevf_82599_vf_hv_info, 48 [board_X540_vf] = &ixgbevf_X540_vf_info, 49 [board_X540_vf_hv] = &ixgbevf_X540_vf_hv_info, 50 [board_X550_vf] = &ixgbevf_X550_vf_info, 51 [board_X550_vf_hv] = &ixgbevf_X550_vf_hv_info, 52 [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info, 53 [board_X550EM_x_vf_hv] = &ixgbevf_X550EM_x_vf_hv_info, 54 [board_x550em_a_vf] = &ixgbevf_x550em_a_vf_info, 55 }; 56 57 /* ixgbevf_pci_tbl - PCI Device ID Table 58 * 59 * Wildcard entries (PCI_ANY_ID) should come last 60 * Last entry must be all 0s 61 * 62 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 63 * Class, Class Mask, private data (not used) } 64 */ 65 static const struct pci_device_id ixgbevf_pci_tbl[] = { 66 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf }, 67 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF_HV), board_82599_vf_hv }, 68 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf }, 69 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF_HV), board_X540_vf_hv }, 70 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf }, 71 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF_HV), board_X550_vf_hv }, 72 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf }, 73 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF_HV), board_X550EM_x_vf_hv}, 74 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_VF), board_x550em_a_vf }, 75 /* required last entry */ 76 {0, } 77 }; 78 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 79 80 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 81 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); 82 MODULE_LICENSE("GPL v2"); 83 MODULE_VERSION(DRV_VERSION); 84 85 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 86 static int debug = -1; 87 module_param(debug, int, 0); 88 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 89 90 static struct workqueue_struct *ixgbevf_wq; 91 92 static void ixgbevf_service_event_schedule(struct ixgbevf_adapter *adapter) 93 { 94 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 95 !test_bit(__IXGBEVF_REMOVING, &adapter->state) && 96 !test_and_set_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)) 97 queue_work(ixgbevf_wq, &adapter->service_task); 98 } 99 100 static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter) 101 { 102 BUG_ON(!test_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)); 103 104 /* flush memory to make sure state is correct before next watchdog */ 105 smp_mb__before_atomic(); 106 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 107 } 108 109 /* forward decls */ 110 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); 111 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); 112 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); 113 static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer); 114 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, 115 struct ixgbevf_rx_buffer *old_buff); 116 117 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) 118 { 119 struct ixgbevf_adapter *adapter = hw->back; 120 121 if (!hw->hw_addr) 122 return; 123 hw->hw_addr = NULL; 124 dev_err(&adapter->pdev->dev, "Adapter removed\n"); 125 if (test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 126 ixgbevf_service_event_schedule(adapter); 127 } 128 129 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg) 130 { 131 u32 value; 132 133 /* The following check not only optimizes a bit by not 134 * performing a read on the status register when the 135 * register just read was a status register read that 136 * returned IXGBE_FAILED_READ_REG. It also blocks any 137 * potential recursion. 138 */ 139 if (reg == IXGBE_VFSTATUS) { 140 ixgbevf_remove_adapter(hw); 141 return; 142 } 143 value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS); 144 if (value == IXGBE_FAILED_READ_REG) 145 ixgbevf_remove_adapter(hw); 146 } 147 148 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg) 149 { 150 u8 __iomem *reg_addr = READ_ONCE(hw->hw_addr); 151 u32 value; 152 153 if (IXGBE_REMOVED(reg_addr)) 154 return IXGBE_FAILED_READ_REG; 155 value = readl(reg_addr + reg); 156 if (unlikely(value == IXGBE_FAILED_READ_REG)) 157 ixgbevf_check_remove(hw, reg); 158 return value; 159 } 160 161 /** 162 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 163 * @adapter: pointer to adapter struct 164 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 165 * @queue: queue to map the corresponding interrupt to 166 * @msix_vector: the vector to map to the corresponding queue 167 **/ 168 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 169 u8 queue, u8 msix_vector) 170 { 171 u32 ivar, index; 172 struct ixgbe_hw *hw = &adapter->hw; 173 174 if (direction == -1) { 175 /* other causes */ 176 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 177 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 178 ivar &= ~0xFF; 179 ivar |= msix_vector; 180 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 181 } else { 182 /* Tx or Rx causes */ 183 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 184 index = ((16 * (queue & 1)) + (8 * direction)); 185 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 186 ivar &= ~(0xFF << index); 187 ivar |= (msix_vector << index); 188 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 189 } 190 } 191 192 static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring) 193 { 194 return ring->stats.packets; 195 } 196 197 static u32 ixgbevf_get_tx_pending(struct ixgbevf_ring *ring) 198 { 199 struct ixgbevf_adapter *adapter = netdev_priv(ring->netdev); 200 struct ixgbe_hw *hw = &adapter->hw; 201 202 u32 head = IXGBE_READ_REG(hw, IXGBE_VFTDH(ring->reg_idx)); 203 u32 tail = IXGBE_READ_REG(hw, IXGBE_VFTDT(ring->reg_idx)); 204 205 if (head != tail) 206 return (head < tail) ? 207 tail - head : (tail + ring->count - head); 208 209 return 0; 210 } 211 212 static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring) 213 { 214 u32 tx_done = ixgbevf_get_tx_completed(tx_ring); 215 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 216 u32 tx_pending = ixgbevf_get_tx_pending(tx_ring); 217 218 clear_check_for_tx_hang(tx_ring); 219 220 /* Check for a hung queue, but be thorough. This verifies 221 * that a transmit has been completed since the previous 222 * check AND there is at least one packet pending. The 223 * ARMED bit is set to indicate a potential hang. 224 */ 225 if ((tx_done_old == tx_done) && tx_pending) { 226 /* make sure it is true for two checks in a row */ 227 return test_and_set_bit(__IXGBEVF_HANG_CHECK_ARMED, 228 &tx_ring->state); 229 } 230 /* reset the countdown */ 231 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &tx_ring->state); 232 233 /* update completed stats and continue */ 234 tx_ring->tx_stats.tx_done_old = tx_done; 235 236 return false; 237 } 238 239 static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter) 240 { 241 /* Do the reset outside of interrupt context */ 242 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 243 set_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state); 244 ixgbevf_service_event_schedule(adapter); 245 } 246 } 247 248 /** 249 * ixgbevf_tx_timeout - Respond to a Tx Hang 250 * @netdev: network interface device structure 251 **/ 252 static void ixgbevf_tx_timeout(struct net_device *netdev) 253 { 254 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 255 256 ixgbevf_tx_timeout_reset(adapter); 257 } 258 259 /** 260 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 261 * @q_vector: board private structure 262 * @tx_ring: tx ring to clean 263 * @napi_budget: Used to determine if we are in netpoll 264 **/ 265 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, 266 struct ixgbevf_ring *tx_ring, int napi_budget) 267 { 268 struct ixgbevf_adapter *adapter = q_vector->adapter; 269 struct ixgbevf_tx_buffer *tx_buffer; 270 union ixgbe_adv_tx_desc *tx_desc; 271 unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0; 272 unsigned int budget = tx_ring->count / 2; 273 unsigned int i = tx_ring->next_to_clean; 274 275 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 276 return true; 277 278 tx_buffer = &tx_ring->tx_buffer_info[i]; 279 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 280 i -= tx_ring->count; 281 282 do { 283 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 284 285 /* if next_to_watch is not set then there is no work pending */ 286 if (!eop_desc) 287 break; 288 289 /* prevent any other reads prior to eop_desc */ 290 smp_rmb(); 291 292 /* if DD is not set pending work has not been completed */ 293 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 294 break; 295 296 /* clear next_to_watch to prevent false hangs */ 297 tx_buffer->next_to_watch = NULL; 298 299 /* update the statistics for this packet */ 300 total_bytes += tx_buffer->bytecount; 301 total_packets += tx_buffer->gso_segs; 302 if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC) 303 total_ipsec++; 304 305 /* free the skb */ 306 if (ring_is_xdp(tx_ring)) 307 page_frag_free(tx_buffer->data); 308 else 309 napi_consume_skb(tx_buffer->skb, napi_budget); 310 311 /* unmap skb header data */ 312 dma_unmap_single(tx_ring->dev, 313 dma_unmap_addr(tx_buffer, dma), 314 dma_unmap_len(tx_buffer, len), 315 DMA_TO_DEVICE); 316 317 /* clear tx_buffer data */ 318 dma_unmap_len_set(tx_buffer, len, 0); 319 320 /* unmap remaining buffers */ 321 while (tx_desc != eop_desc) { 322 tx_buffer++; 323 tx_desc++; 324 i++; 325 if (unlikely(!i)) { 326 i -= tx_ring->count; 327 tx_buffer = tx_ring->tx_buffer_info; 328 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 329 } 330 331 /* unmap any remaining paged data */ 332 if (dma_unmap_len(tx_buffer, len)) { 333 dma_unmap_page(tx_ring->dev, 334 dma_unmap_addr(tx_buffer, dma), 335 dma_unmap_len(tx_buffer, len), 336 DMA_TO_DEVICE); 337 dma_unmap_len_set(tx_buffer, len, 0); 338 } 339 } 340 341 /* move us one more past the eop_desc for start of next pkt */ 342 tx_buffer++; 343 tx_desc++; 344 i++; 345 if (unlikely(!i)) { 346 i -= tx_ring->count; 347 tx_buffer = tx_ring->tx_buffer_info; 348 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 349 } 350 351 /* issue prefetch for next Tx descriptor */ 352 prefetch(tx_desc); 353 354 /* update budget accounting */ 355 budget--; 356 } while (likely(budget)); 357 358 i += tx_ring->count; 359 tx_ring->next_to_clean = i; 360 u64_stats_update_begin(&tx_ring->syncp); 361 tx_ring->stats.bytes += total_bytes; 362 tx_ring->stats.packets += total_packets; 363 u64_stats_update_end(&tx_ring->syncp); 364 q_vector->tx.total_bytes += total_bytes; 365 q_vector->tx.total_packets += total_packets; 366 adapter->tx_ipsec += total_ipsec; 367 368 if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) { 369 struct ixgbe_hw *hw = &adapter->hw; 370 union ixgbe_adv_tx_desc *eop_desc; 371 372 eop_desc = tx_ring->tx_buffer_info[i].next_to_watch; 373 374 pr_err("Detected Tx Unit Hang%s\n" 375 " Tx Queue <%d>\n" 376 " TDH, TDT <%x>, <%x>\n" 377 " next_to_use <%x>\n" 378 " next_to_clean <%x>\n" 379 "tx_buffer_info[next_to_clean]\n" 380 " next_to_watch <%p>\n" 381 " eop_desc->wb.status <%x>\n" 382 " time_stamp <%lx>\n" 383 " jiffies <%lx>\n", 384 ring_is_xdp(tx_ring) ? " XDP" : "", 385 tx_ring->queue_index, 386 IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)), 387 IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)), 388 tx_ring->next_to_use, i, 389 eop_desc, (eop_desc ? eop_desc->wb.status : 0), 390 tx_ring->tx_buffer_info[i].time_stamp, jiffies); 391 392 if (!ring_is_xdp(tx_ring)) 393 netif_stop_subqueue(tx_ring->netdev, 394 tx_ring->queue_index); 395 396 /* schedule immediate reset if we believe we hung */ 397 ixgbevf_tx_timeout_reset(adapter); 398 399 return true; 400 } 401 402 if (ring_is_xdp(tx_ring)) 403 return !!budget; 404 405 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 406 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 407 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 408 /* Make sure that anybody stopping the queue after this 409 * sees the new next_to_clean. 410 */ 411 smp_mb(); 412 413 if (__netif_subqueue_stopped(tx_ring->netdev, 414 tx_ring->queue_index) && 415 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 416 netif_wake_subqueue(tx_ring->netdev, 417 tx_ring->queue_index); 418 ++tx_ring->tx_stats.restart_queue; 419 } 420 } 421 422 return !!budget; 423 } 424 425 /** 426 * ixgbevf_rx_skb - Helper function to determine proper Rx method 427 * @q_vector: structure containing interrupt and ring information 428 * @skb: packet to send up 429 **/ 430 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, 431 struct sk_buff *skb) 432 { 433 napi_gro_receive(&q_vector->napi, skb); 434 } 435 436 #define IXGBE_RSS_L4_TYPES_MASK \ 437 ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ 438 (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ 439 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ 440 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) 441 442 static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring, 443 union ixgbe_adv_rx_desc *rx_desc, 444 struct sk_buff *skb) 445 { 446 u16 rss_type; 447 448 if (!(ring->netdev->features & NETIF_F_RXHASH)) 449 return; 450 451 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & 452 IXGBE_RXDADV_RSSTYPE_MASK; 453 454 if (!rss_type) 455 return; 456 457 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 458 (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? 459 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); 460 } 461 462 /** 463 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 464 * @ring: structure containig ring specific data 465 * @rx_desc: current Rx descriptor being processed 466 * @skb: skb currently being received and modified 467 **/ 468 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, 469 union ixgbe_adv_rx_desc *rx_desc, 470 struct sk_buff *skb) 471 { 472 skb_checksum_none_assert(skb); 473 474 /* Rx csum disabled */ 475 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 476 return; 477 478 /* if IP and error */ 479 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 480 ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 481 ring->rx_stats.csum_err++; 482 return; 483 } 484 485 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 486 return; 487 488 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 489 ring->rx_stats.csum_err++; 490 return; 491 } 492 493 /* It must be a TCP or UDP packet with a valid checksum */ 494 skb->ip_summed = CHECKSUM_UNNECESSARY; 495 } 496 497 /** 498 * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor 499 * @rx_ring: rx descriptor ring packet is being transacted on 500 * @rx_desc: pointer to the EOP Rx descriptor 501 * @skb: pointer to current skb being populated 502 * 503 * This function checks the ring, descriptor, and packet information in 504 * order to populate the checksum, VLAN, protocol, and other fields within 505 * the skb. 506 **/ 507 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, 508 union ixgbe_adv_rx_desc *rx_desc, 509 struct sk_buff *skb) 510 { 511 ixgbevf_rx_hash(rx_ring, rx_desc, skb); 512 ixgbevf_rx_checksum(rx_ring, rx_desc, skb); 513 514 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 515 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 516 unsigned long *active_vlans = netdev_priv(rx_ring->netdev); 517 518 if (test_bit(vid & VLAN_VID_MASK, active_vlans)) 519 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 520 } 521 522 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP)) 523 ixgbevf_ipsec_rx(rx_ring, rx_desc, skb); 524 525 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 526 } 527 528 static 529 struct ixgbevf_rx_buffer *ixgbevf_get_rx_buffer(struct ixgbevf_ring *rx_ring, 530 const unsigned int size) 531 { 532 struct ixgbevf_rx_buffer *rx_buffer; 533 534 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 535 prefetchw(rx_buffer->page); 536 537 /* we are reusing so sync this buffer for CPU use */ 538 dma_sync_single_range_for_cpu(rx_ring->dev, 539 rx_buffer->dma, 540 rx_buffer->page_offset, 541 size, 542 DMA_FROM_DEVICE); 543 544 rx_buffer->pagecnt_bias--; 545 546 return rx_buffer; 547 } 548 549 static void ixgbevf_put_rx_buffer(struct ixgbevf_ring *rx_ring, 550 struct ixgbevf_rx_buffer *rx_buffer, 551 struct sk_buff *skb) 552 { 553 if (ixgbevf_can_reuse_rx_page(rx_buffer)) { 554 /* hand second half of page back to the ring */ 555 ixgbevf_reuse_rx_page(rx_ring, rx_buffer); 556 } else { 557 if (IS_ERR(skb)) 558 /* We are not reusing the buffer so unmap it and free 559 * any references we are holding to it 560 */ 561 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 562 ixgbevf_rx_pg_size(rx_ring), 563 DMA_FROM_DEVICE, 564 IXGBEVF_RX_DMA_ATTR); 565 __page_frag_cache_drain(rx_buffer->page, 566 rx_buffer->pagecnt_bias); 567 } 568 569 /* clear contents of rx_buffer */ 570 rx_buffer->page = NULL; 571 } 572 573 /** 574 * ixgbevf_is_non_eop - process handling of non-EOP buffers 575 * @rx_ring: Rx ring being processed 576 * @rx_desc: Rx descriptor for current buffer 577 * 578 * This function updates next to clean. If the buffer is an EOP buffer 579 * this function exits returning false, otherwise it will place the 580 * sk_buff in the next buffer to be chained and return true indicating 581 * that this is in fact a non-EOP buffer. 582 **/ 583 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, 584 union ixgbe_adv_rx_desc *rx_desc) 585 { 586 u32 ntc = rx_ring->next_to_clean + 1; 587 588 /* fetch, update, and store next to clean */ 589 ntc = (ntc < rx_ring->count) ? ntc : 0; 590 rx_ring->next_to_clean = ntc; 591 592 prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); 593 594 if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 595 return false; 596 597 return true; 598 } 599 600 static inline unsigned int ixgbevf_rx_offset(struct ixgbevf_ring *rx_ring) 601 { 602 return ring_uses_build_skb(rx_ring) ? IXGBEVF_SKB_PAD : 0; 603 } 604 605 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, 606 struct ixgbevf_rx_buffer *bi) 607 { 608 struct page *page = bi->page; 609 dma_addr_t dma; 610 611 /* since we are recycling buffers we should seldom need to alloc */ 612 if (likely(page)) 613 return true; 614 615 /* alloc new page for storage */ 616 page = dev_alloc_pages(ixgbevf_rx_pg_order(rx_ring)); 617 if (unlikely(!page)) { 618 rx_ring->rx_stats.alloc_rx_page_failed++; 619 return false; 620 } 621 622 /* map page for use */ 623 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 624 ixgbevf_rx_pg_size(rx_ring), 625 DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR); 626 627 /* if mapping failed free memory back to system since 628 * there isn't much point in holding memory we can't use 629 */ 630 if (dma_mapping_error(rx_ring->dev, dma)) { 631 __free_pages(page, ixgbevf_rx_pg_order(rx_ring)); 632 633 rx_ring->rx_stats.alloc_rx_page_failed++; 634 return false; 635 } 636 637 bi->dma = dma; 638 bi->page = page; 639 bi->page_offset = ixgbevf_rx_offset(rx_ring); 640 bi->pagecnt_bias = 1; 641 rx_ring->rx_stats.alloc_rx_page++; 642 643 return true; 644 } 645 646 /** 647 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 648 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on 649 * @cleaned_count: number of buffers to replace 650 **/ 651 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, 652 u16 cleaned_count) 653 { 654 union ixgbe_adv_rx_desc *rx_desc; 655 struct ixgbevf_rx_buffer *bi; 656 unsigned int i = rx_ring->next_to_use; 657 658 /* nothing to do or no valid netdev defined */ 659 if (!cleaned_count || !rx_ring->netdev) 660 return; 661 662 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 663 bi = &rx_ring->rx_buffer_info[i]; 664 i -= rx_ring->count; 665 666 do { 667 if (!ixgbevf_alloc_mapped_page(rx_ring, bi)) 668 break; 669 670 /* sync the buffer for use by the device */ 671 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 672 bi->page_offset, 673 ixgbevf_rx_bufsz(rx_ring), 674 DMA_FROM_DEVICE); 675 676 /* Refresh the desc even if pkt_addr didn't change 677 * because each write-back erases this info. 678 */ 679 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 680 681 rx_desc++; 682 bi++; 683 i++; 684 if (unlikely(!i)) { 685 rx_desc = IXGBEVF_RX_DESC(rx_ring, 0); 686 bi = rx_ring->rx_buffer_info; 687 i -= rx_ring->count; 688 } 689 690 /* clear the length for the next_to_use descriptor */ 691 rx_desc->wb.upper.length = 0; 692 693 cleaned_count--; 694 } while (cleaned_count); 695 696 i += rx_ring->count; 697 698 if (rx_ring->next_to_use != i) { 699 /* record the next descriptor to use */ 700 rx_ring->next_to_use = i; 701 702 /* update next to alloc since we have filled the ring */ 703 rx_ring->next_to_alloc = i; 704 705 /* Force memory writes to complete before letting h/w 706 * know there are new descriptors to fetch. (Only 707 * applicable for weak-ordered memory model archs, 708 * such as IA-64). 709 */ 710 wmb(); 711 ixgbevf_write_tail(rx_ring, i); 712 } 713 } 714 715 /** 716 * ixgbevf_cleanup_headers - Correct corrupted or empty headers 717 * @rx_ring: rx descriptor ring packet is being transacted on 718 * @rx_desc: pointer to the EOP Rx descriptor 719 * @skb: pointer to current skb being fixed 720 * 721 * Check for corrupted packet headers caused by senders on the local L2 722 * embedded NIC switch not setting up their Tx Descriptors right. These 723 * should be very rare. 724 * 725 * Also address the case where we are pulling data in on pages only 726 * and as such no data is present in the skb header. 727 * 728 * In addition if skb is not at least 60 bytes we need to pad it so that 729 * it is large enough to qualify as a valid Ethernet frame. 730 * 731 * Returns true if an error was encountered and skb was freed. 732 **/ 733 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, 734 union ixgbe_adv_rx_desc *rx_desc, 735 struct sk_buff *skb) 736 { 737 /* XDP packets use error pointer so abort at this point */ 738 if (IS_ERR(skb)) 739 return true; 740 741 /* verify that the packet does not have any known errors */ 742 if (unlikely(ixgbevf_test_staterr(rx_desc, 743 IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { 744 struct net_device *netdev = rx_ring->netdev; 745 746 if (!(netdev->features & NETIF_F_RXALL)) { 747 dev_kfree_skb_any(skb); 748 return true; 749 } 750 } 751 752 /* if eth_skb_pad returns an error the skb was freed */ 753 if (eth_skb_pad(skb)) 754 return true; 755 756 return false; 757 } 758 759 /** 760 * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring 761 * @rx_ring: rx descriptor ring to store buffers on 762 * @old_buff: donor buffer to have page reused 763 * 764 * Synchronizes page for reuse by the adapter 765 **/ 766 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, 767 struct ixgbevf_rx_buffer *old_buff) 768 { 769 struct ixgbevf_rx_buffer *new_buff; 770 u16 nta = rx_ring->next_to_alloc; 771 772 new_buff = &rx_ring->rx_buffer_info[nta]; 773 774 /* update, and store next to alloc */ 775 nta++; 776 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 777 778 /* transfer page from old buffer to new buffer */ 779 new_buff->page = old_buff->page; 780 new_buff->dma = old_buff->dma; 781 new_buff->page_offset = old_buff->page_offset; 782 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 783 } 784 785 static inline bool ixgbevf_page_is_reserved(struct page *page) 786 { 787 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 788 } 789 790 static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer) 791 { 792 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 793 struct page *page = rx_buffer->page; 794 795 /* avoid re-using remote pages */ 796 if (unlikely(ixgbevf_page_is_reserved(page))) 797 return false; 798 799 #if (PAGE_SIZE < 8192) 800 /* if we are only owner of page we can reuse it */ 801 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 802 return false; 803 #else 804 #define IXGBEVF_LAST_OFFSET \ 805 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBEVF_RXBUFFER_2048) 806 807 if (rx_buffer->page_offset > IXGBEVF_LAST_OFFSET) 808 return false; 809 810 #endif 811 812 /* If we have drained the page fragment pool we need to update 813 * the pagecnt_bias and page count so that we fully restock the 814 * number of references the driver holds. 815 */ 816 if (unlikely(!pagecnt_bias)) { 817 page_ref_add(page, USHRT_MAX); 818 rx_buffer->pagecnt_bias = USHRT_MAX; 819 } 820 821 return true; 822 } 823 824 /** 825 * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff 826 * @rx_ring: rx descriptor ring to transact packets on 827 * @rx_buffer: buffer containing page to add 828 * @skb: sk_buff to place the data into 829 * @size: size of buffer to be added 830 * 831 * This function will add the data contained in rx_buffer->page to the skb. 832 **/ 833 static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, 834 struct ixgbevf_rx_buffer *rx_buffer, 835 struct sk_buff *skb, 836 unsigned int size) 837 { 838 #if (PAGE_SIZE < 8192) 839 unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; 840 #else 841 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 842 SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) : 843 SKB_DATA_ALIGN(size); 844 #endif 845 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 846 rx_buffer->page_offset, size, truesize); 847 #if (PAGE_SIZE < 8192) 848 rx_buffer->page_offset ^= truesize; 849 #else 850 rx_buffer->page_offset += truesize; 851 #endif 852 } 853 854 static 855 struct sk_buff *ixgbevf_construct_skb(struct ixgbevf_ring *rx_ring, 856 struct ixgbevf_rx_buffer *rx_buffer, 857 struct xdp_buff *xdp, 858 union ixgbe_adv_rx_desc *rx_desc) 859 { 860 unsigned int size = xdp->data_end - xdp->data; 861 #if (PAGE_SIZE < 8192) 862 unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; 863 #else 864 unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end - 865 xdp->data_hard_start); 866 #endif 867 unsigned int headlen; 868 struct sk_buff *skb; 869 870 /* prefetch first cache line of first page */ 871 prefetch(xdp->data); 872 #if L1_CACHE_BYTES < 128 873 prefetch(xdp->data + L1_CACHE_BYTES); 874 #endif 875 /* Note, we get here by enabling legacy-rx via: 876 * 877 * ethtool --set-priv-flags <dev> legacy-rx on 878 * 879 * In this mode, we currently get 0 extra XDP headroom as 880 * opposed to having legacy-rx off, where we process XDP 881 * packets going to stack via ixgbevf_build_skb(). 882 * 883 * For ixgbevf_construct_skb() mode it means that the 884 * xdp->data_meta will always point to xdp->data, since 885 * the helper cannot expand the head. Should this ever 886 * changed in future for legacy-rx mode on, then lets also 887 * add xdp->data_meta handling here. 888 */ 889 890 /* allocate a skb to store the frags */ 891 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBEVF_RX_HDR_SIZE); 892 if (unlikely(!skb)) 893 return NULL; 894 895 /* Determine available headroom for copy */ 896 headlen = size; 897 if (headlen > IXGBEVF_RX_HDR_SIZE) 898 headlen = eth_get_headlen(skb->dev, xdp->data, 899 IXGBEVF_RX_HDR_SIZE); 900 901 /* align pull length to size of long to optimize memcpy performance */ 902 memcpy(__skb_put(skb, headlen), xdp->data, 903 ALIGN(headlen, sizeof(long))); 904 905 /* update all of the pointers */ 906 size -= headlen; 907 if (size) { 908 skb_add_rx_frag(skb, 0, rx_buffer->page, 909 (xdp->data + headlen) - 910 page_address(rx_buffer->page), 911 size, truesize); 912 #if (PAGE_SIZE < 8192) 913 rx_buffer->page_offset ^= truesize; 914 #else 915 rx_buffer->page_offset += truesize; 916 #endif 917 } else { 918 rx_buffer->pagecnt_bias++; 919 } 920 921 return skb; 922 } 923 924 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 925 u32 qmask) 926 { 927 struct ixgbe_hw *hw = &adapter->hw; 928 929 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); 930 } 931 932 static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring, 933 struct ixgbevf_rx_buffer *rx_buffer, 934 struct xdp_buff *xdp, 935 union ixgbe_adv_rx_desc *rx_desc) 936 { 937 unsigned int metasize = xdp->data - xdp->data_meta; 938 #if (PAGE_SIZE < 8192) 939 unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; 940 #else 941 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 942 SKB_DATA_ALIGN(xdp->data_end - 943 xdp->data_hard_start); 944 #endif 945 struct sk_buff *skb; 946 947 /* Prefetch first cache line of first page. If xdp->data_meta 948 * is unused, this points to xdp->data, otherwise, we likely 949 * have a consumer accessing first few bytes of meta data, 950 * and then actual data. 951 */ 952 prefetch(xdp->data_meta); 953 #if L1_CACHE_BYTES < 128 954 prefetch(xdp->data_meta + L1_CACHE_BYTES); 955 #endif 956 957 /* build an skb around the page buffer */ 958 skb = build_skb(xdp->data_hard_start, truesize); 959 if (unlikely(!skb)) 960 return NULL; 961 962 /* update pointers within the skb to store the data */ 963 skb_reserve(skb, xdp->data - xdp->data_hard_start); 964 __skb_put(skb, xdp->data_end - xdp->data); 965 if (metasize) 966 skb_metadata_set(skb, metasize); 967 968 /* update buffer offset */ 969 #if (PAGE_SIZE < 8192) 970 rx_buffer->page_offset ^= truesize; 971 #else 972 rx_buffer->page_offset += truesize; 973 #endif 974 975 return skb; 976 } 977 978 #define IXGBEVF_XDP_PASS 0 979 #define IXGBEVF_XDP_CONSUMED 1 980 #define IXGBEVF_XDP_TX 2 981 982 static int ixgbevf_xmit_xdp_ring(struct ixgbevf_ring *ring, 983 struct xdp_buff *xdp) 984 { 985 struct ixgbevf_tx_buffer *tx_buffer; 986 union ixgbe_adv_tx_desc *tx_desc; 987 u32 len, cmd_type; 988 dma_addr_t dma; 989 u16 i; 990 991 len = xdp->data_end - xdp->data; 992 993 if (unlikely(!ixgbevf_desc_unused(ring))) 994 return IXGBEVF_XDP_CONSUMED; 995 996 dma = dma_map_single(ring->dev, xdp->data, len, DMA_TO_DEVICE); 997 if (dma_mapping_error(ring->dev, dma)) 998 return IXGBEVF_XDP_CONSUMED; 999 1000 /* record the location of the first descriptor for this packet */ 1001 i = ring->next_to_use; 1002 tx_buffer = &ring->tx_buffer_info[i]; 1003 1004 dma_unmap_len_set(tx_buffer, len, len); 1005 dma_unmap_addr_set(tx_buffer, dma, dma); 1006 tx_buffer->data = xdp->data; 1007 tx_buffer->bytecount = len; 1008 tx_buffer->gso_segs = 1; 1009 tx_buffer->protocol = 0; 1010 1011 /* Populate minimal context descriptor that will provide for the 1012 * fact that we are expected to process Ethernet frames. 1013 */ 1014 if (!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state)) { 1015 struct ixgbe_adv_tx_context_desc *context_desc; 1016 1017 set_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); 1018 1019 context_desc = IXGBEVF_TX_CTXTDESC(ring, 0); 1020 context_desc->vlan_macip_lens = 1021 cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT); 1022 context_desc->fceof_saidx = 0; 1023 context_desc->type_tucmd_mlhl = 1024 cpu_to_le32(IXGBE_TXD_CMD_DEXT | 1025 IXGBE_ADVTXD_DTYP_CTXT); 1026 context_desc->mss_l4len_idx = 0; 1027 1028 i = 1; 1029 } 1030 1031 /* put descriptor type bits */ 1032 cmd_type = IXGBE_ADVTXD_DTYP_DATA | 1033 IXGBE_ADVTXD_DCMD_DEXT | 1034 IXGBE_ADVTXD_DCMD_IFCS; 1035 cmd_type |= len | IXGBE_TXD_CMD; 1036 1037 tx_desc = IXGBEVF_TX_DESC(ring, i); 1038 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1039 1040 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1041 tx_desc->read.olinfo_status = 1042 cpu_to_le32((len << IXGBE_ADVTXD_PAYLEN_SHIFT) | 1043 IXGBE_ADVTXD_CC); 1044 1045 /* Avoid any potential race with cleanup */ 1046 smp_wmb(); 1047 1048 /* set next_to_watch value indicating a packet is present */ 1049 i++; 1050 if (i == ring->count) 1051 i = 0; 1052 1053 tx_buffer->next_to_watch = tx_desc; 1054 ring->next_to_use = i; 1055 1056 return IXGBEVF_XDP_TX; 1057 } 1058 1059 static struct sk_buff *ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, 1060 struct ixgbevf_ring *rx_ring, 1061 struct xdp_buff *xdp) 1062 { 1063 int result = IXGBEVF_XDP_PASS; 1064 struct ixgbevf_ring *xdp_ring; 1065 struct bpf_prog *xdp_prog; 1066 u32 act; 1067 1068 rcu_read_lock(); 1069 xdp_prog = READ_ONCE(rx_ring->xdp_prog); 1070 1071 if (!xdp_prog) 1072 goto xdp_out; 1073 1074 act = bpf_prog_run_xdp(xdp_prog, xdp); 1075 switch (act) { 1076 case XDP_PASS: 1077 break; 1078 case XDP_TX: 1079 xdp_ring = adapter->xdp_ring[rx_ring->queue_index]; 1080 result = ixgbevf_xmit_xdp_ring(xdp_ring, xdp); 1081 break; 1082 default: 1083 bpf_warn_invalid_xdp_action(act); 1084 /* fallthrough */ 1085 case XDP_ABORTED: 1086 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); 1087 /* fallthrough -- handle aborts by dropping packet */ 1088 case XDP_DROP: 1089 result = IXGBEVF_XDP_CONSUMED; 1090 break; 1091 } 1092 xdp_out: 1093 rcu_read_unlock(); 1094 return ERR_PTR(-result); 1095 } 1096 1097 static void ixgbevf_rx_buffer_flip(struct ixgbevf_ring *rx_ring, 1098 struct ixgbevf_rx_buffer *rx_buffer, 1099 unsigned int size) 1100 { 1101 #if (PAGE_SIZE < 8192) 1102 unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; 1103 1104 rx_buffer->page_offset ^= truesize; 1105 #else 1106 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1107 SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) : 1108 SKB_DATA_ALIGN(size); 1109 1110 rx_buffer->page_offset += truesize; 1111 #endif 1112 } 1113 1114 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 1115 struct ixgbevf_ring *rx_ring, 1116 int budget) 1117 { 1118 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1119 struct ixgbevf_adapter *adapter = q_vector->adapter; 1120 u16 cleaned_count = ixgbevf_desc_unused(rx_ring); 1121 struct sk_buff *skb = rx_ring->skb; 1122 bool xdp_xmit = false; 1123 struct xdp_buff xdp; 1124 1125 xdp.rxq = &rx_ring->xdp_rxq; 1126 1127 while (likely(total_rx_packets < budget)) { 1128 struct ixgbevf_rx_buffer *rx_buffer; 1129 union ixgbe_adv_rx_desc *rx_desc; 1130 unsigned int size; 1131 1132 /* return some buffers to hardware, one at a time is too slow */ 1133 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 1134 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count); 1135 cleaned_count = 0; 1136 } 1137 1138 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); 1139 size = le16_to_cpu(rx_desc->wb.upper.length); 1140 if (!size) 1141 break; 1142 1143 /* This memory barrier is needed to keep us from reading 1144 * any other fields out of the rx_desc until we know the 1145 * RXD_STAT_DD bit is set 1146 */ 1147 rmb(); 1148 1149 rx_buffer = ixgbevf_get_rx_buffer(rx_ring, size); 1150 1151 /* retrieve a buffer from the ring */ 1152 if (!skb) { 1153 xdp.data = page_address(rx_buffer->page) + 1154 rx_buffer->page_offset; 1155 xdp.data_meta = xdp.data; 1156 xdp.data_hard_start = xdp.data - 1157 ixgbevf_rx_offset(rx_ring); 1158 xdp.data_end = xdp.data + size; 1159 1160 skb = ixgbevf_run_xdp(adapter, rx_ring, &xdp); 1161 } 1162 1163 if (IS_ERR(skb)) { 1164 if (PTR_ERR(skb) == -IXGBEVF_XDP_TX) { 1165 xdp_xmit = true; 1166 ixgbevf_rx_buffer_flip(rx_ring, rx_buffer, 1167 size); 1168 } else { 1169 rx_buffer->pagecnt_bias++; 1170 } 1171 total_rx_packets++; 1172 total_rx_bytes += size; 1173 } else if (skb) { 1174 ixgbevf_add_rx_frag(rx_ring, rx_buffer, skb, size); 1175 } else if (ring_uses_build_skb(rx_ring)) { 1176 skb = ixgbevf_build_skb(rx_ring, rx_buffer, 1177 &xdp, rx_desc); 1178 } else { 1179 skb = ixgbevf_construct_skb(rx_ring, rx_buffer, 1180 &xdp, rx_desc); 1181 } 1182 1183 /* exit if we failed to retrieve a buffer */ 1184 if (!skb) { 1185 rx_ring->rx_stats.alloc_rx_buff_failed++; 1186 rx_buffer->pagecnt_bias++; 1187 break; 1188 } 1189 1190 ixgbevf_put_rx_buffer(rx_ring, rx_buffer, skb); 1191 cleaned_count++; 1192 1193 /* fetch next buffer in frame if non-eop */ 1194 if (ixgbevf_is_non_eop(rx_ring, rx_desc)) 1195 continue; 1196 1197 /* verify the packet layout is correct */ 1198 if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) { 1199 skb = NULL; 1200 continue; 1201 } 1202 1203 /* probably a little skewed due to removing CRC */ 1204 total_rx_bytes += skb->len; 1205 1206 /* Workaround hardware that can't do proper VEPA multicast 1207 * source pruning. 1208 */ 1209 if ((skb->pkt_type == PACKET_BROADCAST || 1210 skb->pkt_type == PACKET_MULTICAST) && 1211 ether_addr_equal(rx_ring->netdev->dev_addr, 1212 eth_hdr(skb)->h_source)) { 1213 dev_kfree_skb_irq(skb); 1214 continue; 1215 } 1216 1217 /* populate checksum, VLAN, and protocol */ 1218 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); 1219 1220 ixgbevf_rx_skb(q_vector, skb); 1221 1222 /* reset skb pointer */ 1223 skb = NULL; 1224 1225 /* update budget accounting */ 1226 total_rx_packets++; 1227 } 1228 1229 /* place incomplete frames back on ring for completion */ 1230 rx_ring->skb = skb; 1231 1232 if (xdp_xmit) { 1233 struct ixgbevf_ring *xdp_ring = 1234 adapter->xdp_ring[rx_ring->queue_index]; 1235 1236 /* Force memory writes to complete before letting h/w 1237 * know there are new descriptors to fetch. 1238 */ 1239 wmb(); 1240 ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); 1241 } 1242 1243 u64_stats_update_begin(&rx_ring->syncp); 1244 rx_ring->stats.packets += total_rx_packets; 1245 rx_ring->stats.bytes += total_rx_bytes; 1246 u64_stats_update_end(&rx_ring->syncp); 1247 q_vector->rx.total_packets += total_rx_packets; 1248 q_vector->rx.total_bytes += total_rx_bytes; 1249 1250 return total_rx_packets; 1251 } 1252 1253 /** 1254 * ixgbevf_poll - NAPI polling calback 1255 * @napi: napi struct with our devices info in it 1256 * @budget: amount of work driver is allowed to do this pass, in packets 1257 * 1258 * This function will clean more than one or more rings associated with a 1259 * q_vector. 1260 **/ 1261 static int ixgbevf_poll(struct napi_struct *napi, int budget) 1262 { 1263 struct ixgbevf_q_vector *q_vector = 1264 container_of(napi, struct ixgbevf_q_vector, napi); 1265 struct ixgbevf_adapter *adapter = q_vector->adapter; 1266 struct ixgbevf_ring *ring; 1267 int per_ring_budget, work_done = 0; 1268 bool clean_complete = true; 1269 1270 ixgbevf_for_each_ring(ring, q_vector->tx) { 1271 if (!ixgbevf_clean_tx_irq(q_vector, ring, budget)) 1272 clean_complete = false; 1273 } 1274 1275 if (budget <= 0) 1276 return budget; 1277 1278 /* attempt to distribute budget to each queue fairly, but don't allow 1279 * the budget to go below 1 because we'll exit polling 1280 */ 1281 if (q_vector->rx.count > 1) 1282 per_ring_budget = max(budget/q_vector->rx.count, 1); 1283 else 1284 per_ring_budget = budget; 1285 1286 ixgbevf_for_each_ring(ring, q_vector->rx) { 1287 int cleaned = ixgbevf_clean_rx_irq(q_vector, ring, 1288 per_ring_budget); 1289 work_done += cleaned; 1290 if (cleaned >= per_ring_budget) 1291 clean_complete = false; 1292 } 1293 1294 /* If all work not completed, return budget and keep polling */ 1295 if (!clean_complete) 1296 return budget; 1297 1298 /* Exit the polling mode, but don't re-enable interrupts if stack might 1299 * poll us due to busy-polling 1300 */ 1301 if (likely(napi_complete_done(napi, work_done))) { 1302 if (adapter->rx_itr_setting == 1) 1303 ixgbevf_set_itr(q_vector); 1304 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 1305 !test_bit(__IXGBEVF_REMOVING, &adapter->state)) 1306 ixgbevf_irq_enable_queues(adapter, 1307 BIT(q_vector->v_idx)); 1308 } 1309 1310 return min(work_done, budget - 1); 1311 } 1312 1313 /** 1314 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 1315 * @q_vector: structure containing interrupt and ring information 1316 **/ 1317 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) 1318 { 1319 struct ixgbevf_adapter *adapter = q_vector->adapter; 1320 struct ixgbe_hw *hw = &adapter->hw; 1321 int v_idx = q_vector->v_idx; 1322 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 1323 1324 /* set the WDIS bit to not clear the timer bits and cause an 1325 * immediate assertion of the interrupt 1326 */ 1327 itr_reg |= IXGBE_EITR_CNT_WDIS; 1328 1329 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 1330 } 1331 1332 /** 1333 * ixgbevf_configure_msix - Configure MSI-X hardware 1334 * @adapter: board private structure 1335 * 1336 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 1337 * interrupts. 1338 **/ 1339 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 1340 { 1341 struct ixgbevf_q_vector *q_vector; 1342 int q_vectors, v_idx; 1343 1344 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1345 adapter->eims_enable_mask = 0; 1346 1347 /* Populate the IVAR table and set the ITR values to the 1348 * corresponding register. 1349 */ 1350 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 1351 struct ixgbevf_ring *ring; 1352 1353 q_vector = adapter->q_vector[v_idx]; 1354 1355 ixgbevf_for_each_ring(ring, q_vector->rx) 1356 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx); 1357 1358 ixgbevf_for_each_ring(ring, q_vector->tx) 1359 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx); 1360 1361 if (q_vector->tx.ring && !q_vector->rx.ring) { 1362 /* Tx only vector */ 1363 if (adapter->tx_itr_setting == 1) 1364 q_vector->itr = IXGBE_12K_ITR; 1365 else 1366 q_vector->itr = adapter->tx_itr_setting; 1367 } else { 1368 /* Rx or Rx/Tx vector */ 1369 if (adapter->rx_itr_setting == 1) 1370 q_vector->itr = IXGBE_20K_ITR; 1371 else 1372 q_vector->itr = adapter->rx_itr_setting; 1373 } 1374 1375 /* add q_vector eims value to global eims_enable_mask */ 1376 adapter->eims_enable_mask |= BIT(v_idx); 1377 1378 ixgbevf_write_eitr(q_vector); 1379 } 1380 1381 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 1382 /* setup eims_other and add value to global eims_enable_mask */ 1383 adapter->eims_other = BIT(v_idx); 1384 adapter->eims_enable_mask |= adapter->eims_other; 1385 } 1386 1387 enum latency_range { 1388 lowest_latency = 0, 1389 low_latency = 1, 1390 bulk_latency = 2, 1391 latency_invalid = 255 1392 }; 1393 1394 /** 1395 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 1396 * @q_vector: structure containing interrupt and ring information 1397 * @ring_container: structure containing ring performance data 1398 * 1399 * Stores a new ITR value based on packets and byte 1400 * counts during the last interrupt. The advantage of per interrupt 1401 * computation is faster updates and more accurate ITR for the current 1402 * traffic pattern. Constants in this function were computed 1403 * based on theoretical maximum wire speed and thresholds were set based 1404 * on testing data as well as attempting to minimize response time 1405 * while increasing bulk throughput. 1406 **/ 1407 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector, 1408 struct ixgbevf_ring_container *ring_container) 1409 { 1410 int bytes = ring_container->total_bytes; 1411 int packets = ring_container->total_packets; 1412 u32 timepassed_us; 1413 u64 bytes_perint; 1414 u8 itr_setting = ring_container->itr; 1415 1416 if (packets == 0) 1417 return; 1418 1419 /* simple throttle rate management 1420 * 0-20MB/s lowest (100000 ints/s) 1421 * 20-100MB/s low (20000 ints/s) 1422 * 100-1249MB/s bulk (12000 ints/s) 1423 */ 1424 /* what was last interrupt timeslice? */ 1425 timepassed_us = q_vector->itr >> 2; 1426 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 1427 1428 switch (itr_setting) { 1429 case lowest_latency: 1430 if (bytes_perint > 10) 1431 itr_setting = low_latency; 1432 break; 1433 case low_latency: 1434 if (bytes_perint > 20) 1435 itr_setting = bulk_latency; 1436 else if (bytes_perint <= 10) 1437 itr_setting = lowest_latency; 1438 break; 1439 case bulk_latency: 1440 if (bytes_perint <= 20) 1441 itr_setting = low_latency; 1442 break; 1443 } 1444 1445 /* clear work counters since we have the values we need */ 1446 ring_container->total_bytes = 0; 1447 ring_container->total_packets = 0; 1448 1449 /* write updated itr to ring container */ 1450 ring_container->itr = itr_setting; 1451 } 1452 1453 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector) 1454 { 1455 u32 new_itr = q_vector->itr; 1456 u8 current_itr; 1457 1458 ixgbevf_update_itr(q_vector, &q_vector->tx); 1459 ixgbevf_update_itr(q_vector, &q_vector->rx); 1460 1461 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 1462 1463 switch (current_itr) { 1464 /* counts and packets in update_itr are dependent on these numbers */ 1465 case lowest_latency: 1466 new_itr = IXGBE_100K_ITR; 1467 break; 1468 case low_latency: 1469 new_itr = IXGBE_20K_ITR; 1470 break; 1471 case bulk_latency: 1472 new_itr = IXGBE_12K_ITR; 1473 break; 1474 default: 1475 break; 1476 } 1477 1478 if (new_itr != q_vector->itr) { 1479 /* do an exponential smoothing */ 1480 new_itr = (10 * new_itr * q_vector->itr) / 1481 ((9 * new_itr) + q_vector->itr); 1482 1483 /* save the algorithm value here */ 1484 q_vector->itr = new_itr; 1485 1486 ixgbevf_write_eitr(q_vector); 1487 } 1488 } 1489 1490 static irqreturn_t ixgbevf_msix_other(int irq, void *data) 1491 { 1492 struct ixgbevf_adapter *adapter = data; 1493 struct ixgbe_hw *hw = &adapter->hw; 1494 1495 hw->mac.get_link_status = 1; 1496 1497 ixgbevf_service_event_schedule(adapter); 1498 1499 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); 1500 1501 return IRQ_HANDLED; 1502 } 1503 1504 /** 1505 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) 1506 * @irq: unused 1507 * @data: pointer to our q_vector struct for this interrupt vector 1508 **/ 1509 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data) 1510 { 1511 struct ixgbevf_q_vector *q_vector = data; 1512 1513 /* EIAM disabled interrupts (on this vector) for us */ 1514 if (q_vector->rx.ring || q_vector->tx.ring) 1515 napi_schedule_irqoff(&q_vector->napi); 1516 1517 return IRQ_HANDLED; 1518 } 1519 1520 /** 1521 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 1522 * @adapter: board private structure 1523 * 1524 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 1525 * interrupts from the kernel. 1526 **/ 1527 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 1528 { 1529 struct net_device *netdev = adapter->netdev; 1530 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1531 unsigned int ri = 0, ti = 0; 1532 int vector, err; 1533 1534 for (vector = 0; vector < q_vectors; vector++) { 1535 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector]; 1536 struct msix_entry *entry = &adapter->msix_entries[vector]; 1537 1538 if (q_vector->tx.ring && q_vector->rx.ring) { 1539 snprintf(q_vector->name, sizeof(q_vector->name), 1540 "%s-TxRx-%u", netdev->name, ri++); 1541 ti++; 1542 } else if (q_vector->rx.ring) { 1543 snprintf(q_vector->name, sizeof(q_vector->name), 1544 "%s-rx-%u", netdev->name, ri++); 1545 } else if (q_vector->tx.ring) { 1546 snprintf(q_vector->name, sizeof(q_vector->name), 1547 "%s-tx-%u", netdev->name, ti++); 1548 } else { 1549 /* skip this unused q_vector */ 1550 continue; 1551 } 1552 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0, 1553 q_vector->name, q_vector); 1554 if (err) { 1555 hw_dbg(&adapter->hw, 1556 "request_irq failed for MSIX interrupt Error: %d\n", 1557 err); 1558 goto free_queue_irqs; 1559 } 1560 } 1561 1562 err = request_irq(adapter->msix_entries[vector].vector, 1563 &ixgbevf_msix_other, 0, netdev->name, adapter); 1564 if (err) { 1565 hw_dbg(&adapter->hw, "request_irq for msix_other failed: %d\n", 1566 err); 1567 goto free_queue_irqs; 1568 } 1569 1570 return 0; 1571 1572 free_queue_irqs: 1573 while (vector) { 1574 vector--; 1575 free_irq(adapter->msix_entries[vector].vector, 1576 adapter->q_vector[vector]); 1577 } 1578 /* This failure is non-recoverable - it indicates the system is 1579 * out of MSIX vector resources and the VF driver cannot run 1580 * without them. Set the number of msix vectors to zero 1581 * indicating that not enough can be allocated. The error 1582 * will be returned to the user indicating device open failed. 1583 * Any further attempts to force the driver to open will also 1584 * fail. The only way to recover is to unload the driver and 1585 * reload it again. If the system has recovered some MSIX 1586 * vectors then it may succeed. 1587 */ 1588 adapter->num_msix_vectors = 0; 1589 return err; 1590 } 1591 1592 /** 1593 * ixgbevf_request_irq - initialize interrupts 1594 * @adapter: board private structure 1595 * 1596 * Attempts to configure interrupts using the best available 1597 * capabilities of the hardware and kernel. 1598 **/ 1599 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 1600 { 1601 int err = ixgbevf_request_msix_irqs(adapter); 1602 1603 if (err) 1604 hw_dbg(&adapter->hw, "request_irq failed, Error %d\n", err); 1605 1606 return err; 1607 } 1608 1609 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 1610 { 1611 int i, q_vectors; 1612 1613 if (!adapter->msix_entries) 1614 return; 1615 1616 q_vectors = adapter->num_msix_vectors; 1617 i = q_vectors - 1; 1618 1619 free_irq(adapter->msix_entries[i].vector, adapter); 1620 i--; 1621 1622 for (; i >= 0; i--) { 1623 /* free only the irqs that were actually requested */ 1624 if (!adapter->q_vector[i]->rx.ring && 1625 !adapter->q_vector[i]->tx.ring) 1626 continue; 1627 1628 free_irq(adapter->msix_entries[i].vector, 1629 adapter->q_vector[i]); 1630 } 1631 } 1632 1633 /** 1634 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1635 * @adapter: board private structure 1636 **/ 1637 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1638 { 1639 struct ixgbe_hw *hw = &adapter->hw; 1640 int i; 1641 1642 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0); 1643 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1644 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0); 1645 1646 IXGBE_WRITE_FLUSH(hw); 1647 1648 for (i = 0; i < adapter->num_msix_vectors; i++) 1649 synchronize_irq(adapter->msix_entries[i].vector); 1650 } 1651 1652 /** 1653 * ixgbevf_irq_enable - Enable default interrupt generation settings 1654 * @adapter: board private structure 1655 **/ 1656 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) 1657 { 1658 struct ixgbe_hw *hw = &adapter->hw; 1659 1660 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask); 1661 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask); 1662 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); 1663 } 1664 1665 /** 1666 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset 1667 * @adapter: board private structure 1668 * @ring: structure containing ring specific data 1669 * 1670 * Configure the Tx descriptor ring after a reset. 1671 **/ 1672 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 1673 struct ixgbevf_ring *ring) 1674 { 1675 struct ixgbe_hw *hw = &adapter->hw; 1676 u64 tdba = ring->dma; 1677 int wait_loop = 10; 1678 u32 txdctl = IXGBE_TXDCTL_ENABLE; 1679 u8 reg_idx = ring->reg_idx; 1680 1681 /* disable queue to avoid issues while updating state */ 1682 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 1683 IXGBE_WRITE_FLUSH(hw); 1684 1685 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 1686 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32); 1687 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx), 1688 ring->count * sizeof(union ixgbe_adv_tx_desc)); 1689 1690 /* disable head writeback */ 1691 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0); 1692 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0); 1693 1694 /* enable relaxed ordering */ 1695 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx), 1696 (IXGBE_DCA_TXCTRL_DESC_RRO_EN | 1697 IXGBE_DCA_TXCTRL_DATA_RRO_EN)); 1698 1699 /* reset head and tail pointers */ 1700 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0); 1701 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0); 1702 ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx); 1703 1704 /* reset ntu and ntc to place SW in sync with hardwdare */ 1705 ring->next_to_clean = 0; 1706 ring->next_to_use = 0; 1707 1708 /* In order to avoid issues WTHRESH + PTHRESH should always be equal 1709 * to or less than the number of on chip descriptors, which is 1710 * currently 40. 1711 */ 1712 txdctl |= (8 << 16); /* WTHRESH = 8 */ 1713 1714 /* Setting PTHRESH to 32 both improves performance */ 1715 txdctl |= (1u << 8) | /* HTHRESH = 1 */ 1716 32; /* PTHRESH = 32 */ 1717 1718 /* reinitialize tx_buffer_info */ 1719 memset(ring->tx_buffer_info, 0, 1720 sizeof(struct ixgbevf_tx_buffer) * ring->count); 1721 1722 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); 1723 clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); 1724 1725 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl); 1726 1727 /* poll to verify queue is enabled */ 1728 do { 1729 usleep_range(1000, 2000); 1730 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx)); 1731 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 1732 if (!wait_loop) 1733 hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx); 1734 } 1735 1736 /** 1737 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1738 * @adapter: board private structure 1739 * 1740 * Configure the Tx unit of the MAC after a reset. 1741 **/ 1742 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1743 { 1744 u32 i; 1745 1746 /* Setup the HW Tx Head and Tail descriptor pointers */ 1747 for (i = 0; i < adapter->num_tx_queues; i++) 1748 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]); 1749 for (i = 0; i < adapter->num_xdp_queues; i++) 1750 ixgbevf_configure_tx_ring(adapter, adapter->xdp_ring[i]); 1751 } 1752 1753 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1754 1755 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, 1756 struct ixgbevf_ring *ring, int index) 1757 { 1758 struct ixgbe_hw *hw = &adapter->hw; 1759 u32 srrctl; 1760 1761 srrctl = IXGBE_SRRCTL_DROP_EN; 1762 1763 srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; 1764 if (ring_uses_large_buffer(ring)) 1765 srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1766 else 1767 srrctl |= IXGBEVF_RXBUFFER_2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1768 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1769 1770 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1771 } 1772 1773 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) 1774 { 1775 struct ixgbe_hw *hw = &adapter->hw; 1776 1777 /* PSRTYPE must be initialized in 82599 */ 1778 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 1779 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR | 1780 IXGBE_PSRTYPE_L2HDR; 1781 1782 if (adapter->num_rx_queues > 1) 1783 psrtype |= BIT(29); 1784 1785 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1786 } 1787 1788 #define IXGBEVF_MAX_RX_DESC_POLL 10 1789 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, 1790 struct ixgbevf_ring *ring) 1791 { 1792 struct ixgbe_hw *hw = &adapter->hw; 1793 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1794 u32 rxdctl; 1795 u8 reg_idx = ring->reg_idx; 1796 1797 if (IXGBE_REMOVED(hw->hw_addr)) 1798 return; 1799 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1800 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1801 1802 /* write value back with RXDCTL.ENABLE bit cleared */ 1803 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1804 1805 /* the hardware may take up to 100us to really disable the Rx queue */ 1806 do { 1807 udelay(10); 1808 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1809 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 1810 1811 if (!wait_loop) 1812 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", 1813 reg_idx); 1814 } 1815 1816 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1817 struct ixgbevf_ring *ring) 1818 { 1819 struct ixgbe_hw *hw = &adapter->hw; 1820 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1821 u32 rxdctl; 1822 u8 reg_idx = ring->reg_idx; 1823 1824 if (IXGBE_REMOVED(hw->hw_addr)) 1825 return; 1826 do { 1827 usleep_range(1000, 2000); 1828 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1829 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 1830 1831 if (!wait_loop) 1832 pr_err("RXDCTL.ENABLE queue %d not set while polling\n", 1833 reg_idx); 1834 } 1835 1836 /** 1837 * ixgbevf_init_rss_key - Initialize adapter RSS key 1838 * @adapter: device handle 1839 * 1840 * Allocates and initializes the RSS key if it is not allocated. 1841 **/ 1842 static inline int ixgbevf_init_rss_key(struct ixgbevf_adapter *adapter) 1843 { 1844 u32 *rss_key; 1845 1846 if (!adapter->rss_key) { 1847 rss_key = kzalloc(IXGBEVF_RSS_HASH_KEY_SIZE, GFP_KERNEL); 1848 if (unlikely(!rss_key)) 1849 return -ENOMEM; 1850 1851 netdev_rss_key_fill(rss_key, IXGBEVF_RSS_HASH_KEY_SIZE); 1852 adapter->rss_key = rss_key; 1853 } 1854 1855 return 0; 1856 } 1857 1858 static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) 1859 { 1860 struct ixgbe_hw *hw = &adapter->hw; 1861 u32 vfmrqc = 0, vfreta = 0; 1862 u16 rss_i = adapter->num_rx_queues; 1863 u8 i, j; 1864 1865 /* Fill out hash function seeds */ 1866 for (i = 0; i < IXGBEVF_VFRSSRK_REGS; i++) 1867 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), *(adapter->rss_key + i)); 1868 1869 for (i = 0, j = 0; i < IXGBEVF_X550_VFRETA_SIZE; i++, j++) { 1870 if (j == rss_i) 1871 j = 0; 1872 1873 adapter->rss_indir_tbl[i] = j; 1874 1875 vfreta |= j << (i & 0x3) * 8; 1876 if ((i & 3) == 3) { 1877 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), vfreta); 1878 vfreta = 0; 1879 } 1880 } 1881 1882 /* Perform hash on these packet types */ 1883 vfmrqc |= IXGBE_VFMRQC_RSS_FIELD_IPV4 | 1884 IXGBE_VFMRQC_RSS_FIELD_IPV4_TCP | 1885 IXGBE_VFMRQC_RSS_FIELD_IPV6 | 1886 IXGBE_VFMRQC_RSS_FIELD_IPV6_TCP; 1887 1888 vfmrqc |= IXGBE_VFMRQC_RSSEN; 1889 1890 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); 1891 } 1892 1893 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, 1894 struct ixgbevf_ring *ring) 1895 { 1896 struct ixgbe_hw *hw = &adapter->hw; 1897 union ixgbe_adv_rx_desc *rx_desc; 1898 u64 rdba = ring->dma; 1899 u32 rxdctl; 1900 u8 reg_idx = ring->reg_idx; 1901 1902 /* disable queue to avoid issues while updating state */ 1903 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1904 ixgbevf_disable_rx_queue(adapter, ring); 1905 1906 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 1907 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32); 1908 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx), 1909 ring->count * sizeof(union ixgbe_adv_rx_desc)); 1910 1911 #ifndef CONFIG_SPARC 1912 /* enable relaxed ordering */ 1913 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1914 IXGBE_DCA_RXCTRL_DESC_RRO_EN); 1915 #else 1916 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1917 IXGBE_DCA_RXCTRL_DESC_RRO_EN | 1918 IXGBE_DCA_RXCTRL_DATA_WRO_EN); 1919 #endif 1920 1921 /* reset head and tail pointers */ 1922 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0); 1923 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); 1924 ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); 1925 1926 /* initialize rx_buffer_info */ 1927 memset(ring->rx_buffer_info, 0, 1928 sizeof(struct ixgbevf_rx_buffer) * ring->count); 1929 1930 /* initialize Rx descriptor 0 */ 1931 rx_desc = IXGBEVF_RX_DESC(ring, 0); 1932 rx_desc->wb.upper.length = 0; 1933 1934 /* reset ntu and ntc to place SW in sync with hardwdare */ 1935 ring->next_to_clean = 0; 1936 ring->next_to_use = 0; 1937 ring->next_to_alloc = 0; 1938 1939 ixgbevf_configure_srrctl(adapter, ring, reg_idx); 1940 1941 /* RXDCTL.RLPML does not work on 82599 */ 1942 if (adapter->hw.mac.type != ixgbe_mac_82599_vf) { 1943 rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | 1944 IXGBE_RXDCTL_RLPML_EN); 1945 1946 #if (PAGE_SIZE < 8192) 1947 /* Limit the maximum frame size so we don't overrun the skb */ 1948 if (ring_uses_build_skb(ring) && 1949 !ring_uses_large_buffer(ring)) 1950 rxdctl |= IXGBEVF_MAX_FRAME_BUILD_SKB | 1951 IXGBE_RXDCTL_RLPML_EN; 1952 #endif 1953 } 1954 1955 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1956 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1957 1958 ixgbevf_rx_desc_queue_enable(adapter, ring); 1959 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); 1960 } 1961 1962 static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter, 1963 struct ixgbevf_ring *rx_ring) 1964 { 1965 struct net_device *netdev = adapter->netdev; 1966 unsigned int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 1967 1968 /* set build_skb and buffer size flags */ 1969 clear_ring_build_skb_enabled(rx_ring); 1970 clear_ring_uses_large_buffer(rx_ring); 1971 1972 if (adapter->flags & IXGBEVF_FLAGS_LEGACY_RX) 1973 return; 1974 1975 set_ring_build_skb_enabled(rx_ring); 1976 1977 if (PAGE_SIZE < 8192) { 1978 if (max_frame <= IXGBEVF_MAX_FRAME_BUILD_SKB) 1979 return; 1980 1981 set_ring_uses_large_buffer(rx_ring); 1982 } 1983 } 1984 1985 /** 1986 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1987 * @adapter: board private structure 1988 * 1989 * Configure the Rx unit of the MAC after a reset. 1990 **/ 1991 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1992 { 1993 struct ixgbe_hw *hw = &adapter->hw; 1994 struct net_device *netdev = adapter->netdev; 1995 int i, ret; 1996 1997 ixgbevf_setup_psrtype(adapter); 1998 if (hw->mac.type >= ixgbe_mac_X550_vf) 1999 ixgbevf_setup_vfmrqc(adapter); 2000 2001 spin_lock_bh(&adapter->mbx_lock); 2002 /* notify the PF of our intent to use this size of frame */ 2003 ret = hw->mac.ops.set_rlpml(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN); 2004 spin_unlock_bh(&adapter->mbx_lock); 2005 if (ret) 2006 dev_err(&adapter->pdev->dev, 2007 "Failed to set MTU at %d\n", netdev->mtu); 2008 2009 /* Setup the HW Rx Head and Tail Descriptor Pointers and 2010 * the Base and Length of the Rx Descriptor Ring 2011 */ 2012 for (i = 0; i < adapter->num_rx_queues; i++) { 2013 struct ixgbevf_ring *rx_ring = adapter->rx_ring[i]; 2014 2015 ixgbevf_set_rx_buffer_len(adapter, rx_ring); 2016 ixgbevf_configure_rx_ring(adapter, rx_ring); 2017 } 2018 } 2019 2020 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, 2021 __be16 proto, u16 vid) 2022 { 2023 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2024 struct ixgbe_hw *hw = &adapter->hw; 2025 int err; 2026 2027 spin_lock_bh(&adapter->mbx_lock); 2028 2029 /* add VID to filter table */ 2030 err = hw->mac.ops.set_vfta(hw, vid, 0, true); 2031 2032 spin_unlock_bh(&adapter->mbx_lock); 2033 2034 /* translate error return types so error makes sense */ 2035 if (err == IXGBE_ERR_MBX) 2036 return -EIO; 2037 2038 if (err == IXGBE_ERR_INVALID_ARGUMENT) 2039 return -EACCES; 2040 2041 set_bit(vid, adapter->active_vlans); 2042 2043 return err; 2044 } 2045 2046 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, 2047 __be16 proto, u16 vid) 2048 { 2049 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2050 struct ixgbe_hw *hw = &adapter->hw; 2051 int err; 2052 2053 spin_lock_bh(&adapter->mbx_lock); 2054 2055 /* remove VID from filter table */ 2056 err = hw->mac.ops.set_vfta(hw, vid, 0, false); 2057 2058 spin_unlock_bh(&adapter->mbx_lock); 2059 2060 clear_bit(vid, adapter->active_vlans); 2061 2062 return err; 2063 } 2064 2065 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 2066 { 2067 u16 vid; 2068 2069 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2070 ixgbevf_vlan_rx_add_vid(adapter->netdev, 2071 htons(ETH_P_8021Q), vid); 2072 } 2073 2074 static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 2075 { 2076 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2077 struct ixgbe_hw *hw = &adapter->hw; 2078 int count = 0; 2079 2080 if ((netdev_uc_count(netdev)) > 10) { 2081 pr_err("Too many unicast filters - No Space\n"); 2082 return -ENOSPC; 2083 } 2084 2085 if (!netdev_uc_empty(netdev)) { 2086 struct netdev_hw_addr *ha; 2087 2088 netdev_for_each_uc_addr(ha, netdev) { 2089 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 2090 udelay(200); 2091 } 2092 } else { 2093 /* If the list is empty then send message to PF driver to 2094 * clear all MAC VLANs on this VF. 2095 */ 2096 hw->mac.ops.set_uc_addr(hw, 0, NULL); 2097 } 2098 2099 return count; 2100 } 2101 2102 /** 2103 * ixgbevf_set_rx_mode - Multicast and unicast set 2104 * @netdev: network interface device structure 2105 * 2106 * The set_rx_method entry point is called whenever the multicast address 2107 * list, unicast address list or the network interface flags are updated. 2108 * This routine is responsible for configuring the hardware for proper 2109 * multicast mode and configuring requested unicast filters. 2110 **/ 2111 static void ixgbevf_set_rx_mode(struct net_device *netdev) 2112 { 2113 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2114 struct ixgbe_hw *hw = &adapter->hw; 2115 unsigned int flags = netdev->flags; 2116 int xcast_mode; 2117 2118 /* request the most inclusive mode we need */ 2119 if (flags & IFF_PROMISC) 2120 xcast_mode = IXGBEVF_XCAST_MODE_PROMISC; 2121 else if (flags & IFF_ALLMULTI) 2122 xcast_mode = IXGBEVF_XCAST_MODE_ALLMULTI; 2123 else if (flags & (IFF_BROADCAST | IFF_MULTICAST)) 2124 xcast_mode = IXGBEVF_XCAST_MODE_MULTI; 2125 else 2126 xcast_mode = IXGBEVF_XCAST_MODE_NONE; 2127 2128 spin_lock_bh(&adapter->mbx_lock); 2129 2130 hw->mac.ops.update_xcast_mode(hw, xcast_mode); 2131 2132 /* reprogram multicast list */ 2133 hw->mac.ops.update_mc_addr_list(hw, netdev); 2134 2135 ixgbevf_write_uc_addr_list(netdev); 2136 2137 spin_unlock_bh(&adapter->mbx_lock); 2138 } 2139 2140 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 2141 { 2142 int q_idx; 2143 struct ixgbevf_q_vector *q_vector; 2144 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2145 2146 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 2147 q_vector = adapter->q_vector[q_idx]; 2148 napi_enable(&q_vector->napi); 2149 } 2150 } 2151 2152 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 2153 { 2154 int q_idx; 2155 struct ixgbevf_q_vector *q_vector; 2156 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2157 2158 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 2159 q_vector = adapter->q_vector[q_idx]; 2160 napi_disable(&q_vector->napi); 2161 } 2162 } 2163 2164 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter) 2165 { 2166 struct ixgbe_hw *hw = &adapter->hw; 2167 unsigned int def_q = 0; 2168 unsigned int num_tcs = 0; 2169 unsigned int num_rx_queues = adapter->num_rx_queues; 2170 unsigned int num_tx_queues = adapter->num_tx_queues; 2171 int err; 2172 2173 spin_lock_bh(&adapter->mbx_lock); 2174 2175 /* fetch queue configuration from the PF */ 2176 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2177 2178 spin_unlock_bh(&adapter->mbx_lock); 2179 2180 if (err) 2181 return err; 2182 2183 if (num_tcs > 1) { 2184 /* we need only one Tx queue */ 2185 num_tx_queues = 1; 2186 2187 /* update default Tx ring register index */ 2188 adapter->tx_ring[0]->reg_idx = def_q; 2189 2190 /* we need as many queues as traffic classes */ 2191 num_rx_queues = num_tcs; 2192 } 2193 2194 /* if we have a bad config abort request queue reset */ 2195 if ((adapter->num_rx_queues != num_rx_queues) || 2196 (adapter->num_tx_queues != num_tx_queues)) { 2197 /* force mailbox timeout to prevent further messages */ 2198 hw->mbx.timeout = 0; 2199 2200 /* wait for watchdog to come around and bail us out */ 2201 set_bit(__IXGBEVF_QUEUE_RESET_REQUESTED, &adapter->state); 2202 } 2203 2204 return 0; 2205 } 2206 2207 static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 2208 { 2209 ixgbevf_configure_dcb(adapter); 2210 2211 ixgbevf_set_rx_mode(adapter->netdev); 2212 2213 ixgbevf_restore_vlan(adapter); 2214 ixgbevf_ipsec_restore(adapter); 2215 2216 ixgbevf_configure_tx(adapter); 2217 ixgbevf_configure_rx(adapter); 2218 } 2219 2220 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 2221 { 2222 /* Only save pre-reset stats if there are some */ 2223 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 2224 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 2225 adapter->stats.base_vfgprc; 2226 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 2227 adapter->stats.base_vfgptc; 2228 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 2229 adapter->stats.base_vfgorc; 2230 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 2231 adapter->stats.base_vfgotc; 2232 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 2233 adapter->stats.base_vfmprc; 2234 } 2235 } 2236 2237 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 2238 { 2239 struct ixgbe_hw *hw = &adapter->hw; 2240 2241 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2242 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2243 adapter->stats.last_vfgorc |= 2244 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2245 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2246 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2247 adapter->stats.last_vfgotc |= 2248 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2249 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2250 2251 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 2252 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 2253 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 2254 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 2255 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 2256 } 2257 2258 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) 2259 { 2260 struct ixgbe_hw *hw = &adapter->hw; 2261 int api[] = { ixgbe_mbox_api_14, 2262 ixgbe_mbox_api_13, 2263 ixgbe_mbox_api_12, 2264 ixgbe_mbox_api_11, 2265 ixgbe_mbox_api_10, 2266 ixgbe_mbox_api_unknown }; 2267 int err, idx = 0; 2268 2269 spin_lock_bh(&adapter->mbx_lock); 2270 2271 while (api[idx] != ixgbe_mbox_api_unknown) { 2272 err = hw->mac.ops.negotiate_api_version(hw, api[idx]); 2273 if (!err) 2274 break; 2275 idx++; 2276 } 2277 2278 spin_unlock_bh(&adapter->mbx_lock); 2279 } 2280 2281 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 2282 { 2283 struct net_device *netdev = adapter->netdev; 2284 struct ixgbe_hw *hw = &adapter->hw; 2285 2286 ixgbevf_configure_msix(adapter); 2287 2288 spin_lock_bh(&adapter->mbx_lock); 2289 2290 if (is_valid_ether_addr(hw->mac.addr)) 2291 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 2292 else 2293 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 2294 2295 spin_unlock_bh(&adapter->mbx_lock); 2296 2297 smp_mb__before_atomic(); 2298 clear_bit(__IXGBEVF_DOWN, &adapter->state); 2299 ixgbevf_napi_enable_all(adapter); 2300 2301 /* clear any pending interrupts, may auto mask */ 2302 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2303 ixgbevf_irq_enable(adapter); 2304 2305 /* enable transmits */ 2306 netif_tx_start_all_queues(netdev); 2307 2308 ixgbevf_save_reset_stats(adapter); 2309 ixgbevf_init_last_counter_stats(adapter); 2310 2311 hw->mac.get_link_status = 1; 2312 mod_timer(&adapter->service_timer, jiffies); 2313 } 2314 2315 void ixgbevf_up(struct ixgbevf_adapter *adapter) 2316 { 2317 ixgbevf_configure(adapter); 2318 2319 ixgbevf_up_complete(adapter); 2320 } 2321 2322 /** 2323 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 2324 * @rx_ring: ring to free buffers from 2325 **/ 2326 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) 2327 { 2328 u16 i = rx_ring->next_to_clean; 2329 2330 /* Free Rx ring sk_buff */ 2331 if (rx_ring->skb) { 2332 dev_kfree_skb(rx_ring->skb); 2333 rx_ring->skb = NULL; 2334 } 2335 2336 /* Free all the Rx ring pages */ 2337 while (i != rx_ring->next_to_alloc) { 2338 struct ixgbevf_rx_buffer *rx_buffer; 2339 2340 rx_buffer = &rx_ring->rx_buffer_info[i]; 2341 2342 /* Invalidate cache lines that may have been written to by 2343 * device so that we avoid corrupting memory. 2344 */ 2345 dma_sync_single_range_for_cpu(rx_ring->dev, 2346 rx_buffer->dma, 2347 rx_buffer->page_offset, 2348 ixgbevf_rx_bufsz(rx_ring), 2349 DMA_FROM_DEVICE); 2350 2351 /* free resources associated with mapping */ 2352 dma_unmap_page_attrs(rx_ring->dev, 2353 rx_buffer->dma, 2354 ixgbevf_rx_pg_size(rx_ring), 2355 DMA_FROM_DEVICE, 2356 IXGBEVF_RX_DMA_ATTR); 2357 2358 __page_frag_cache_drain(rx_buffer->page, 2359 rx_buffer->pagecnt_bias); 2360 2361 i++; 2362 if (i == rx_ring->count) 2363 i = 0; 2364 } 2365 2366 rx_ring->next_to_alloc = 0; 2367 rx_ring->next_to_clean = 0; 2368 rx_ring->next_to_use = 0; 2369 } 2370 2371 /** 2372 * ixgbevf_clean_tx_ring - Free Tx Buffers 2373 * @tx_ring: ring to be cleaned 2374 **/ 2375 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) 2376 { 2377 u16 i = tx_ring->next_to_clean; 2378 struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 2379 2380 while (i != tx_ring->next_to_use) { 2381 union ixgbe_adv_tx_desc *eop_desc, *tx_desc; 2382 2383 /* Free all the Tx ring sk_buffs */ 2384 if (ring_is_xdp(tx_ring)) 2385 page_frag_free(tx_buffer->data); 2386 else 2387 dev_kfree_skb_any(tx_buffer->skb); 2388 2389 /* unmap skb header data */ 2390 dma_unmap_single(tx_ring->dev, 2391 dma_unmap_addr(tx_buffer, dma), 2392 dma_unmap_len(tx_buffer, len), 2393 DMA_TO_DEVICE); 2394 2395 /* check for eop_desc to determine the end of the packet */ 2396 eop_desc = tx_buffer->next_to_watch; 2397 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 2398 2399 /* unmap remaining buffers */ 2400 while (tx_desc != eop_desc) { 2401 tx_buffer++; 2402 tx_desc++; 2403 i++; 2404 if (unlikely(i == tx_ring->count)) { 2405 i = 0; 2406 tx_buffer = tx_ring->tx_buffer_info; 2407 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 2408 } 2409 2410 /* unmap any remaining paged data */ 2411 if (dma_unmap_len(tx_buffer, len)) 2412 dma_unmap_page(tx_ring->dev, 2413 dma_unmap_addr(tx_buffer, dma), 2414 dma_unmap_len(tx_buffer, len), 2415 DMA_TO_DEVICE); 2416 } 2417 2418 /* move us one more past the eop_desc for start of next pkt */ 2419 tx_buffer++; 2420 i++; 2421 if (unlikely(i == tx_ring->count)) { 2422 i = 0; 2423 tx_buffer = tx_ring->tx_buffer_info; 2424 } 2425 } 2426 2427 /* reset next_to_use and next_to_clean */ 2428 tx_ring->next_to_use = 0; 2429 tx_ring->next_to_clean = 0; 2430 2431 } 2432 2433 /** 2434 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 2435 * @adapter: board private structure 2436 **/ 2437 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 2438 { 2439 int i; 2440 2441 for (i = 0; i < adapter->num_rx_queues; i++) 2442 ixgbevf_clean_rx_ring(adapter->rx_ring[i]); 2443 } 2444 2445 /** 2446 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 2447 * @adapter: board private structure 2448 **/ 2449 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 2450 { 2451 int i; 2452 2453 for (i = 0; i < adapter->num_tx_queues; i++) 2454 ixgbevf_clean_tx_ring(adapter->tx_ring[i]); 2455 for (i = 0; i < adapter->num_xdp_queues; i++) 2456 ixgbevf_clean_tx_ring(adapter->xdp_ring[i]); 2457 } 2458 2459 void ixgbevf_down(struct ixgbevf_adapter *adapter) 2460 { 2461 struct net_device *netdev = adapter->netdev; 2462 struct ixgbe_hw *hw = &adapter->hw; 2463 int i; 2464 2465 /* signal that we are down to the interrupt handler */ 2466 if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state)) 2467 return; /* do nothing if already down */ 2468 2469 /* disable all enabled Rx queues */ 2470 for (i = 0; i < adapter->num_rx_queues; i++) 2471 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]); 2472 2473 usleep_range(10000, 20000); 2474 2475 netif_tx_stop_all_queues(netdev); 2476 2477 /* call carrier off first to avoid false dev_watchdog timeouts */ 2478 netif_carrier_off(netdev); 2479 netif_tx_disable(netdev); 2480 2481 ixgbevf_irq_disable(adapter); 2482 2483 ixgbevf_napi_disable_all(adapter); 2484 2485 del_timer_sync(&adapter->service_timer); 2486 2487 /* disable transmits in the hardware now that interrupts are off */ 2488 for (i = 0; i < adapter->num_tx_queues; i++) { 2489 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 2490 2491 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), 2492 IXGBE_TXDCTL_SWFLSH); 2493 } 2494 2495 for (i = 0; i < adapter->num_xdp_queues; i++) { 2496 u8 reg_idx = adapter->xdp_ring[i]->reg_idx; 2497 2498 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), 2499 IXGBE_TXDCTL_SWFLSH); 2500 } 2501 2502 if (!pci_channel_offline(adapter->pdev)) 2503 ixgbevf_reset(adapter); 2504 2505 ixgbevf_clean_all_tx_rings(adapter); 2506 ixgbevf_clean_all_rx_rings(adapter); 2507 } 2508 2509 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 2510 { 2511 WARN_ON(in_interrupt()); 2512 2513 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 2514 msleep(1); 2515 2516 ixgbevf_down(adapter); 2517 ixgbevf_up(adapter); 2518 2519 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 2520 } 2521 2522 void ixgbevf_reset(struct ixgbevf_adapter *adapter) 2523 { 2524 struct ixgbe_hw *hw = &adapter->hw; 2525 struct net_device *netdev = adapter->netdev; 2526 2527 if (hw->mac.ops.reset_hw(hw)) { 2528 hw_dbg(hw, "PF still resetting\n"); 2529 } else { 2530 hw->mac.ops.init_hw(hw); 2531 ixgbevf_negotiate_api(adapter); 2532 } 2533 2534 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 2535 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr); 2536 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr); 2537 } 2538 2539 adapter->last_reset = jiffies; 2540 } 2541 2542 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 2543 int vectors) 2544 { 2545 int vector_threshold; 2546 2547 /* We'll want at least 2 (vector_threshold): 2548 * 1) TxQ[0] + RxQ[0] handler 2549 * 2) Other (Link Status Change, etc.) 2550 */ 2551 vector_threshold = MIN_MSIX_COUNT; 2552 2553 /* The more we get, the more we will assign to Tx/Rx Cleanup 2554 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 2555 * Right now, we simply care about how many we'll get; we'll 2556 * set them up later while requesting irq's. 2557 */ 2558 vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 2559 vector_threshold, vectors); 2560 2561 if (vectors < 0) { 2562 dev_err(&adapter->pdev->dev, 2563 "Unable to allocate MSI-X interrupts\n"); 2564 kfree(adapter->msix_entries); 2565 adapter->msix_entries = NULL; 2566 return vectors; 2567 } 2568 2569 /* Adjust for only the vectors we'll use, which is minimum 2570 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 2571 * vectors we were allocated. 2572 */ 2573 adapter->num_msix_vectors = vectors; 2574 2575 return 0; 2576 } 2577 2578 /** 2579 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent 2580 * @adapter: board private structure to initialize 2581 * 2582 * This is the top level queue allocation routine. The order here is very 2583 * important, starting with the "most" number of features turned on at once, 2584 * and ending with the smallest set of features. This way large combinations 2585 * can be allocated if they're turned on, and smaller combinations are the 2586 * fallthrough conditions. 2587 * 2588 **/ 2589 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 2590 { 2591 struct ixgbe_hw *hw = &adapter->hw; 2592 unsigned int def_q = 0; 2593 unsigned int num_tcs = 0; 2594 int err; 2595 2596 /* Start with base case */ 2597 adapter->num_rx_queues = 1; 2598 adapter->num_tx_queues = 1; 2599 adapter->num_xdp_queues = 0; 2600 2601 spin_lock_bh(&adapter->mbx_lock); 2602 2603 /* fetch queue configuration from the PF */ 2604 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2605 2606 spin_unlock_bh(&adapter->mbx_lock); 2607 2608 if (err) 2609 return; 2610 2611 /* we need as many queues as traffic classes */ 2612 if (num_tcs > 1) { 2613 adapter->num_rx_queues = num_tcs; 2614 } else { 2615 u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES); 2616 2617 switch (hw->api_version) { 2618 case ixgbe_mbox_api_11: 2619 case ixgbe_mbox_api_12: 2620 case ixgbe_mbox_api_13: 2621 case ixgbe_mbox_api_14: 2622 if (adapter->xdp_prog && 2623 hw->mac.max_tx_queues == rss) 2624 rss = rss > 3 ? 2 : 1; 2625 2626 adapter->num_rx_queues = rss; 2627 adapter->num_tx_queues = rss; 2628 adapter->num_xdp_queues = adapter->xdp_prog ? rss : 0; 2629 default: 2630 break; 2631 } 2632 } 2633 } 2634 2635 /** 2636 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 2637 * @adapter: board private structure to initialize 2638 * 2639 * Attempt to configure the interrupts using the best available 2640 * capabilities of the hardware and the kernel. 2641 **/ 2642 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 2643 { 2644 int vector, v_budget; 2645 2646 /* It's easy to be greedy for MSI-X vectors, but it really 2647 * doesn't do us much good if we have a lot more vectors 2648 * than CPU's. So let's be conservative and only ask for 2649 * (roughly) the same number of vectors as there are CPU's. 2650 * The default is to use pairs of vectors. 2651 */ 2652 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); 2653 v_budget = min_t(int, v_budget, num_online_cpus()); 2654 v_budget += NON_Q_VECTORS; 2655 2656 adapter->msix_entries = kcalloc(v_budget, 2657 sizeof(struct msix_entry), GFP_KERNEL); 2658 if (!adapter->msix_entries) 2659 return -ENOMEM; 2660 2661 for (vector = 0; vector < v_budget; vector++) 2662 adapter->msix_entries[vector].entry = vector; 2663 2664 /* A failure in MSI-X entry allocation isn't fatal, but the VF driver 2665 * does not support any other modes, so we will simply fail here. Note 2666 * that we clean up the msix_entries pointer else-where. 2667 */ 2668 return ixgbevf_acquire_msix_vectors(adapter, v_budget); 2669 } 2670 2671 static void ixgbevf_add_ring(struct ixgbevf_ring *ring, 2672 struct ixgbevf_ring_container *head) 2673 { 2674 ring->next = head->ring; 2675 head->ring = ring; 2676 head->count++; 2677 } 2678 2679 /** 2680 * ixgbevf_alloc_q_vector - Allocate memory for a single interrupt vector 2681 * @adapter: board private structure to initialize 2682 * @v_idx: index of vector in adapter struct 2683 * @txr_count: number of Tx rings for q vector 2684 * @txr_idx: index of first Tx ring to assign 2685 * @xdp_count: total number of XDP rings to allocate 2686 * @xdp_idx: index of first XDP ring to allocate 2687 * @rxr_count: number of Rx rings for q vector 2688 * @rxr_idx: index of first Rx ring to assign 2689 * 2690 * We allocate one q_vector. If allocation fails we return -ENOMEM. 2691 **/ 2692 static int ixgbevf_alloc_q_vector(struct ixgbevf_adapter *adapter, int v_idx, 2693 int txr_count, int txr_idx, 2694 int xdp_count, int xdp_idx, 2695 int rxr_count, int rxr_idx) 2696 { 2697 struct ixgbevf_q_vector *q_vector; 2698 int reg_idx = txr_idx + xdp_idx; 2699 struct ixgbevf_ring *ring; 2700 int ring_count, size; 2701 2702 ring_count = txr_count + xdp_count + rxr_count; 2703 size = sizeof(*q_vector) + (sizeof(*ring) * ring_count); 2704 2705 /* allocate q_vector and rings */ 2706 q_vector = kzalloc(size, GFP_KERNEL); 2707 if (!q_vector) 2708 return -ENOMEM; 2709 2710 /* initialize NAPI */ 2711 netif_napi_add(adapter->netdev, &q_vector->napi, ixgbevf_poll, 64); 2712 2713 /* tie q_vector and adapter together */ 2714 adapter->q_vector[v_idx] = q_vector; 2715 q_vector->adapter = adapter; 2716 q_vector->v_idx = v_idx; 2717 2718 /* initialize pointer to rings */ 2719 ring = q_vector->ring; 2720 2721 while (txr_count) { 2722 /* assign generic ring traits */ 2723 ring->dev = &adapter->pdev->dev; 2724 ring->netdev = adapter->netdev; 2725 2726 /* configure backlink on ring */ 2727 ring->q_vector = q_vector; 2728 2729 /* update q_vector Tx values */ 2730 ixgbevf_add_ring(ring, &q_vector->tx); 2731 2732 /* apply Tx specific ring traits */ 2733 ring->count = adapter->tx_ring_count; 2734 ring->queue_index = txr_idx; 2735 ring->reg_idx = reg_idx; 2736 2737 /* assign ring to adapter */ 2738 adapter->tx_ring[txr_idx] = ring; 2739 2740 /* update count and index */ 2741 txr_count--; 2742 txr_idx++; 2743 reg_idx++; 2744 2745 /* push pointer to next ring */ 2746 ring++; 2747 } 2748 2749 while (xdp_count) { 2750 /* assign generic ring traits */ 2751 ring->dev = &adapter->pdev->dev; 2752 ring->netdev = adapter->netdev; 2753 2754 /* configure backlink on ring */ 2755 ring->q_vector = q_vector; 2756 2757 /* update q_vector Tx values */ 2758 ixgbevf_add_ring(ring, &q_vector->tx); 2759 2760 /* apply Tx specific ring traits */ 2761 ring->count = adapter->tx_ring_count; 2762 ring->queue_index = xdp_idx; 2763 ring->reg_idx = reg_idx; 2764 set_ring_xdp(ring); 2765 2766 /* assign ring to adapter */ 2767 adapter->xdp_ring[xdp_idx] = ring; 2768 2769 /* update count and index */ 2770 xdp_count--; 2771 xdp_idx++; 2772 reg_idx++; 2773 2774 /* push pointer to next ring */ 2775 ring++; 2776 } 2777 2778 while (rxr_count) { 2779 /* assign generic ring traits */ 2780 ring->dev = &adapter->pdev->dev; 2781 ring->netdev = adapter->netdev; 2782 2783 /* configure backlink on ring */ 2784 ring->q_vector = q_vector; 2785 2786 /* update q_vector Rx values */ 2787 ixgbevf_add_ring(ring, &q_vector->rx); 2788 2789 /* apply Rx specific ring traits */ 2790 ring->count = adapter->rx_ring_count; 2791 ring->queue_index = rxr_idx; 2792 ring->reg_idx = rxr_idx; 2793 2794 /* assign ring to adapter */ 2795 adapter->rx_ring[rxr_idx] = ring; 2796 2797 /* update count and index */ 2798 rxr_count--; 2799 rxr_idx++; 2800 2801 /* push pointer to next ring */ 2802 ring++; 2803 } 2804 2805 return 0; 2806 } 2807 2808 /** 2809 * ixgbevf_free_q_vector - Free memory allocated for specific interrupt vector 2810 * @adapter: board private structure to initialize 2811 * @v_idx: index of vector in adapter struct 2812 * 2813 * This function frees the memory allocated to the q_vector. In addition if 2814 * NAPI is enabled it will delete any references to the NAPI struct prior 2815 * to freeing the q_vector. 2816 **/ 2817 static void ixgbevf_free_q_vector(struct ixgbevf_adapter *adapter, int v_idx) 2818 { 2819 struct ixgbevf_q_vector *q_vector = adapter->q_vector[v_idx]; 2820 struct ixgbevf_ring *ring; 2821 2822 ixgbevf_for_each_ring(ring, q_vector->tx) { 2823 if (ring_is_xdp(ring)) 2824 adapter->xdp_ring[ring->queue_index] = NULL; 2825 else 2826 adapter->tx_ring[ring->queue_index] = NULL; 2827 } 2828 2829 ixgbevf_for_each_ring(ring, q_vector->rx) 2830 adapter->rx_ring[ring->queue_index] = NULL; 2831 2832 adapter->q_vector[v_idx] = NULL; 2833 netif_napi_del(&q_vector->napi); 2834 2835 /* ixgbevf_get_stats() might access the rings on this vector, 2836 * we must wait a grace period before freeing it. 2837 */ 2838 kfree_rcu(q_vector, rcu); 2839 } 2840 2841 /** 2842 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 2843 * @adapter: board private structure to initialize 2844 * 2845 * We allocate one q_vector per queue interrupt. If allocation fails we 2846 * return -ENOMEM. 2847 **/ 2848 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 2849 { 2850 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2851 int rxr_remaining = adapter->num_rx_queues; 2852 int txr_remaining = adapter->num_tx_queues; 2853 int xdp_remaining = adapter->num_xdp_queues; 2854 int rxr_idx = 0, txr_idx = 0, xdp_idx = 0, v_idx = 0; 2855 int err; 2856 2857 if (q_vectors >= (rxr_remaining + txr_remaining + xdp_remaining)) { 2858 for (; rxr_remaining; v_idx++, q_vectors--) { 2859 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors); 2860 2861 err = ixgbevf_alloc_q_vector(adapter, v_idx, 2862 0, 0, 0, 0, rqpv, rxr_idx); 2863 if (err) 2864 goto err_out; 2865 2866 /* update counts and index */ 2867 rxr_remaining -= rqpv; 2868 rxr_idx += rqpv; 2869 } 2870 } 2871 2872 for (; q_vectors; v_idx++, q_vectors--) { 2873 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors); 2874 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors); 2875 int xqpv = DIV_ROUND_UP(xdp_remaining, q_vectors); 2876 2877 err = ixgbevf_alloc_q_vector(adapter, v_idx, 2878 tqpv, txr_idx, 2879 xqpv, xdp_idx, 2880 rqpv, rxr_idx); 2881 2882 if (err) 2883 goto err_out; 2884 2885 /* update counts and index */ 2886 rxr_remaining -= rqpv; 2887 rxr_idx += rqpv; 2888 txr_remaining -= tqpv; 2889 txr_idx += tqpv; 2890 xdp_remaining -= xqpv; 2891 xdp_idx += xqpv; 2892 } 2893 2894 return 0; 2895 2896 err_out: 2897 while (v_idx) { 2898 v_idx--; 2899 ixgbevf_free_q_vector(adapter, v_idx); 2900 } 2901 2902 return -ENOMEM; 2903 } 2904 2905 /** 2906 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 2907 * @adapter: board private structure to initialize 2908 * 2909 * This function frees the memory allocated to the q_vectors. In addition if 2910 * NAPI is enabled it will delete any references to the NAPI struct prior 2911 * to freeing the q_vector. 2912 **/ 2913 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 2914 { 2915 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2916 2917 while (q_vectors) { 2918 q_vectors--; 2919 ixgbevf_free_q_vector(adapter, q_vectors); 2920 } 2921 } 2922 2923 /** 2924 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 2925 * @adapter: board private structure 2926 * 2927 **/ 2928 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 2929 { 2930 if (!adapter->msix_entries) 2931 return; 2932 2933 pci_disable_msix(adapter->pdev); 2934 kfree(adapter->msix_entries); 2935 adapter->msix_entries = NULL; 2936 } 2937 2938 /** 2939 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 2940 * @adapter: board private structure to initialize 2941 * 2942 **/ 2943 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 2944 { 2945 int err; 2946 2947 /* Number of supported queues */ 2948 ixgbevf_set_num_queues(adapter); 2949 2950 err = ixgbevf_set_interrupt_capability(adapter); 2951 if (err) { 2952 hw_dbg(&adapter->hw, 2953 "Unable to setup interrupt capabilities\n"); 2954 goto err_set_interrupt; 2955 } 2956 2957 err = ixgbevf_alloc_q_vectors(adapter); 2958 if (err) { 2959 hw_dbg(&adapter->hw, "Unable to allocate memory for queue vectors\n"); 2960 goto err_alloc_q_vectors; 2961 } 2962 2963 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u XDP Queue count %u\n", 2964 (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled", 2965 adapter->num_rx_queues, adapter->num_tx_queues, 2966 adapter->num_xdp_queues); 2967 2968 set_bit(__IXGBEVF_DOWN, &adapter->state); 2969 2970 return 0; 2971 err_alloc_q_vectors: 2972 ixgbevf_reset_interrupt_capability(adapter); 2973 err_set_interrupt: 2974 return err; 2975 } 2976 2977 /** 2978 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings 2979 * @adapter: board private structure to clear interrupt scheme on 2980 * 2981 * We go through and clear interrupt specific resources and reset the structure 2982 * to pre-load conditions 2983 **/ 2984 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) 2985 { 2986 adapter->num_tx_queues = 0; 2987 adapter->num_xdp_queues = 0; 2988 adapter->num_rx_queues = 0; 2989 2990 ixgbevf_free_q_vectors(adapter); 2991 ixgbevf_reset_interrupt_capability(adapter); 2992 } 2993 2994 /** 2995 * ixgbevf_sw_init - Initialize general software structures 2996 * @adapter: board private structure to initialize 2997 * 2998 * ixgbevf_sw_init initializes the Adapter private data structure. 2999 * Fields are initialized based on PCI device information and 3000 * OS network device settings (MTU size). 3001 **/ 3002 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 3003 { 3004 struct ixgbe_hw *hw = &adapter->hw; 3005 struct pci_dev *pdev = adapter->pdev; 3006 struct net_device *netdev = adapter->netdev; 3007 int err; 3008 3009 /* PCI config space info */ 3010 hw->vendor_id = pdev->vendor; 3011 hw->device_id = pdev->device; 3012 hw->revision_id = pdev->revision; 3013 hw->subsystem_vendor_id = pdev->subsystem_vendor; 3014 hw->subsystem_device_id = pdev->subsystem_device; 3015 3016 hw->mbx.ops.init_params(hw); 3017 3018 if (hw->mac.type >= ixgbe_mac_X550_vf) { 3019 err = ixgbevf_init_rss_key(adapter); 3020 if (err) 3021 goto out; 3022 } 3023 3024 /* assume legacy case in which PF would only give VF 2 queues */ 3025 hw->mac.max_tx_queues = 2; 3026 hw->mac.max_rx_queues = 2; 3027 3028 /* lock to protect mailbox accesses */ 3029 spin_lock_init(&adapter->mbx_lock); 3030 3031 err = hw->mac.ops.reset_hw(hw); 3032 if (err) { 3033 dev_info(&pdev->dev, 3034 "PF still in reset state. Is the PF interface up?\n"); 3035 } else { 3036 err = hw->mac.ops.init_hw(hw); 3037 if (err) { 3038 pr_err("init_shared_code failed: %d\n", err); 3039 goto out; 3040 } 3041 ixgbevf_negotiate_api(adapter); 3042 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr); 3043 if (err) 3044 dev_info(&pdev->dev, "Error reading MAC address\n"); 3045 else if (is_zero_ether_addr(adapter->hw.mac.addr)) 3046 dev_info(&pdev->dev, 3047 "MAC address not assigned by administrator.\n"); 3048 ether_addr_copy(netdev->dev_addr, hw->mac.addr); 3049 } 3050 3051 if (!is_valid_ether_addr(netdev->dev_addr)) { 3052 dev_info(&pdev->dev, "Assigning random MAC address\n"); 3053 eth_hw_addr_random(netdev); 3054 ether_addr_copy(hw->mac.addr, netdev->dev_addr); 3055 ether_addr_copy(hw->mac.perm_addr, netdev->dev_addr); 3056 } 3057 3058 /* Enable dynamic interrupt throttling rates */ 3059 adapter->rx_itr_setting = 1; 3060 adapter->tx_itr_setting = 1; 3061 3062 /* set default ring sizes */ 3063 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 3064 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 3065 3066 set_bit(__IXGBEVF_DOWN, &adapter->state); 3067 return 0; 3068 3069 out: 3070 return err; 3071 } 3072 3073 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 3074 { \ 3075 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 3076 if (current_counter < last_counter) \ 3077 counter += 0x100000000LL; \ 3078 last_counter = current_counter; \ 3079 counter &= 0xFFFFFFFF00000000LL; \ 3080 counter |= current_counter; \ 3081 } 3082 3083 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 3084 { \ 3085 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 3086 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 3087 u64 current_counter = (current_counter_msb << 32) | \ 3088 current_counter_lsb; \ 3089 if (current_counter < last_counter) \ 3090 counter += 0x1000000000LL; \ 3091 last_counter = current_counter; \ 3092 counter &= 0xFFFFFFF000000000LL; \ 3093 counter |= current_counter; \ 3094 } 3095 /** 3096 * ixgbevf_update_stats - Update the board statistics counters. 3097 * @adapter: board private structure 3098 **/ 3099 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 3100 { 3101 struct ixgbe_hw *hw = &adapter->hw; 3102 u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0; 3103 u64 alloc_rx_page = 0, hw_csum_rx_error = 0; 3104 int i; 3105 3106 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3107 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3108 return; 3109 3110 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 3111 adapter->stats.vfgprc); 3112 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 3113 adapter->stats.vfgptc); 3114 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3115 adapter->stats.last_vfgorc, 3116 adapter->stats.vfgorc); 3117 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3118 adapter->stats.last_vfgotc, 3119 adapter->stats.vfgotc); 3120 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 3121 adapter->stats.vfmprc); 3122 3123 for (i = 0; i < adapter->num_rx_queues; i++) { 3124 struct ixgbevf_ring *rx_ring = adapter->rx_ring[i]; 3125 3126 hw_csum_rx_error += rx_ring->rx_stats.csum_err; 3127 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed; 3128 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed; 3129 alloc_rx_page += rx_ring->rx_stats.alloc_rx_page; 3130 } 3131 3132 adapter->hw_csum_rx_error = hw_csum_rx_error; 3133 adapter->alloc_rx_page_failed = alloc_rx_page_failed; 3134 adapter->alloc_rx_buff_failed = alloc_rx_buff_failed; 3135 adapter->alloc_rx_page = alloc_rx_page; 3136 } 3137 3138 /** 3139 * ixgbevf_service_timer - Timer Call-back 3140 * @t: pointer to timer_list struct 3141 **/ 3142 static void ixgbevf_service_timer(struct timer_list *t) 3143 { 3144 struct ixgbevf_adapter *adapter = from_timer(adapter, t, 3145 service_timer); 3146 3147 /* Reset the timer */ 3148 mod_timer(&adapter->service_timer, (HZ * 2) + jiffies); 3149 3150 ixgbevf_service_event_schedule(adapter); 3151 } 3152 3153 static void ixgbevf_reset_subtask(struct ixgbevf_adapter *adapter) 3154 { 3155 if (!test_and_clear_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state)) 3156 return; 3157 3158 rtnl_lock(); 3159 /* If we're already down or resetting, just bail */ 3160 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3161 test_bit(__IXGBEVF_REMOVING, &adapter->state) || 3162 test_bit(__IXGBEVF_RESETTING, &adapter->state)) { 3163 rtnl_unlock(); 3164 return; 3165 } 3166 3167 adapter->tx_timeout_count++; 3168 3169 ixgbevf_reinit_locked(adapter); 3170 rtnl_unlock(); 3171 } 3172 3173 /** 3174 * ixgbevf_check_hang_subtask - check for hung queues and dropped interrupts 3175 * @adapter: pointer to the device adapter structure 3176 * 3177 * This function serves two purposes. First it strobes the interrupt lines 3178 * in order to make certain interrupts are occurring. Secondly it sets the 3179 * bits needed to check for TX hangs. As a result we should immediately 3180 * determine if a hang has occurred. 3181 **/ 3182 static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter) 3183 { 3184 struct ixgbe_hw *hw = &adapter->hw; 3185 u32 eics = 0; 3186 int i; 3187 3188 /* If we're down or resetting, just bail */ 3189 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3190 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3191 return; 3192 3193 /* Force detection of hung controller */ 3194 if (netif_carrier_ok(adapter->netdev)) { 3195 for (i = 0; i < adapter->num_tx_queues; i++) 3196 set_check_for_tx_hang(adapter->tx_ring[i]); 3197 for (i = 0; i < adapter->num_xdp_queues; i++) 3198 set_check_for_tx_hang(adapter->xdp_ring[i]); 3199 } 3200 3201 /* get one bit for every active Tx/Rx interrupt vector */ 3202 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 3203 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 3204 3205 if (qv->rx.ring || qv->tx.ring) 3206 eics |= BIT(i); 3207 } 3208 3209 /* Cause software interrupt to ensure rings are cleaned */ 3210 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics); 3211 } 3212 3213 /** 3214 * ixgbevf_watchdog_update_link - update the link status 3215 * @adapter: pointer to the device adapter structure 3216 **/ 3217 static void ixgbevf_watchdog_update_link(struct ixgbevf_adapter *adapter) 3218 { 3219 struct ixgbe_hw *hw = &adapter->hw; 3220 u32 link_speed = adapter->link_speed; 3221 bool link_up = adapter->link_up; 3222 s32 err; 3223 3224 spin_lock_bh(&adapter->mbx_lock); 3225 3226 err = hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 3227 3228 spin_unlock_bh(&adapter->mbx_lock); 3229 3230 /* if check for link returns error we will need to reset */ 3231 if (err && time_after(jiffies, adapter->last_reset + (10 * HZ))) { 3232 set_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state); 3233 link_up = false; 3234 } 3235 3236 adapter->link_up = link_up; 3237 adapter->link_speed = link_speed; 3238 } 3239 3240 /** 3241 * ixgbevf_watchdog_link_is_up - update netif_carrier status and 3242 * print link up message 3243 * @adapter: pointer to the device adapter structure 3244 **/ 3245 static void ixgbevf_watchdog_link_is_up(struct ixgbevf_adapter *adapter) 3246 { 3247 struct net_device *netdev = adapter->netdev; 3248 3249 /* only continue if link was previously down */ 3250 if (netif_carrier_ok(netdev)) 3251 return; 3252 3253 dev_info(&adapter->pdev->dev, "NIC Link is Up %s\n", 3254 (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 3255 "10 Gbps" : 3256 (adapter->link_speed == IXGBE_LINK_SPEED_1GB_FULL) ? 3257 "1 Gbps" : 3258 (adapter->link_speed == IXGBE_LINK_SPEED_100_FULL) ? 3259 "100 Mbps" : 3260 "unknown speed"); 3261 3262 netif_carrier_on(netdev); 3263 } 3264 3265 /** 3266 * ixgbevf_watchdog_link_is_down - update netif_carrier status and 3267 * print link down message 3268 * @adapter: pointer to the adapter structure 3269 **/ 3270 static void ixgbevf_watchdog_link_is_down(struct ixgbevf_adapter *adapter) 3271 { 3272 struct net_device *netdev = adapter->netdev; 3273 3274 adapter->link_speed = 0; 3275 3276 /* only continue if link was up previously */ 3277 if (!netif_carrier_ok(netdev)) 3278 return; 3279 3280 dev_info(&adapter->pdev->dev, "NIC Link is Down\n"); 3281 3282 netif_carrier_off(netdev); 3283 } 3284 3285 /** 3286 * ixgbevf_watchdog_subtask - worker thread to bring link up 3287 * @adapter: board private structure 3288 **/ 3289 static void ixgbevf_watchdog_subtask(struct ixgbevf_adapter *adapter) 3290 { 3291 /* if interface is down do nothing */ 3292 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3293 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3294 return; 3295 3296 ixgbevf_watchdog_update_link(adapter); 3297 3298 if (adapter->link_up) 3299 ixgbevf_watchdog_link_is_up(adapter); 3300 else 3301 ixgbevf_watchdog_link_is_down(adapter); 3302 3303 ixgbevf_update_stats(adapter); 3304 } 3305 3306 /** 3307 * ixgbevf_service_task - manages and runs subtasks 3308 * @work: pointer to work_struct containing our data 3309 **/ 3310 static void ixgbevf_service_task(struct work_struct *work) 3311 { 3312 struct ixgbevf_adapter *adapter = container_of(work, 3313 struct ixgbevf_adapter, 3314 service_task); 3315 struct ixgbe_hw *hw = &adapter->hw; 3316 3317 if (IXGBE_REMOVED(hw->hw_addr)) { 3318 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 3319 rtnl_lock(); 3320 ixgbevf_down(adapter); 3321 rtnl_unlock(); 3322 } 3323 return; 3324 } 3325 3326 ixgbevf_queue_reset_subtask(adapter); 3327 ixgbevf_reset_subtask(adapter); 3328 ixgbevf_watchdog_subtask(adapter); 3329 ixgbevf_check_hang_subtask(adapter); 3330 3331 ixgbevf_service_event_complete(adapter); 3332 } 3333 3334 /** 3335 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 3336 * @tx_ring: Tx descriptor ring for a specific queue 3337 * 3338 * Free all transmit software resources 3339 **/ 3340 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) 3341 { 3342 ixgbevf_clean_tx_ring(tx_ring); 3343 3344 vfree(tx_ring->tx_buffer_info); 3345 tx_ring->tx_buffer_info = NULL; 3346 3347 /* if not set, then don't free */ 3348 if (!tx_ring->desc) 3349 return; 3350 3351 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, 3352 tx_ring->dma); 3353 3354 tx_ring->desc = NULL; 3355 } 3356 3357 /** 3358 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 3359 * @adapter: board private structure 3360 * 3361 * Free all transmit software resources 3362 **/ 3363 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 3364 { 3365 int i; 3366 3367 for (i = 0; i < adapter->num_tx_queues; i++) 3368 if (adapter->tx_ring[i]->desc) 3369 ixgbevf_free_tx_resources(adapter->tx_ring[i]); 3370 for (i = 0; i < adapter->num_xdp_queues; i++) 3371 if (adapter->xdp_ring[i]->desc) 3372 ixgbevf_free_tx_resources(adapter->xdp_ring[i]); 3373 } 3374 3375 /** 3376 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 3377 * @tx_ring: Tx descriptor ring (for a specific queue) to setup 3378 * 3379 * Return 0 on success, negative on failure 3380 **/ 3381 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) 3382 { 3383 struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); 3384 int size; 3385 3386 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 3387 tx_ring->tx_buffer_info = vmalloc(size); 3388 if (!tx_ring->tx_buffer_info) 3389 goto err; 3390 3391 u64_stats_init(&tx_ring->syncp); 3392 3393 /* round up to nearest 4K */ 3394 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 3395 tx_ring->size = ALIGN(tx_ring->size, 4096); 3396 3397 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, 3398 &tx_ring->dma, GFP_KERNEL); 3399 if (!tx_ring->desc) 3400 goto err; 3401 3402 return 0; 3403 3404 err: 3405 vfree(tx_ring->tx_buffer_info); 3406 tx_ring->tx_buffer_info = NULL; 3407 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit descriptor ring\n"); 3408 return -ENOMEM; 3409 } 3410 3411 /** 3412 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 3413 * @adapter: board private structure 3414 * 3415 * If this function returns with an error, then it's possible one or 3416 * more of the rings is populated (while the rest are not). It is the 3417 * callers duty to clean those orphaned rings. 3418 * 3419 * Return 0 on success, negative on failure 3420 **/ 3421 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 3422 { 3423 int i, j = 0, err = 0; 3424 3425 for (i = 0; i < adapter->num_tx_queues; i++) { 3426 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]); 3427 if (!err) 3428 continue; 3429 hw_dbg(&adapter->hw, "Allocation for Tx Queue %u failed\n", i); 3430 goto err_setup_tx; 3431 } 3432 3433 for (j = 0; j < adapter->num_xdp_queues; j++) { 3434 err = ixgbevf_setup_tx_resources(adapter->xdp_ring[j]); 3435 if (!err) 3436 continue; 3437 hw_dbg(&adapter->hw, "Allocation for XDP Queue %u failed\n", j); 3438 goto err_setup_tx; 3439 } 3440 3441 return 0; 3442 err_setup_tx: 3443 /* rewind the index freeing the rings as we go */ 3444 while (j--) 3445 ixgbevf_free_tx_resources(adapter->xdp_ring[j]); 3446 while (i--) 3447 ixgbevf_free_tx_resources(adapter->tx_ring[i]); 3448 3449 return err; 3450 } 3451 3452 /** 3453 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 3454 * @adapter: board private structure 3455 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 3456 * 3457 * Returns 0 on success, negative on failure 3458 **/ 3459 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, 3460 struct ixgbevf_ring *rx_ring) 3461 { 3462 int size; 3463 3464 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 3465 rx_ring->rx_buffer_info = vmalloc(size); 3466 if (!rx_ring->rx_buffer_info) 3467 goto err; 3468 3469 u64_stats_init(&rx_ring->syncp); 3470 3471 /* Round up to nearest 4K */ 3472 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 3473 rx_ring->size = ALIGN(rx_ring->size, 4096); 3474 3475 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, 3476 &rx_ring->dma, GFP_KERNEL); 3477 3478 if (!rx_ring->desc) 3479 goto err; 3480 3481 /* XDP RX-queue info */ 3482 if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, 3483 rx_ring->queue_index) < 0) 3484 goto err; 3485 3486 rx_ring->xdp_prog = adapter->xdp_prog; 3487 3488 return 0; 3489 err: 3490 vfree(rx_ring->rx_buffer_info); 3491 rx_ring->rx_buffer_info = NULL; 3492 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); 3493 return -ENOMEM; 3494 } 3495 3496 /** 3497 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 3498 * @adapter: board private structure 3499 * 3500 * If this function returns with an error, then it's possible one or 3501 * more of the rings is populated (while the rest are not). It is the 3502 * callers duty to clean those orphaned rings. 3503 * 3504 * Return 0 on success, negative on failure 3505 **/ 3506 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 3507 { 3508 int i, err = 0; 3509 3510 for (i = 0; i < adapter->num_rx_queues; i++) { 3511 err = ixgbevf_setup_rx_resources(adapter, adapter->rx_ring[i]); 3512 if (!err) 3513 continue; 3514 hw_dbg(&adapter->hw, "Allocation for Rx Queue %u failed\n", i); 3515 goto err_setup_rx; 3516 } 3517 3518 return 0; 3519 err_setup_rx: 3520 /* rewind the index freeing the rings as we go */ 3521 while (i--) 3522 ixgbevf_free_rx_resources(adapter->rx_ring[i]); 3523 return err; 3524 } 3525 3526 /** 3527 * ixgbevf_free_rx_resources - Free Rx Resources 3528 * @rx_ring: ring to clean the resources from 3529 * 3530 * Free all receive software resources 3531 **/ 3532 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) 3533 { 3534 ixgbevf_clean_rx_ring(rx_ring); 3535 3536 rx_ring->xdp_prog = NULL; 3537 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 3538 vfree(rx_ring->rx_buffer_info); 3539 rx_ring->rx_buffer_info = NULL; 3540 3541 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, 3542 rx_ring->dma); 3543 3544 rx_ring->desc = NULL; 3545 } 3546 3547 /** 3548 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 3549 * @adapter: board private structure 3550 * 3551 * Free all receive software resources 3552 **/ 3553 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 3554 { 3555 int i; 3556 3557 for (i = 0; i < adapter->num_rx_queues; i++) 3558 if (adapter->rx_ring[i]->desc) 3559 ixgbevf_free_rx_resources(adapter->rx_ring[i]); 3560 } 3561 3562 /** 3563 * ixgbevf_open - Called when a network interface is made active 3564 * @netdev: network interface device structure 3565 * 3566 * Returns 0 on success, negative value on failure 3567 * 3568 * The open entry point is called when a network interface is made 3569 * active by the system (IFF_UP). At this point all resources needed 3570 * for transmit and receive operations are allocated, the interrupt 3571 * handler is registered with the OS, the watchdog timer is started, 3572 * and the stack is notified that the interface is ready. 3573 **/ 3574 int ixgbevf_open(struct net_device *netdev) 3575 { 3576 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3577 struct ixgbe_hw *hw = &adapter->hw; 3578 int err; 3579 3580 /* A previous failure to open the device because of a lack of 3581 * available MSIX vector resources may have reset the number 3582 * of msix vectors variable to zero. The only way to recover 3583 * is to unload/reload the driver and hope that the system has 3584 * been able to recover some MSIX vector resources. 3585 */ 3586 if (!adapter->num_msix_vectors) 3587 return -ENOMEM; 3588 3589 if (hw->adapter_stopped) { 3590 ixgbevf_reset(adapter); 3591 /* if adapter is still stopped then PF isn't up and 3592 * the VF can't start. 3593 */ 3594 if (hw->adapter_stopped) { 3595 err = IXGBE_ERR_MBX; 3596 pr_err("Unable to start - perhaps the PF Driver isn't up yet\n"); 3597 goto err_setup_reset; 3598 } 3599 } 3600 3601 /* disallow open during test */ 3602 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 3603 return -EBUSY; 3604 3605 netif_carrier_off(netdev); 3606 3607 /* allocate transmit descriptors */ 3608 err = ixgbevf_setup_all_tx_resources(adapter); 3609 if (err) 3610 goto err_setup_tx; 3611 3612 /* allocate receive descriptors */ 3613 err = ixgbevf_setup_all_rx_resources(adapter); 3614 if (err) 3615 goto err_setup_rx; 3616 3617 ixgbevf_configure(adapter); 3618 3619 err = ixgbevf_request_irq(adapter); 3620 if (err) 3621 goto err_req_irq; 3622 3623 /* Notify the stack of the actual queue counts. */ 3624 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 3625 if (err) 3626 goto err_set_queues; 3627 3628 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 3629 if (err) 3630 goto err_set_queues; 3631 3632 ixgbevf_up_complete(adapter); 3633 3634 return 0; 3635 3636 err_set_queues: 3637 ixgbevf_free_irq(adapter); 3638 err_req_irq: 3639 ixgbevf_free_all_rx_resources(adapter); 3640 err_setup_rx: 3641 ixgbevf_free_all_tx_resources(adapter); 3642 err_setup_tx: 3643 ixgbevf_reset(adapter); 3644 err_setup_reset: 3645 3646 return err; 3647 } 3648 3649 /** 3650 * ixgbevf_close_suspend - actions necessary to both suspend and close flows 3651 * @adapter: the private adapter struct 3652 * 3653 * This function should contain the necessary work common to both suspending 3654 * and closing of the device. 3655 */ 3656 static void ixgbevf_close_suspend(struct ixgbevf_adapter *adapter) 3657 { 3658 ixgbevf_down(adapter); 3659 ixgbevf_free_irq(adapter); 3660 ixgbevf_free_all_tx_resources(adapter); 3661 ixgbevf_free_all_rx_resources(adapter); 3662 } 3663 3664 /** 3665 * ixgbevf_close - Disables a network interface 3666 * @netdev: network interface device structure 3667 * 3668 * Returns 0, this is not allowed to fail 3669 * 3670 * The close entry point is called when an interface is de-activated 3671 * by the OS. The hardware is still under the drivers control, but 3672 * needs to be disabled. A global MAC reset is issued to stop the 3673 * hardware, and all transmit and receive resources are freed. 3674 **/ 3675 int ixgbevf_close(struct net_device *netdev) 3676 { 3677 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3678 3679 if (netif_device_present(netdev)) 3680 ixgbevf_close_suspend(adapter); 3681 3682 return 0; 3683 } 3684 3685 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter) 3686 { 3687 struct net_device *dev = adapter->netdev; 3688 3689 if (!test_and_clear_bit(__IXGBEVF_QUEUE_RESET_REQUESTED, 3690 &adapter->state)) 3691 return; 3692 3693 /* if interface is down do nothing */ 3694 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3695 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3696 return; 3697 3698 /* Hardware has to reinitialize queues and interrupts to 3699 * match packet buffer alignment. Unfortunately, the 3700 * hardware is not flexible enough to do this dynamically. 3701 */ 3702 rtnl_lock(); 3703 3704 if (netif_running(dev)) 3705 ixgbevf_close(dev); 3706 3707 ixgbevf_clear_interrupt_scheme(adapter); 3708 ixgbevf_init_interrupt_scheme(adapter); 3709 3710 if (netif_running(dev)) 3711 ixgbevf_open(dev); 3712 3713 rtnl_unlock(); 3714 } 3715 3716 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, 3717 u32 vlan_macip_lens, u32 fceof_saidx, 3718 u32 type_tucmd, u32 mss_l4len_idx) 3719 { 3720 struct ixgbe_adv_tx_context_desc *context_desc; 3721 u16 i = tx_ring->next_to_use; 3722 3723 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i); 3724 3725 i++; 3726 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 3727 3728 /* set bits to identify this as an advanced context descriptor */ 3729 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3730 3731 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 3732 context_desc->fceof_saidx = cpu_to_le32(fceof_saidx); 3733 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 3734 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 3735 } 3736 3737 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, 3738 struct ixgbevf_tx_buffer *first, 3739 u8 *hdr_len, 3740 struct ixgbevf_ipsec_tx_data *itd) 3741 { 3742 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 3743 struct sk_buff *skb = first->skb; 3744 union { 3745 struct iphdr *v4; 3746 struct ipv6hdr *v6; 3747 unsigned char *hdr; 3748 } ip; 3749 union { 3750 struct tcphdr *tcp; 3751 unsigned char *hdr; 3752 } l4; 3753 u32 paylen, l4_offset; 3754 u32 fceof_saidx = 0; 3755 int err; 3756 3757 if (skb->ip_summed != CHECKSUM_PARTIAL) 3758 return 0; 3759 3760 if (!skb_is_gso(skb)) 3761 return 0; 3762 3763 err = skb_cow_head(skb, 0); 3764 if (err < 0) 3765 return err; 3766 3767 if (eth_p_mpls(first->protocol)) 3768 ip.hdr = skb_inner_network_header(skb); 3769 else 3770 ip.hdr = skb_network_header(skb); 3771 l4.hdr = skb_checksum_start(skb); 3772 3773 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 3774 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3775 3776 /* initialize outer IP header fields */ 3777 if (ip.v4->version == 4) { 3778 unsigned char *csum_start = skb_checksum_start(skb); 3779 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 3780 int len = csum_start - trans_start; 3781 3782 /* IP header will have to cancel out any data that 3783 * is not a part of the outer IP header, so set to 3784 * a reverse csum if needed, else init check to 0. 3785 */ 3786 ip.v4->check = (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) ? 3787 csum_fold(csum_partial(trans_start, 3788 len, 0)) : 0; 3789 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3790 3791 ip.v4->tot_len = 0; 3792 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3793 IXGBE_TX_FLAGS_CSUM | 3794 IXGBE_TX_FLAGS_IPV4; 3795 } else { 3796 ip.v6->payload_len = 0; 3797 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3798 IXGBE_TX_FLAGS_CSUM; 3799 } 3800 3801 /* determine offset of inner transport header */ 3802 l4_offset = l4.hdr - skb->data; 3803 3804 /* compute length of segmentation header */ 3805 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 3806 3807 /* remove payload length from inner checksum */ 3808 paylen = skb->len - l4_offset; 3809 csum_replace_by_diff(&l4.tcp->check, htonl(paylen)); 3810 3811 /* update gso size and bytecount with header size */ 3812 first->gso_segs = skb_shinfo(skb)->gso_segs; 3813 first->bytecount += (first->gso_segs - 1) * *hdr_len; 3814 3815 /* mss_l4len_id: use 1 as index for TSO */ 3816 mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT; 3817 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 3818 mss_l4len_idx |= (1u << IXGBE_ADVTXD_IDX_SHIFT); 3819 3820 fceof_saidx |= itd->pfsa; 3821 type_tucmd |= itd->flags | itd->trailer_len; 3822 3823 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 3824 vlan_macip_lens = l4.hdr - ip.hdr; 3825 vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT; 3826 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3827 3828 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd, 3829 mss_l4len_idx); 3830 3831 return 1; 3832 } 3833 3834 static inline bool ixgbevf_ipv6_csum_is_sctp(struct sk_buff *skb) 3835 { 3836 unsigned int offset = 0; 3837 3838 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 3839 3840 return offset == skb_checksum_start_offset(skb); 3841 } 3842 3843 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, 3844 struct ixgbevf_tx_buffer *first, 3845 struct ixgbevf_ipsec_tx_data *itd) 3846 { 3847 struct sk_buff *skb = first->skb; 3848 u32 vlan_macip_lens = 0; 3849 u32 fceof_saidx = 0; 3850 u32 type_tucmd = 0; 3851 3852 if (skb->ip_summed != CHECKSUM_PARTIAL) 3853 goto no_csum; 3854 3855 switch (skb->csum_offset) { 3856 case offsetof(struct tcphdr, check): 3857 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3858 /* fall through */ 3859 case offsetof(struct udphdr, check): 3860 break; 3861 case offsetof(struct sctphdr, checksum): 3862 /* validate that this is actually an SCTP request */ 3863 if (((first->protocol == htons(ETH_P_IP)) && 3864 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 3865 ((first->protocol == htons(ETH_P_IPV6)) && 3866 ixgbevf_ipv6_csum_is_sctp(skb))) { 3867 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_SCTP; 3868 break; 3869 } 3870 /* fall through */ 3871 default: 3872 skb_checksum_help(skb); 3873 goto no_csum; 3874 } 3875 3876 if (first->protocol == htons(ETH_P_IP)) 3877 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3878 3879 /* update TX checksum flag */ 3880 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 3881 vlan_macip_lens = skb_checksum_start_offset(skb) - 3882 skb_network_offset(skb); 3883 no_csum: 3884 /* vlan_macip_lens: MACLEN, VLAN tag */ 3885 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3886 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3887 3888 fceof_saidx |= itd->pfsa; 3889 type_tucmd |= itd->flags | itd->trailer_len; 3890 3891 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3892 fceof_saidx, type_tucmd, 0); 3893 } 3894 3895 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags) 3896 { 3897 /* set type for advanced descriptor with frame checksum insertion */ 3898 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | 3899 IXGBE_ADVTXD_DCMD_IFCS | 3900 IXGBE_ADVTXD_DCMD_DEXT); 3901 3902 /* set HW VLAN bit if VLAN is present */ 3903 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 3904 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); 3905 3906 /* set segmentation enable bits for TSO/FSO */ 3907 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3908 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); 3909 3910 return cmd_type; 3911 } 3912 3913 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 3914 u32 tx_flags, unsigned int paylen) 3915 { 3916 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT); 3917 3918 /* enable L4 checksum for TSO and TX checksum offload */ 3919 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 3920 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM); 3921 3922 /* enble IPv4 checksum for TSO */ 3923 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 3924 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM); 3925 3926 /* enable IPsec */ 3927 if (tx_flags & IXGBE_TX_FLAGS_IPSEC) 3928 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IPSEC); 3929 3930 /* use index 1 context for TSO/FSO/FCOE/IPSEC */ 3931 if (tx_flags & (IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_IPSEC)) 3932 olinfo_status |= cpu_to_le32(1u << IXGBE_ADVTXD_IDX_SHIFT); 3933 3934 /* Check Context must be set if Tx switch is enabled, which it 3935 * always is for case where virtual functions are running 3936 */ 3937 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC); 3938 3939 tx_desc->read.olinfo_status = olinfo_status; 3940 } 3941 3942 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, 3943 struct ixgbevf_tx_buffer *first, 3944 const u8 hdr_len) 3945 { 3946 struct sk_buff *skb = first->skb; 3947 struct ixgbevf_tx_buffer *tx_buffer; 3948 union ixgbe_adv_tx_desc *tx_desc; 3949 struct skb_frag_struct *frag; 3950 dma_addr_t dma; 3951 unsigned int data_len, size; 3952 u32 tx_flags = first->tx_flags; 3953 __le32 cmd_type = ixgbevf_tx_cmd_type(tx_flags); 3954 u16 i = tx_ring->next_to_use; 3955 3956 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 3957 3958 ixgbevf_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len); 3959 3960 size = skb_headlen(skb); 3961 data_len = skb->data_len; 3962 3963 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 3964 3965 tx_buffer = first; 3966 3967 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 3968 if (dma_mapping_error(tx_ring->dev, dma)) 3969 goto dma_error; 3970 3971 /* record length, and DMA address */ 3972 dma_unmap_len_set(tx_buffer, len, size); 3973 dma_unmap_addr_set(tx_buffer, dma, dma); 3974 3975 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3976 3977 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 3978 tx_desc->read.cmd_type_len = 3979 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); 3980 3981 i++; 3982 tx_desc++; 3983 if (i == tx_ring->count) { 3984 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3985 i = 0; 3986 } 3987 tx_desc->read.olinfo_status = 0; 3988 3989 dma += IXGBE_MAX_DATA_PER_TXD; 3990 size -= IXGBE_MAX_DATA_PER_TXD; 3991 3992 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3993 } 3994 3995 if (likely(!data_len)) 3996 break; 3997 3998 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); 3999 4000 i++; 4001 tx_desc++; 4002 if (i == tx_ring->count) { 4003 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 4004 i = 0; 4005 } 4006 tx_desc->read.olinfo_status = 0; 4007 4008 size = skb_frag_size(frag); 4009 data_len -= size; 4010 4011 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 4012 DMA_TO_DEVICE); 4013 4014 tx_buffer = &tx_ring->tx_buffer_info[i]; 4015 } 4016 4017 /* write last descriptor with RS and EOP bits */ 4018 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); 4019 tx_desc->read.cmd_type_len = cmd_type; 4020 4021 /* set the timestamp */ 4022 first->time_stamp = jiffies; 4023 4024 skb_tx_timestamp(skb); 4025 4026 /* Force memory writes to complete before letting h/w know there 4027 * are new descriptors to fetch. (Only applicable for weak-ordered 4028 * memory model archs, such as IA-64). 4029 * 4030 * We also need this memory barrier (wmb) to make certain all of the 4031 * status bits have been updated before next_to_watch is written. 4032 */ 4033 wmb(); 4034 4035 /* set next_to_watch value indicating a packet is present */ 4036 first->next_to_watch = tx_desc; 4037 4038 i++; 4039 if (i == tx_ring->count) 4040 i = 0; 4041 4042 tx_ring->next_to_use = i; 4043 4044 /* notify HW of packet */ 4045 ixgbevf_write_tail(tx_ring, i); 4046 4047 return; 4048 dma_error: 4049 dev_err(tx_ring->dev, "TX DMA map failed\n"); 4050 tx_buffer = &tx_ring->tx_buffer_info[i]; 4051 4052 /* clear dma mappings for failed tx_buffer_info map */ 4053 while (tx_buffer != first) { 4054 if (dma_unmap_len(tx_buffer, len)) 4055 dma_unmap_page(tx_ring->dev, 4056 dma_unmap_addr(tx_buffer, dma), 4057 dma_unmap_len(tx_buffer, len), 4058 DMA_TO_DEVICE); 4059 dma_unmap_len_set(tx_buffer, len, 0); 4060 4061 if (i-- == 0) 4062 i += tx_ring->count; 4063 tx_buffer = &tx_ring->tx_buffer_info[i]; 4064 } 4065 4066 if (dma_unmap_len(tx_buffer, len)) 4067 dma_unmap_single(tx_ring->dev, 4068 dma_unmap_addr(tx_buffer, dma), 4069 dma_unmap_len(tx_buffer, len), 4070 DMA_TO_DEVICE); 4071 dma_unmap_len_set(tx_buffer, len, 0); 4072 4073 dev_kfree_skb_any(tx_buffer->skb); 4074 tx_buffer->skb = NULL; 4075 4076 tx_ring->next_to_use = i; 4077 } 4078 4079 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 4080 { 4081 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 4082 /* Herbert's original patch had: 4083 * smp_mb__after_netif_stop_queue(); 4084 * but since that doesn't exist yet, just open code it. 4085 */ 4086 smp_mb(); 4087 4088 /* We need to check again in a case another CPU has just 4089 * made room available. 4090 */ 4091 if (likely(ixgbevf_desc_unused(tx_ring) < size)) 4092 return -EBUSY; 4093 4094 /* A reprieve! - use start_queue because it doesn't call schedule */ 4095 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 4096 ++tx_ring->tx_stats.restart_queue; 4097 4098 return 0; 4099 } 4100 4101 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 4102 { 4103 if (likely(ixgbevf_desc_unused(tx_ring) >= size)) 4104 return 0; 4105 return __ixgbevf_maybe_stop_tx(tx_ring, size); 4106 } 4107 4108 static int ixgbevf_xmit_frame_ring(struct sk_buff *skb, 4109 struct ixgbevf_ring *tx_ring) 4110 { 4111 struct ixgbevf_tx_buffer *first; 4112 int tso; 4113 u32 tx_flags = 0; 4114 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 4115 struct ixgbevf_ipsec_tx_data ipsec_tx = { 0 }; 4116 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 4117 unsigned short f; 4118 #endif 4119 u8 hdr_len = 0; 4120 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL); 4121 4122 if (!dst_mac || is_link_local_ether_addr(dst_mac)) { 4123 dev_kfree_skb_any(skb); 4124 return NETDEV_TX_OK; 4125 } 4126 4127 /* need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 4128 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 4129 * + 2 desc gap to keep tail from touching head, 4130 * + 1 desc for context descriptor, 4131 * otherwise try next time 4132 */ 4133 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 4134 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 4135 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 4136 #else 4137 count += skb_shinfo(skb)->nr_frags; 4138 #endif 4139 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { 4140 tx_ring->tx_stats.tx_busy++; 4141 return NETDEV_TX_BUSY; 4142 } 4143 4144 /* record the location of the first descriptor for this packet */ 4145 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 4146 first->skb = skb; 4147 first->bytecount = skb->len; 4148 first->gso_segs = 1; 4149 4150 if (skb_vlan_tag_present(skb)) { 4151 tx_flags |= skb_vlan_tag_get(skb); 4152 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 4153 tx_flags |= IXGBE_TX_FLAGS_VLAN; 4154 } 4155 4156 /* record initial flags and protocol */ 4157 first->tx_flags = tx_flags; 4158 first->protocol = vlan_get_protocol(skb); 4159 4160 #ifdef CONFIG_IXGBEVF_IPSEC 4161 if (secpath_exists(skb) && !ixgbevf_ipsec_tx(tx_ring, first, &ipsec_tx)) 4162 goto out_drop; 4163 #endif 4164 tso = ixgbevf_tso(tx_ring, first, &hdr_len, &ipsec_tx); 4165 if (tso < 0) 4166 goto out_drop; 4167 else if (!tso) 4168 ixgbevf_tx_csum(tx_ring, first, &ipsec_tx); 4169 4170 ixgbevf_tx_map(tx_ring, first, hdr_len); 4171 4172 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); 4173 4174 return NETDEV_TX_OK; 4175 4176 out_drop: 4177 dev_kfree_skb_any(first->skb); 4178 first->skb = NULL; 4179 4180 return NETDEV_TX_OK; 4181 } 4182 4183 static netdev_tx_t ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 4184 { 4185 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4186 struct ixgbevf_ring *tx_ring; 4187 4188 if (skb->len <= 0) { 4189 dev_kfree_skb_any(skb); 4190 return NETDEV_TX_OK; 4191 } 4192 4193 /* The minimum packet size for olinfo paylen is 17 so pad the skb 4194 * in order to meet this minimum size requirement. 4195 */ 4196 if (skb->len < 17) { 4197 if (skb_padto(skb, 17)) 4198 return NETDEV_TX_OK; 4199 skb->len = 17; 4200 } 4201 4202 tx_ring = adapter->tx_ring[skb->queue_mapping]; 4203 return ixgbevf_xmit_frame_ring(skb, tx_ring); 4204 } 4205 4206 /** 4207 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 4208 * @netdev: network interface device structure 4209 * @p: pointer to an address structure 4210 * 4211 * Returns 0 on success, negative on failure 4212 **/ 4213 static int ixgbevf_set_mac(struct net_device *netdev, void *p) 4214 { 4215 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4216 struct ixgbe_hw *hw = &adapter->hw; 4217 struct sockaddr *addr = p; 4218 int err; 4219 4220 if (!is_valid_ether_addr(addr->sa_data)) 4221 return -EADDRNOTAVAIL; 4222 4223 spin_lock_bh(&adapter->mbx_lock); 4224 4225 err = hw->mac.ops.set_rar(hw, 0, addr->sa_data, 0); 4226 4227 spin_unlock_bh(&adapter->mbx_lock); 4228 4229 if (err) 4230 return -EPERM; 4231 4232 ether_addr_copy(hw->mac.addr, addr->sa_data); 4233 ether_addr_copy(hw->mac.perm_addr, addr->sa_data); 4234 ether_addr_copy(netdev->dev_addr, addr->sa_data); 4235 4236 return 0; 4237 } 4238 4239 /** 4240 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 4241 * @netdev: network interface device structure 4242 * @new_mtu: new value for maximum frame size 4243 * 4244 * Returns 0 on success, negative on failure 4245 **/ 4246 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 4247 { 4248 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4249 struct ixgbe_hw *hw = &adapter->hw; 4250 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 4251 int ret; 4252 4253 /* prevent MTU being changed to a size unsupported by XDP */ 4254 if (adapter->xdp_prog) { 4255 dev_warn(&adapter->pdev->dev, "MTU cannot be changed while XDP program is loaded\n"); 4256 return -EPERM; 4257 } 4258 4259 spin_lock_bh(&adapter->mbx_lock); 4260 /* notify the PF of our intent to use this size of frame */ 4261 ret = hw->mac.ops.set_rlpml(hw, max_frame); 4262 spin_unlock_bh(&adapter->mbx_lock); 4263 if (ret) 4264 return -EINVAL; 4265 4266 hw_dbg(hw, "changing MTU from %d to %d\n", 4267 netdev->mtu, new_mtu); 4268 4269 /* must set new MTU before calling down or up */ 4270 netdev->mtu = new_mtu; 4271 4272 if (netif_running(netdev)) 4273 ixgbevf_reinit_locked(adapter); 4274 4275 return 0; 4276 } 4277 4278 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) 4279 { 4280 struct net_device *netdev = pci_get_drvdata(pdev); 4281 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4282 #ifdef CONFIG_PM 4283 int retval = 0; 4284 #endif 4285 4286 rtnl_lock(); 4287 netif_device_detach(netdev); 4288 4289 if (netif_running(netdev)) 4290 ixgbevf_close_suspend(adapter); 4291 4292 ixgbevf_clear_interrupt_scheme(adapter); 4293 rtnl_unlock(); 4294 4295 #ifdef CONFIG_PM 4296 retval = pci_save_state(pdev); 4297 if (retval) 4298 return retval; 4299 4300 #endif 4301 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 4302 pci_disable_device(pdev); 4303 4304 return 0; 4305 } 4306 4307 #ifdef CONFIG_PM 4308 static int ixgbevf_resume(struct pci_dev *pdev) 4309 { 4310 struct net_device *netdev = pci_get_drvdata(pdev); 4311 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4312 u32 err; 4313 4314 pci_restore_state(pdev); 4315 /* pci_restore_state clears dev->state_saved so call 4316 * pci_save_state to restore it. 4317 */ 4318 pci_save_state(pdev); 4319 4320 err = pci_enable_device_mem(pdev); 4321 if (err) { 4322 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 4323 return err; 4324 } 4325 4326 adapter->hw.hw_addr = adapter->io_addr; 4327 smp_mb__before_atomic(); 4328 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 4329 pci_set_master(pdev); 4330 4331 ixgbevf_reset(adapter); 4332 4333 rtnl_lock(); 4334 err = ixgbevf_init_interrupt_scheme(adapter); 4335 if (!err && netif_running(netdev)) 4336 err = ixgbevf_open(netdev); 4337 rtnl_unlock(); 4338 if (err) 4339 return err; 4340 4341 netif_device_attach(netdev); 4342 4343 return err; 4344 } 4345 4346 #endif /* CONFIG_PM */ 4347 static void ixgbevf_shutdown(struct pci_dev *pdev) 4348 { 4349 ixgbevf_suspend(pdev, PMSG_SUSPEND); 4350 } 4351 4352 static void ixgbevf_get_tx_ring_stats(struct rtnl_link_stats64 *stats, 4353 const struct ixgbevf_ring *ring) 4354 { 4355 u64 bytes, packets; 4356 unsigned int start; 4357 4358 if (ring) { 4359 do { 4360 start = u64_stats_fetch_begin_irq(&ring->syncp); 4361 bytes = ring->stats.bytes; 4362 packets = ring->stats.packets; 4363 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 4364 stats->tx_bytes += bytes; 4365 stats->tx_packets += packets; 4366 } 4367 } 4368 4369 static void ixgbevf_get_stats(struct net_device *netdev, 4370 struct rtnl_link_stats64 *stats) 4371 { 4372 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4373 unsigned int start; 4374 u64 bytes, packets; 4375 const struct ixgbevf_ring *ring; 4376 int i; 4377 4378 ixgbevf_update_stats(adapter); 4379 4380 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 4381 4382 rcu_read_lock(); 4383 for (i = 0; i < adapter->num_rx_queues; i++) { 4384 ring = adapter->rx_ring[i]; 4385 do { 4386 start = u64_stats_fetch_begin_irq(&ring->syncp); 4387 bytes = ring->stats.bytes; 4388 packets = ring->stats.packets; 4389 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 4390 stats->rx_bytes += bytes; 4391 stats->rx_packets += packets; 4392 } 4393 4394 for (i = 0; i < adapter->num_tx_queues; i++) { 4395 ring = adapter->tx_ring[i]; 4396 ixgbevf_get_tx_ring_stats(stats, ring); 4397 } 4398 4399 for (i = 0; i < adapter->num_xdp_queues; i++) { 4400 ring = adapter->xdp_ring[i]; 4401 ixgbevf_get_tx_ring_stats(stats, ring); 4402 } 4403 rcu_read_unlock(); 4404 } 4405 4406 #define IXGBEVF_MAX_MAC_HDR_LEN 127 4407 #define IXGBEVF_MAX_NETWORK_HDR_LEN 511 4408 4409 static netdev_features_t 4410 ixgbevf_features_check(struct sk_buff *skb, struct net_device *dev, 4411 netdev_features_t features) 4412 { 4413 unsigned int network_hdr_len, mac_hdr_len; 4414 4415 /* Make certain the headers can be described by a context descriptor */ 4416 mac_hdr_len = skb_network_header(skb) - skb->data; 4417 if (unlikely(mac_hdr_len > IXGBEVF_MAX_MAC_HDR_LEN)) 4418 return features & ~(NETIF_F_HW_CSUM | 4419 NETIF_F_SCTP_CRC | 4420 NETIF_F_HW_VLAN_CTAG_TX | 4421 NETIF_F_TSO | 4422 NETIF_F_TSO6); 4423 4424 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 4425 if (unlikely(network_hdr_len > IXGBEVF_MAX_NETWORK_HDR_LEN)) 4426 return features & ~(NETIF_F_HW_CSUM | 4427 NETIF_F_SCTP_CRC | 4428 NETIF_F_TSO | 4429 NETIF_F_TSO6); 4430 4431 /* We can only support IPV4 TSO in tunnels if we can mangle the 4432 * inner IP ID field, so strip TSO if MANGLEID is not supported. 4433 */ 4434 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 4435 features &= ~NETIF_F_TSO; 4436 4437 return features; 4438 } 4439 4440 static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) 4441 { 4442 int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 4443 struct ixgbevf_adapter *adapter = netdev_priv(dev); 4444 struct bpf_prog *old_prog; 4445 4446 /* verify ixgbevf ring attributes are sufficient for XDP */ 4447 for (i = 0; i < adapter->num_rx_queues; i++) { 4448 struct ixgbevf_ring *ring = adapter->rx_ring[i]; 4449 4450 if (frame_size > ixgbevf_rx_bufsz(ring)) 4451 return -EINVAL; 4452 } 4453 4454 old_prog = xchg(&adapter->xdp_prog, prog); 4455 4456 /* If transitioning XDP modes reconfigure rings */ 4457 if (!!prog != !!old_prog) { 4458 /* Hardware has to reinitialize queues and interrupts to 4459 * match packet buffer alignment. Unfortunately, the 4460 * hardware is not flexible enough to do this dynamically. 4461 */ 4462 if (netif_running(dev)) 4463 ixgbevf_close(dev); 4464 4465 ixgbevf_clear_interrupt_scheme(adapter); 4466 ixgbevf_init_interrupt_scheme(adapter); 4467 4468 if (netif_running(dev)) 4469 ixgbevf_open(dev); 4470 } else { 4471 for (i = 0; i < adapter->num_rx_queues; i++) 4472 xchg(&adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog); 4473 } 4474 4475 if (old_prog) 4476 bpf_prog_put(old_prog); 4477 4478 return 0; 4479 } 4480 4481 static int ixgbevf_xdp(struct net_device *dev, struct netdev_bpf *xdp) 4482 { 4483 struct ixgbevf_adapter *adapter = netdev_priv(dev); 4484 4485 switch (xdp->command) { 4486 case XDP_SETUP_PROG: 4487 return ixgbevf_xdp_setup(dev, xdp->prog); 4488 case XDP_QUERY_PROG: 4489 xdp->prog_id = adapter->xdp_prog ? 4490 adapter->xdp_prog->aux->id : 0; 4491 return 0; 4492 default: 4493 return -EINVAL; 4494 } 4495 } 4496 4497 static const struct net_device_ops ixgbevf_netdev_ops = { 4498 .ndo_open = ixgbevf_open, 4499 .ndo_stop = ixgbevf_close, 4500 .ndo_start_xmit = ixgbevf_xmit_frame, 4501 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 4502 .ndo_get_stats64 = ixgbevf_get_stats, 4503 .ndo_validate_addr = eth_validate_addr, 4504 .ndo_set_mac_address = ixgbevf_set_mac, 4505 .ndo_change_mtu = ixgbevf_change_mtu, 4506 .ndo_tx_timeout = ixgbevf_tx_timeout, 4507 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 4508 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 4509 .ndo_features_check = ixgbevf_features_check, 4510 .ndo_bpf = ixgbevf_xdp, 4511 }; 4512 4513 static void ixgbevf_assign_netdev_ops(struct net_device *dev) 4514 { 4515 dev->netdev_ops = &ixgbevf_netdev_ops; 4516 ixgbevf_set_ethtool_ops(dev); 4517 dev->watchdog_timeo = 5 * HZ; 4518 } 4519 4520 /** 4521 * ixgbevf_probe - Device Initialization Routine 4522 * @pdev: PCI device information struct 4523 * @ent: entry in ixgbevf_pci_tbl 4524 * 4525 * Returns 0 on success, negative on failure 4526 * 4527 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 4528 * The OS initialization, configuring of the adapter private structure, 4529 * and a hardware reset occur. 4530 **/ 4531 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 4532 { 4533 struct net_device *netdev; 4534 struct ixgbevf_adapter *adapter = NULL; 4535 struct ixgbe_hw *hw = NULL; 4536 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 4537 int err, pci_using_dac; 4538 bool disable_dev = false; 4539 4540 err = pci_enable_device(pdev); 4541 if (err) 4542 return err; 4543 4544 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 4545 pci_using_dac = 1; 4546 } else { 4547 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 4548 if (err) { 4549 dev_err(&pdev->dev, "No usable DMA configuration, aborting\n"); 4550 goto err_dma; 4551 } 4552 pci_using_dac = 0; 4553 } 4554 4555 err = pci_request_regions(pdev, ixgbevf_driver_name); 4556 if (err) { 4557 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 4558 goto err_pci_reg; 4559 } 4560 4561 pci_set_master(pdev); 4562 4563 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 4564 MAX_TX_QUEUES); 4565 if (!netdev) { 4566 err = -ENOMEM; 4567 goto err_alloc_etherdev; 4568 } 4569 4570 SET_NETDEV_DEV(netdev, &pdev->dev); 4571 4572 adapter = netdev_priv(netdev); 4573 4574 adapter->netdev = netdev; 4575 adapter->pdev = pdev; 4576 hw = &adapter->hw; 4577 hw->back = adapter; 4578 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 4579 4580 /* call save state here in standalone driver because it relies on 4581 * adapter struct to exist, and needs to call netdev_priv 4582 */ 4583 pci_save_state(pdev); 4584 4585 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 4586 pci_resource_len(pdev, 0)); 4587 adapter->io_addr = hw->hw_addr; 4588 if (!hw->hw_addr) { 4589 err = -EIO; 4590 goto err_ioremap; 4591 } 4592 4593 ixgbevf_assign_netdev_ops(netdev); 4594 4595 /* Setup HW API */ 4596 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 4597 hw->mac.type = ii->mac; 4598 4599 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 4600 sizeof(struct ixgbe_mbx_operations)); 4601 4602 /* setup the private structure */ 4603 err = ixgbevf_sw_init(adapter); 4604 if (err) 4605 goto err_sw_init; 4606 4607 /* The HW MAC address was set and/or determined in sw_init */ 4608 if (!is_valid_ether_addr(netdev->dev_addr)) { 4609 pr_err("invalid MAC address\n"); 4610 err = -EIO; 4611 goto err_sw_init; 4612 } 4613 4614 netdev->hw_features = NETIF_F_SG | 4615 NETIF_F_TSO | 4616 NETIF_F_TSO6 | 4617 NETIF_F_RXCSUM | 4618 NETIF_F_HW_CSUM | 4619 NETIF_F_SCTP_CRC; 4620 4621 #define IXGBEVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 4622 NETIF_F_GSO_GRE_CSUM | \ 4623 NETIF_F_GSO_IPXIP4 | \ 4624 NETIF_F_GSO_IPXIP6 | \ 4625 NETIF_F_GSO_UDP_TUNNEL | \ 4626 NETIF_F_GSO_UDP_TUNNEL_CSUM) 4627 4628 netdev->gso_partial_features = IXGBEVF_GSO_PARTIAL_FEATURES; 4629 netdev->hw_features |= NETIF_F_GSO_PARTIAL | 4630 IXGBEVF_GSO_PARTIAL_FEATURES; 4631 4632 netdev->features = netdev->hw_features; 4633 4634 if (pci_using_dac) 4635 netdev->features |= NETIF_F_HIGHDMA; 4636 4637 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; 4638 netdev->mpls_features |= NETIF_F_SG | 4639 NETIF_F_TSO | 4640 NETIF_F_TSO6 | 4641 NETIF_F_HW_CSUM; 4642 netdev->mpls_features |= IXGBEVF_GSO_PARTIAL_FEATURES; 4643 netdev->hw_enc_features |= netdev->vlan_features; 4644 4645 /* set this bit last since it cannot be part of vlan_features */ 4646 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | 4647 NETIF_F_HW_VLAN_CTAG_RX | 4648 NETIF_F_HW_VLAN_CTAG_TX; 4649 4650 netdev->priv_flags |= IFF_UNICAST_FLT; 4651 4652 /* MTU range: 68 - 1504 or 9710 */ 4653 netdev->min_mtu = ETH_MIN_MTU; 4654 switch (adapter->hw.api_version) { 4655 case ixgbe_mbox_api_11: 4656 case ixgbe_mbox_api_12: 4657 case ixgbe_mbox_api_13: 4658 case ixgbe_mbox_api_14: 4659 netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - 4660 (ETH_HLEN + ETH_FCS_LEN); 4661 break; 4662 default: 4663 if (adapter->hw.mac.type != ixgbe_mac_82599_vf) 4664 netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - 4665 (ETH_HLEN + ETH_FCS_LEN); 4666 else 4667 netdev->max_mtu = ETH_DATA_LEN + ETH_FCS_LEN; 4668 break; 4669 } 4670 4671 if (IXGBE_REMOVED(hw->hw_addr)) { 4672 err = -EIO; 4673 goto err_sw_init; 4674 } 4675 4676 timer_setup(&adapter->service_timer, ixgbevf_service_timer, 0); 4677 4678 INIT_WORK(&adapter->service_task, ixgbevf_service_task); 4679 set_bit(__IXGBEVF_SERVICE_INITED, &adapter->state); 4680 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 4681 4682 err = ixgbevf_init_interrupt_scheme(adapter); 4683 if (err) 4684 goto err_sw_init; 4685 4686 strcpy(netdev->name, "eth%d"); 4687 4688 err = register_netdev(netdev); 4689 if (err) 4690 goto err_register; 4691 4692 pci_set_drvdata(pdev, netdev); 4693 netif_carrier_off(netdev); 4694 ixgbevf_init_ipsec_offload(adapter); 4695 4696 ixgbevf_init_last_counter_stats(adapter); 4697 4698 /* print the VF info */ 4699 dev_info(&pdev->dev, "%pM\n", netdev->dev_addr); 4700 dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type); 4701 4702 switch (hw->mac.type) { 4703 case ixgbe_mac_X550_vf: 4704 dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n"); 4705 break; 4706 case ixgbe_mac_X540_vf: 4707 dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n"); 4708 break; 4709 case ixgbe_mac_82599_vf: 4710 default: 4711 dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n"); 4712 break; 4713 } 4714 4715 return 0; 4716 4717 err_register: 4718 ixgbevf_clear_interrupt_scheme(adapter); 4719 err_sw_init: 4720 ixgbevf_reset_interrupt_capability(adapter); 4721 iounmap(adapter->io_addr); 4722 kfree(adapter->rss_key); 4723 err_ioremap: 4724 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4725 free_netdev(netdev); 4726 err_alloc_etherdev: 4727 pci_release_regions(pdev); 4728 err_pci_reg: 4729 err_dma: 4730 if (!adapter || disable_dev) 4731 pci_disable_device(pdev); 4732 return err; 4733 } 4734 4735 /** 4736 * ixgbevf_remove - Device Removal Routine 4737 * @pdev: PCI device information struct 4738 * 4739 * ixgbevf_remove is called by the PCI subsystem to alert the driver 4740 * that it should release a PCI device. The could be caused by a 4741 * Hot-Plug event, or because the driver is going to be removed from 4742 * memory. 4743 **/ 4744 static void ixgbevf_remove(struct pci_dev *pdev) 4745 { 4746 struct net_device *netdev = pci_get_drvdata(pdev); 4747 struct ixgbevf_adapter *adapter; 4748 bool disable_dev; 4749 4750 if (!netdev) 4751 return; 4752 4753 adapter = netdev_priv(netdev); 4754 4755 set_bit(__IXGBEVF_REMOVING, &adapter->state); 4756 cancel_work_sync(&adapter->service_task); 4757 4758 if (netdev->reg_state == NETREG_REGISTERED) 4759 unregister_netdev(netdev); 4760 4761 ixgbevf_stop_ipsec_offload(adapter); 4762 ixgbevf_clear_interrupt_scheme(adapter); 4763 ixgbevf_reset_interrupt_capability(adapter); 4764 4765 iounmap(adapter->io_addr); 4766 pci_release_regions(pdev); 4767 4768 hw_dbg(&adapter->hw, "Remove complete\n"); 4769 4770 kfree(adapter->rss_key); 4771 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4772 free_netdev(netdev); 4773 4774 if (disable_dev) 4775 pci_disable_device(pdev); 4776 } 4777 4778 /** 4779 * ixgbevf_io_error_detected - called when PCI error is detected 4780 * @pdev: Pointer to PCI device 4781 * @state: The current pci connection state 4782 * 4783 * This function is called after a PCI bus error affecting 4784 * this device has been detected. 4785 **/ 4786 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, 4787 pci_channel_state_t state) 4788 { 4789 struct net_device *netdev = pci_get_drvdata(pdev); 4790 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4791 4792 if (!test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 4793 return PCI_ERS_RESULT_DISCONNECT; 4794 4795 rtnl_lock(); 4796 netif_device_detach(netdev); 4797 4798 if (netif_running(netdev)) 4799 ixgbevf_close_suspend(adapter); 4800 4801 if (state == pci_channel_io_perm_failure) { 4802 rtnl_unlock(); 4803 return PCI_ERS_RESULT_DISCONNECT; 4804 } 4805 4806 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 4807 pci_disable_device(pdev); 4808 rtnl_unlock(); 4809 4810 /* Request a slot slot reset. */ 4811 return PCI_ERS_RESULT_NEED_RESET; 4812 } 4813 4814 /** 4815 * ixgbevf_io_slot_reset - called after the pci bus has been reset. 4816 * @pdev: Pointer to PCI device 4817 * 4818 * Restart the card from scratch, as if from a cold-boot. Implementation 4819 * resembles the first-half of the ixgbevf_resume routine. 4820 **/ 4821 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) 4822 { 4823 struct net_device *netdev = pci_get_drvdata(pdev); 4824 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4825 4826 if (pci_enable_device_mem(pdev)) { 4827 dev_err(&pdev->dev, 4828 "Cannot re-enable PCI device after reset.\n"); 4829 return PCI_ERS_RESULT_DISCONNECT; 4830 } 4831 4832 adapter->hw.hw_addr = adapter->io_addr; 4833 smp_mb__before_atomic(); 4834 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 4835 pci_set_master(pdev); 4836 4837 ixgbevf_reset(adapter); 4838 4839 return PCI_ERS_RESULT_RECOVERED; 4840 } 4841 4842 /** 4843 * ixgbevf_io_resume - called when traffic can start flowing again. 4844 * @pdev: Pointer to PCI device 4845 * 4846 * This callback is called when the error recovery driver tells us that 4847 * its OK to resume normal operation. Implementation resembles the 4848 * second-half of the ixgbevf_resume routine. 4849 **/ 4850 static void ixgbevf_io_resume(struct pci_dev *pdev) 4851 { 4852 struct net_device *netdev = pci_get_drvdata(pdev); 4853 4854 rtnl_lock(); 4855 if (netif_running(netdev)) 4856 ixgbevf_open(netdev); 4857 4858 netif_device_attach(netdev); 4859 rtnl_unlock(); 4860 } 4861 4862 /* PCI Error Recovery (ERS) */ 4863 static const struct pci_error_handlers ixgbevf_err_handler = { 4864 .error_detected = ixgbevf_io_error_detected, 4865 .slot_reset = ixgbevf_io_slot_reset, 4866 .resume = ixgbevf_io_resume, 4867 }; 4868 4869 static struct pci_driver ixgbevf_driver = { 4870 .name = ixgbevf_driver_name, 4871 .id_table = ixgbevf_pci_tbl, 4872 .probe = ixgbevf_probe, 4873 .remove = ixgbevf_remove, 4874 #ifdef CONFIG_PM 4875 /* Power Management Hooks */ 4876 .suspend = ixgbevf_suspend, 4877 .resume = ixgbevf_resume, 4878 #endif 4879 .shutdown = ixgbevf_shutdown, 4880 .err_handler = &ixgbevf_err_handler 4881 }; 4882 4883 /** 4884 * ixgbevf_init_module - Driver Registration Routine 4885 * 4886 * ixgbevf_init_module is the first routine called when the driver is 4887 * loaded. All it does is register with the PCI subsystem. 4888 **/ 4889 static int __init ixgbevf_init_module(void) 4890 { 4891 pr_info("%s - version %s\n", ixgbevf_driver_string, 4892 ixgbevf_driver_version); 4893 4894 pr_info("%s\n", ixgbevf_copyright); 4895 ixgbevf_wq = create_singlethread_workqueue(ixgbevf_driver_name); 4896 if (!ixgbevf_wq) { 4897 pr_err("%s: Failed to create workqueue\n", ixgbevf_driver_name); 4898 return -ENOMEM; 4899 } 4900 4901 return pci_register_driver(&ixgbevf_driver); 4902 } 4903 4904 module_init(ixgbevf_init_module); 4905 4906 /** 4907 * ixgbevf_exit_module - Driver Exit Cleanup Routine 4908 * 4909 * ixgbevf_exit_module is called just before the driver is removed 4910 * from memory. 4911 **/ 4912 static void __exit ixgbevf_exit_module(void) 4913 { 4914 pci_unregister_driver(&ixgbevf_driver); 4915 if (ixgbevf_wq) { 4916 destroy_workqueue(ixgbevf_wq); 4917 ixgbevf_wq = NULL; 4918 } 4919 } 4920 4921 #ifdef DEBUG 4922 /** 4923 * ixgbevf_get_hw_dev_name - return device name string 4924 * used by hardware layer to print debugging information 4925 * @hw: pointer to private hardware struct 4926 **/ 4927 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 4928 { 4929 struct ixgbevf_adapter *adapter = hw->back; 4930 4931 return adapter->netdev->name; 4932 } 4933 4934 #endif 4935 module_exit(ixgbevf_exit_module); 4936 4937 /* ixgbevf_main.c */ 4938