1 /******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2015 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, see <http://www.gnu.org/licenses/>. 17 18 The full GNU General Public License is included in this distribution in 19 the file called "COPYING". 20 21 Contact Information: 22 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 25 *******************************************************************************/ 26 27 /****************************************************************************** 28 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 29 ******************************************************************************/ 30 31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 32 33 #include <linux/types.h> 34 #include <linux/bitops.h> 35 #include <linux/module.h> 36 #include <linux/pci.h> 37 #include <linux/netdevice.h> 38 #include <linux/vmalloc.h> 39 #include <linux/string.h> 40 #include <linux/in.h> 41 #include <linux/ip.h> 42 #include <linux/tcp.h> 43 #include <linux/sctp.h> 44 #include <linux/ipv6.h> 45 #include <linux/slab.h> 46 #include <net/checksum.h> 47 #include <net/ip6_checksum.h> 48 #include <linux/ethtool.h> 49 #include <linux/if.h> 50 #include <linux/if_vlan.h> 51 #include <linux/prefetch.h> 52 53 #include "ixgbevf.h" 54 55 const char ixgbevf_driver_name[] = "ixgbevf"; 56 static const char ixgbevf_driver_string[] = 57 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 58 59 #define DRV_VERSION "2.12.1-k" 60 const char ixgbevf_driver_version[] = DRV_VERSION; 61 static char ixgbevf_copyright[] = 62 "Copyright (c) 2009 - 2012 Intel Corporation."; 63 64 static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 65 [board_82599_vf] = &ixgbevf_82599_vf_info, 66 [board_X540_vf] = &ixgbevf_X540_vf_info, 67 [board_X550_vf] = &ixgbevf_X550_vf_info, 68 [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info, 69 }; 70 71 /* ixgbevf_pci_tbl - PCI Device ID Table 72 * 73 * Wildcard entries (PCI_ANY_ID) should come last 74 * Last entry must be all 0s 75 * 76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 77 * Class, Class Mask, private data (not used) } 78 */ 79 static const struct pci_device_id ixgbevf_pci_tbl[] = { 80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf }, 81 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf }, 82 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf }, 83 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf }, 84 /* required last entry */ 85 {0, } 86 }; 87 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 88 89 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 90 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); 91 MODULE_LICENSE("GPL"); 92 MODULE_VERSION(DRV_VERSION); 93 94 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 95 static int debug = -1; 96 module_param(debug, int, 0); 97 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 98 99 static void ixgbevf_service_event_schedule(struct ixgbevf_adapter *adapter) 100 { 101 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 102 !test_bit(__IXGBEVF_REMOVING, &adapter->state) && 103 !test_and_set_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)) 104 schedule_work(&adapter->service_task); 105 } 106 107 static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter) 108 { 109 BUG_ON(!test_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)); 110 111 /* flush memory to make sure state is correct before next watchdog */ 112 smp_mb__before_atomic(); 113 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 114 } 115 116 /* forward decls */ 117 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); 118 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); 119 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); 120 121 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) 122 { 123 struct ixgbevf_adapter *adapter = hw->back; 124 125 if (!hw->hw_addr) 126 return; 127 hw->hw_addr = NULL; 128 dev_err(&adapter->pdev->dev, "Adapter removed\n"); 129 if (test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 130 ixgbevf_service_event_schedule(adapter); 131 } 132 133 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg) 134 { 135 u32 value; 136 137 /* The following check not only optimizes a bit by not 138 * performing a read on the status register when the 139 * register just read was a status register read that 140 * returned IXGBE_FAILED_READ_REG. It also blocks any 141 * potential recursion. 142 */ 143 if (reg == IXGBE_VFSTATUS) { 144 ixgbevf_remove_adapter(hw); 145 return; 146 } 147 value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS); 148 if (value == IXGBE_FAILED_READ_REG) 149 ixgbevf_remove_adapter(hw); 150 } 151 152 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg) 153 { 154 u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr); 155 u32 value; 156 157 if (IXGBE_REMOVED(reg_addr)) 158 return IXGBE_FAILED_READ_REG; 159 value = readl(reg_addr + reg); 160 if (unlikely(value == IXGBE_FAILED_READ_REG)) 161 ixgbevf_check_remove(hw, reg); 162 return value; 163 } 164 165 /** 166 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 167 * @adapter: pointer to adapter struct 168 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 169 * @queue: queue to map the corresponding interrupt to 170 * @msix_vector: the vector to map to the corresponding queue 171 **/ 172 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 173 u8 queue, u8 msix_vector) 174 { 175 u32 ivar, index; 176 struct ixgbe_hw *hw = &adapter->hw; 177 178 if (direction == -1) { 179 /* other causes */ 180 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 181 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 182 ivar &= ~0xFF; 183 ivar |= msix_vector; 184 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 185 } else { 186 /* Tx or Rx causes */ 187 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 188 index = ((16 * (queue & 1)) + (8 * direction)); 189 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 190 ivar &= ~(0xFF << index); 191 ivar |= (msix_vector << index); 192 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 193 } 194 } 195 196 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring, 197 struct ixgbevf_tx_buffer *tx_buffer) 198 { 199 if (tx_buffer->skb) { 200 dev_kfree_skb_any(tx_buffer->skb); 201 if (dma_unmap_len(tx_buffer, len)) 202 dma_unmap_single(tx_ring->dev, 203 dma_unmap_addr(tx_buffer, dma), 204 dma_unmap_len(tx_buffer, len), 205 DMA_TO_DEVICE); 206 } else if (dma_unmap_len(tx_buffer, len)) { 207 dma_unmap_page(tx_ring->dev, 208 dma_unmap_addr(tx_buffer, dma), 209 dma_unmap_len(tx_buffer, len), 210 DMA_TO_DEVICE); 211 } 212 tx_buffer->next_to_watch = NULL; 213 tx_buffer->skb = NULL; 214 dma_unmap_len_set(tx_buffer, len, 0); 215 /* tx_buffer must be completely set up in the transmit path */ 216 } 217 218 static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring) 219 { 220 return ring->stats.packets; 221 } 222 223 static u32 ixgbevf_get_tx_pending(struct ixgbevf_ring *ring) 224 { 225 struct ixgbevf_adapter *adapter = netdev_priv(ring->netdev); 226 struct ixgbe_hw *hw = &adapter->hw; 227 228 u32 head = IXGBE_READ_REG(hw, IXGBE_VFTDH(ring->reg_idx)); 229 u32 tail = IXGBE_READ_REG(hw, IXGBE_VFTDT(ring->reg_idx)); 230 231 if (head != tail) 232 return (head < tail) ? 233 tail - head : (tail + ring->count - head); 234 235 return 0; 236 } 237 238 static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring) 239 { 240 u32 tx_done = ixgbevf_get_tx_completed(tx_ring); 241 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 242 u32 tx_pending = ixgbevf_get_tx_pending(tx_ring); 243 244 clear_check_for_tx_hang(tx_ring); 245 246 /* Check for a hung queue, but be thorough. This verifies 247 * that a transmit has been completed since the previous 248 * check AND there is at least one packet pending. The 249 * ARMED bit is set to indicate a potential hang. 250 */ 251 if ((tx_done_old == tx_done) && tx_pending) { 252 /* make sure it is true for two checks in a row */ 253 return test_and_set_bit(__IXGBEVF_HANG_CHECK_ARMED, 254 &tx_ring->state); 255 } 256 /* reset the countdown */ 257 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &tx_ring->state); 258 259 /* update completed stats and continue */ 260 tx_ring->tx_stats.tx_done_old = tx_done; 261 262 return false; 263 } 264 265 static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter) 266 { 267 /* Do the reset outside of interrupt context */ 268 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 269 adapter->flags |= IXGBEVF_FLAG_RESET_REQUESTED; 270 ixgbevf_service_event_schedule(adapter); 271 } 272 } 273 274 /** 275 * ixgbevf_tx_timeout - Respond to a Tx Hang 276 * @netdev: network interface device structure 277 **/ 278 static void ixgbevf_tx_timeout(struct net_device *netdev) 279 { 280 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 281 282 ixgbevf_tx_timeout_reset(adapter); 283 } 284 285 /** 286 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 287 * @q_vector: board private structure 288 * @tx_ring: tx ring to clean 289 **/ 290 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, 291 struct ixgbevf_ring *tx_ring) 292 { 293 struct ixgbevf_adapter *adapter = q_vector->adapter; 294 struct ixgbevf_tx_buffer *tx_buffer; 295 union ixgbe_adv_tx_desc *tx_desc; 296 unsigned int total_bytes = 0, total_packets = 0; 297 unsigned int budget = tx_ring->count / 2; 298 unsigned int i = tx_ring->next_to_clean; 299 300 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 301 return true; 302 303 tx_buffer = &tx_ring->tx_buffer_info[i]; 304 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 305 i -= tx_ring->count; 306 307 do { 308 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 309 310 /* if next_to_watch is not set then there is no work pending */ 311 if (!eop_desc) 312 break; 313 314 /* prevent any other reads prior to eop_desc */ 315 read_barrier_depends(); 316 317 /* if DD is not set pending work has not been completed */ 318 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 319 break; 320 321 /* clear next_to_watch to prevent false hangs */ 322 tx_buffer->next_to_watch = NULL; 323 324 /* update the statistics for this packet */ 325 total_bytes += tx_buffer->bytecount; 326 total_packets += tx_buffer->gso_segs; 327 328 /* free the skb */ 329 dev_kfree_skb_any(tx_buffer->skb); 330 331 /* unmap skb header data */ 332 dma_unmap_single(tx_ring->dev, 333 dma_unmap_addr(tx_buffer, dma), 334 dma_unmap_len(tx_buffer, len), 335 DMA_TO_DEVICE); 336 337 /* clear tx_buffer data */ 338 tx_buffer->skb = NULL; 339 dma_unmap_len_set(tx_buffer, len, 0); 340 341 /* unmap remaining buffers */ 342 while (tx_desc != eop_desc) { 343 tx_buffer++; 344 tx_desc++; 345 i++; 346 if (unlikely(!i)) { 347 i -= tx_ring->count; 348 tx_buffer = tx_ring->tx_buffer_info; 349 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 350 } 351 352 /* unmap any remaining paged data */ 353 if (dma_unmap_len(tx_buffer, len)) { 354 dma_unmap_page(tx_ring->dev, 355 dma_unmap_addr(tx_buffer, dma), 356 dma_unmap_len(tx_buffer, len), 357 DMA_TO_DEVICE); 358 dma_unmap_len_set(tx_buffer, len, 0); 359 } 360 } 361 362 /* move us one more past the eop_desc for start of next pkt */ 363 tx_buffer++; 364 tx_desc++; 365 i++; 366 if (unlikely(!i)) { 367 i -= tx_ring->count; 368 tx_buffer = tx_ring->tx_buffer_info; 369 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 370 } 371 372 /* issue prefetch for next Tx descriptor */ 373 prefetch(tx_desc); 374 375 /* update budget accounting */ 376 budget--; 377 } while (likely(budget)); 378 379 i += tx_ring->count; 380 tx_ring->next_to_clean = i; 381 u64_stats_update_begin(&tx_ring->syncp); 382 tx_ring->stats.bytes += total_bytes; 383 tx_ring->stats.packets += total_packets; 384 u64_stats_update_end(&tx_ring->syncp); 385 q_vector->tx.total_bytes += total_bytes; 386 q_vector->tx.total_packets += total_packets; 387 388 if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) { 389 struct ixgbe_hw *hw = &adapter->hw; 390 union ixgbe_adv_tx_desc *eop_desc; 391 392 eop_desc = tx_ring->tx_buffer_info[i].next_to_watch; 393 394 pr_err("Detected Tx Unit Hang\n" 395 " Tx Queue <%d>\n" 396 " TDH, TDT <%x>, <%x>\n" 397 " next_to_use <%x>\n" 398 " next_to_clean <%x>\n" 399 "tx_buffer_info[next_to_clean]\n" 400 " next_to_watch <%p>\n" 401 " eop_desc->wb.status <%x>\n" 402 " time_stamp <%lx>\n" 403 " jiffies <%lx>\n", 404 tx_ring->queue_index, 405 IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)), 406 IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)), 407 tx_ring->next_to_use, i, 408 eop_desc, (eop_desc ? eop_desc->wb.status : 0), 409 tx_ring->tx_buffer_info[i].time_stamp, jiffies); 410 411 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 412 413 /* schedule immediate reset if we believe we hung */ 414 ixgbevf_tx_timeout_reset(adapter); 415 416 return true; 417 } 418 419 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 420 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 421 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 422 /* Make sure that anybody stopping the queue after this 423 * sees the new next_to_clean. 424 */ 425 smp_mb(); 426 427 if (__netif_subqueue_stopped(tx_ring->netdev, 428 tx_ring->queue_index) && 429 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 430 netif_wake_subqueue(tx_ring->netdev, 431 tx_ring->queue_index); 432 ++tx_ring->tx_stats.restart_queue; 433 } 434 } 435 436 return !!budget; 437 } 438 439 /** 440 * ixgbevf_rx_skb - Helper function to determine proper Rx method 441 * @q_vector: structure containing interrupt and ring information 442 * @skb: packet to send up 443 **/ 444 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, 445 struct sk_buff *skb) 446 { 447 #ifdef CONFIG_NET_RX_BUSY_POLL 448 skb_mark_napi_id(skb, &q_vector->napi); 449 450 if (ixgbevf_qv_busy_polling(q_vector)) { 451 netif_receive_skb(skb); 452 /* exit early if we busy polled */ 453 return; 454 } 455 #endif /* CONFIG_NET_RX_BUSY_POLL */ 456 457 napi_gro_receive(&q_vector->napi, skb); 458 } 459 460 /** 461 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 462 * @ring: structure containig ring specific data 463 * @rx_desc: current Rx descriptor being processed 464 * @skb: skb currently being received and modified 465 **/ 466 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, 467 union ixgbe_adv_rx_desc *rx_desc, 468 struct sk_buff *skb) 469 { 470 skb_checksum_none_assert(skb); 471 472 /* Rx csum disabled */ 473 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 474 return; 475 476 /* if IP and error */ 477 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 478 ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 479 ring->rx_stats.csum_err++; 480 return; 481 } 482 483 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 484 return; 485 486 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 487 ring->rx_stats.csum_err++; 488 return; 489 } 490 491 /* It must be a TCP or UDP packet with a valid checksum */ 492 skb->ip_summed = CHECKSUM_UNNECESSARY; 493 } 494 495 /** 496 * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor 497 * @rx_ring: rx descriptor ring packet is being transacted on 498 * @rx_desc: pointer to the EOP Rx descriptor 499 * @skb: pointer to current skb being populated 500 * 501 * This function checks the ring, descriptor, and packet information in 502 * order to populate the checksum, VLAN, protocol, and other fields within 503 * the skb. 504 **/ 505 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, 506 union ixgbe_adv_rx_desc *rx_desc, 507 struct sk_buff *skb) 508 { 509 ixgbevf_rx_checksum(rx_ring, rx_desc, skb); 510 511 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 512 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 513 unsigned long *active_vlans = netdev_priv(rx_ring->netdev); 514 515 if (test_bit(vid & VLAN_VID_MASK, active_vlans)) 516 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 517 } 518 519 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 520 } 521 522 /** 523 * ixgbevf_is_non_eop - process handling of non-EOP buffers 524 * @rx_ring: Rx ring being processed 525 * @rx_desc: Rx descriptor for current buffer 526 * @skb: current socket buffer containing buffer in progress 527 * 528 * This function updates next to clean. If the buffer is an EOP buffer 529 * this function exits returning false, otherwise it will place the 530 * sk_buff in the next buffer to be chained and return true indicating 531 * that this is in fact a non-EOP buffer. 532 **/ 533 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, 534 union ixgbe_adv_rx_desc *rx_desc) 535 { 536 u32 ntc = rx_ring->next_to_clean + 1; 537 538 /* fetch, update, and store next to clean */ 539 ntc = (ntc < rx_ring->count) ? ntc : 0; 540 rx_ring->next_to_clean = ntc; 541 542 prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); 543 544 if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 545 return false; 546 547 return true; 548 } 549 550 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, 551 struct ixgbevf_rx_buffer *bi) 552 { 553 struct page *page = bi->page; 554 dma_addr_t dma = bi->dma; 555 556 /* since we are recycling buffers we should seldom need to alloc */ 557 if (likely(page)) 558 return true; 559 560 /* alloc new page for storage */ 561 page = dev_alloc_page(); 562 if (unlikely(!page)) { 563 rx_ring->rx_stats.alloc_rx_page_failed++; 564 return false; 565 } 566 567 /* map page for use */ 568 dma = dma_map_page(rx_ring->dev, page, 0, 569 PAGE_SIZE, DMA_FROM_DEVICE); 570 571 /* if mapping failed free memory back to system since 572 * there isn't much point in holding memory we can't use 573 */ 574 if (dma_mapping_error(rx_ring->dev, dma)) { 575 __free_page(page); 576 577 rx_ring->rx_stats.alloc_rx_buff_failed++; 578 return false; 579 } 580 581 bi->dma = dma; 582 bi->page = page; 583 bi->page_offset = 0; 584 585 return true; 586 } 587 588 /** 589 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 590 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on 591 * @cleaned_count: number of buffers to replace 592 **/ 593 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, 594 u16 cleaned_count) 595 { 596 union ixgbe_adv_rx_desc *rx_desc; 597 struct ixgbevf_rx_buffer *bi; 598 unsigned int i = rx_ring->next_to_use; 599 600 /* nothing to do or no valid netdev defined */ 601 if (!cleaned_count || !rx_ring->netdev) 602 return; 603 604 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 605 bi = &rx_ring->rx_buffer_info[i]; 606 i -= rx_ring->count; 607 608 do { 609 if (!ixgbevf_alloc_mapped_page(rx_ring, bi)) 610 break; 611 612 /* Refresh the desc even if pkt_addr didn't change 613 * because each write-back erases this info. 614 */ 615 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 616 617 rx_desc++; 618 bi++; 619 i++; 620 if (unlikely(!i)) { 621 rx_desc = IXGBEVF_RX_DESC(rx_ring, 0); 622 bi = rx_ring->rx_buffer_info; 623 i -= rx_ring->count; 624 } 625 626 /* clear the hdr_addr for the next_to_use descriptor */ 627 rx_desc->read.hdr_addr = 0; 628 629 cleaned_count--; 630 } while (cleaned_count); 631 632 i += rx_ring->count; 633 634 if (rx_ring->next_to_use != i) { 635 /* record the next descriptor to use */ 636 rx_ring->next_to_use = i; 637 638 /* update next to alloc since we have filled the ring */ 639 rx_ring->next_to_alloc = i; 640 641 /* Force memory writes to complete before letting h/w 642 * know there are new descriptors to fetch. (Only 643 * applicable for weak-ordered memory model archs, 644 * such as IA-64). 645 */ 646 wmb(); 647 ixgbevf_write_tail(rx_ring, i); 648 } 649 } 650 651 /** 652 * ixgbevf_pull_tail - ixgbevf specific version of skb_pull_tail 653 * @rx_ring: rx descriptor ring packet is being transacted on 654 * @skb: pointer to current skb being adjusted 655 * 656 * This function is an ixgbevf specific version of __pskb_pull_tail. The 657 * main difference between this version and the original function is that 658 * this function can make several assumptions about the state of things 659 * that allow for significant optimizations versus the standard function. 660 * As a result we can do things like drop a frag and maintain an accurate 661 * truesize for the skb. 662 **/ 663 static void ixgbevf_pull_tail(struct ixgbevf_ring *rx_ring, 664 struct sk_buff *skb) 665 { 666 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 667 unsigned char *va; 668 unsigned int pull_len; 669 670 /* it is valid to use page_address instead of kmap since we are 671 * working with pages allocated out of the lomem pool per 672 * alloc_page(GFP_ATOMIC) 673 */ 674 va = skb_frag_address(frag); 675 676 /* we need the header to contain the greater of either ETH_HLEN or 677 * 60 bytes if the skb->len is less than 60 for skb_pad. 678 */ 679 pull_len = eth_get_headlen(va, IXGBEVF_RX_HDR_SIZE); 680 681 /* align pull length to size of long to optimize memcpy performance */ 682 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); 683 684 /* update all of the pointers */ 685 skb_frag_size_sub(frag, pull_len); 686 frag->page_offset += pull_len; 687 skb->data_len -= pull_len; 688 skb->tail += pull_len; 689 } 690 691 /** 692 * ixgbevf_cleanup_headers - Correct corrupted or empty headers 693 * @rx_ring: rx descriptor ring packet is being transacted on 694 * @rx_desc: pointer to the EOP Rx descriptor 695 * @skb: pointer to current skb being fixed 696 * 697 * Check for corrupted packet headers caused by senders on the local L2 698 * embedded NIC switch not setting up their Tx Descriptors right. These 699 * should be very rare. 700 * 701 * Also address the case where we are pulling data in on pages only 702 * and as such no data is present in the skb header. 703 * 704 * In addition if skb is not at least 60 bytes we need to pad it so that 705 * it is large enough to qualify as a valid Ethernet frame. 706 * 707 * Returns true if an error was encountered and skb was freed. 708 **/ 709 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, 710 union ixgbe_adv_rx_desc *rx_desc, 711 struct sk_buff *skb) 712 { 713 /* verify that the packet does not have any known errors */ 714 if (unlikely(ixgbevf_test_staterr(rx_desc, 715 IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { 716 struct net_device *netdev = rx_ring->netdev; 717 718 if (!(netdev->features & NETIF_F_RXALL)) { 719 dev_kfree_skb_any(skb); 720 return true; 721 } 722 } 723 724 /* place header in linear portion of buffer */ 725 if (skb_is_nonlinear(skb)) 726 ixgbevf_pull_tail(rx_ring, skb); 727 728 /* if eth_skb_pad returns an error the skb was freed */ 729 if (eth_skb_pad(skb)) 730 return true; 731 732 return false; 733 } 734 735 /** 736 * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring 737 * @rx_ring: rx descriptor ring to store buffers on 738 * @old_buff: donor buffer to have page reused 739 * 740 * Synchronizes page for reuse by the adapter 741 **/ 742 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, 743 struct ixgbevf_rx_buffer *old_buff) 744 { 745 struct ixgbevf_rx_buffer *new_buff; 746 u16 nta = rx_ring->next_to_alloc; 747 748 new_buff = &rx_ring->rx_buffer_info[nta]; 749 750 /* update, and store next to alloc */ 751 nta++; 752 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 753 754 /* transfer page from old buffer to new buffer */ 755 new_buff->page = old_buff->page; 756 new_buff->dma = old_buff->dma; 757 new_buff->page_offset = old_buff->page_offset; 758 759 /* sync the buffer for use by the device */ 760 dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma, 761 new_buff->page_offset, 762 IXGBEVF_RX_BUFSZ, 763 DMA_FROM_DEVICE); 764 } 765 766 static inline bool ixgbevf_page_is_reserved(struct page *page) 767 { 768 return (page_to_nid(page) != numa_mem_id()) || page->pfmemalloc; 769 } 770 771 /** 772 * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff 773 * @rx_ring: rx descriptor ring to transact packets on 774 * @rx_buffer: buffer containing page to add 775 * @rx_desc: descriptor containing length of buffer written by hardware 776 * @skb: sk_buff to place the data into 777 * 778 * This function will add the data contained in rx_buffer->page to the skb. 779 * This is done either through a direct copy if the data in the buffer is 780 * less than the skb header size, otherwise it will just attach the page as 781 * a frag to the skb. 782 * 783 * The function will then update the page offset if necessary and return 784 * true if the buffer can be reused by the adapter. 785 **/ 786 static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, 787 struct ixgbevf_rx_buffer *rx_buffer, 788 union ixgbe_adv_rx_desc *rx_desc, 789 struct sk_buff *skb) 790 { 791 struct page *page = rx_buffer->page; 792 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length); 793 #if (PAGE_SIZE < 8192) 794 unsigned int truesize = IXGBEVF_RX_BUFSZ; 795 #else 796 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES); 797 #endif 798 799 if ((size <= IXGBEVF_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) { 800 unsigned char *va = page_address(page) + rx_buffer->page_offset; 801 802 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 803 804 /* page is not reserved, we can reuse buffer as is */ 805 if (likely(!ixgbevf_page_is_reserved(page))) 806 return true; 807 808 /* this page cannot be reused so discard it */ 809 put_page(page); 810 return false; 811 } 812 813 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 814 rx_buffer->page_offset, size, truesize); 815 816 /* avoid re-using remote pages */ 817 if (unlikely(ixgbevf_page_is_reserved(page))) 818 return false; 819 820 #if (PAGE_SIZE < 8192) 821 /* if we are only owner of page we can reuse it */ 822 if (unlikely(page_count(page) != 1)) 823 return false; 824 825 /* flip page offset to other buffer */ 826 rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ; 827 828 #else 829 /* move offset up to the next cache line */ 830 rx_buffer->page_offset += truesize; 831 832 if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ)) 833 return false; 834 835 #endif 836 /* Even if we own the page, we are not allowed to use atomic_set() 837 * This would break get_page_unless_zero() users. 838 */ 839 atomic_inc(&page->_count); 840 841 return true; 842 } 843 844 static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring, 845 union ixgbe_adv_rx_desc *rx_desc, 846 struct sk_buff *skb) 847 { 848 struct ixgbevf_rx_buffer *rx_buffer; 849 struct page *page; 850 851 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 852 page = rx_buffer->page; 853 prefetchw(page); 854 855 if (likely(!skb)) { 856 void *page_addr = page_address(page) + 857 rx_buffer->page_offset; 858 859 /* prefetch first cache line of first page */ 860 prefetch(page_addr); 861 #if L1_CACHE_BYTES < 128 862 prefetch(page_addr + L1_CACHE_BYTES); 863 #endif 864 865 /* allocate a skb to store the frags */ 866 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 867 IXGBEVF_RX_HDR_SIZE); 868 if (unlikely(!skb)) { 869 rx_ring->rx_stats.alloc_rx_buff_failed++; 870 return NULL; 871 } 872 873 /* we will be copying header into skb->data in 874 * pskb_may_pull so it is in our interest to prefetch 875 * it now to avoid a possible cache miss 876 */ 877 prefetchw(skb->data); 878 } 879 880 /* we are reusing so sync this buffer for CPU use */ 881 dma_sync_single_range_for_cpu(rx_ring->dev, 882 rx_buffer->dma, 883 rx_buffer->page_offset, 884 IXGBEVF_RX_BUFSZ, 885 DMA_FROM_DEVICE); 886 887 /* pull page into skb */ 888 if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) { 889 /* hand second half of page back to the ring */ 890 ixgbevf_reuse_rx_page(rx_ring, rx_buffer); 891 } else { 892 /* we are not reusing the buffer so unmap it */ 893 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 894 PAGE_SIZE, DMA_FROM_DEVICE); 895 } 896 897 /* clear contents of buffer_info */ 898 rx_buffer->dma = 0; 899 rx_buffer->page = NULL; 900 901 return skb; 902 } 903 904 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 905 u32 qmask) 906 { 907 struct ixgbe_hw *hw = &adapter->hw; 908 909 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); 910 } 911 912 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 913 struct ixgbevf_ring *rx_ring, 914 int budget) 915 { 916 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 917 u16 cleaned_count = ixgbevf_desc_unused(rx_ring); 918 struct sk_buff *skb = rx_ring->skb; 919 920 while (likely(total_rx_packets < budget)) { 921 union ixgbe_adv_rx_desc *rx_desc; 922 923 /* return some buffers to hardware, one at a time is too slow */ 924 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 925 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count); 926 cleaned_count = 0; 927 } 928 929 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); 930 931 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD)) 932 break; 933 934 /* This memory barrier is needed to keep us from reading 935 * any other fields out of the rx_desc until we know the 936 * RXD_STAT_DD bit is set 937 */ 938 rmb(); 939 940 /* retrieve a buffer from the ring */ 941 skb = ixgbevf_fetch_rx_buffer(rx_ring, rx_desc, skb); 942 943 /* exit if we failed to retrieve a buffer */ 944 if (!skb) 945 break; 946 947 cleaned_count++; 948 949 /* fetch next buffer in frame if non-eop */ 950 if (ixgbevf_is_non_eop(rx_ring, rx_desc)) 951 continue; 952 953 /* verify the packet layout is correct */ 954 if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) { 955 skb = NULL; 956 continue; 957 } 958 959 /* probably a little skewed due to removing CRC */ 960 total_rx_bytes += skb->len; 961 962 /* Workaround hardware that can't do proper VEPA multicast 963 * source pruning. 964 */ 965 if ((skb->pkt_type == PACKET_BROADCAST || 966 skb->pkt_type == PACKET_MULTICAST) && 967 ether_addr_equal(rx_ring->netdev->dev_addr, 968 eth_hdr(skb)->h_source)) { 969 dev_kfree_skb_irq(skb); 970 continue; 971 } 972 973 /* populate checksum, VLAN, and protocol */ 974 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); 975 976 ixgbevf_rx_skb(q_vector, skb); 977 978 /* reset skb pointer */ 979 skb = NULL; 980 981 /* update budget accounting */ 982 total_rx_packets++; 983 } 984 985 /* place incomplete frames back on ring for completion */ 986 rx_ring->skb = skb; 987 988 u64_stats_update_begin(&rx_ring->syncp); 989 rx_ring->stats.packets += total_rx_packets; 990 rx_ring->stats.bytes += total_rx_bytes; 991 u64_stats_update_end(&rx_ring->syncp); 992 q_vector->rx.total_packets += total_rx_packets; 993 q_vector->rx.total_bytes += total_rx_bytes; 994 995 return total_rx_packets; 996 } 997 998 /** 999 * ixgbevf_poll - NAPI polling calback 1000 * @napi: napi struct with our devices info in it 1001 * @budget: amount of work driver is allowed to do this pass, in packets 1002 * 1003 * This function will clean more than one or more rings associated with a 1004 * q_vector. 1005 **/ 1006 static int ixgbevf_poll(struct napi_struct *napi, int budget) 1007 { 1008 struct ixgbevf_q_vector *q_vector = 1009 container_of(napi, struct ixgbevf_q_vector, napi); 1010 struct ixgbevf_adapter *adapter = q_vector->adapter; 1011 struct ixgbevf_ring *ring; 1012 int per_ring_budget; 1013 bool clean_complete = true; 1014 1015 ixgbevf_for_each_ring(ring, q_vector->tx) 1016 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring); 1017 1018 #ifdef CONFIG_NET_RX_BUSY_POLL 1019 if (!ixgbevf_qv_lock_napi(q_vector)) 1020 return budget; 1021 #endif 1022 1023 /* attempt to distribute budget to each queue fairly, but don't allow 1024 * the budget to go below 1 because we'll exit polling 1025 */ 1026 if (q_vector->rx.count > 1) 1027 per_ring_budget = max(budget/q_vector->rx.count, 1); 1028 else 1029 per_ring_budget = budget; 1030 1031 ixgbevf_for_each_ring(ring, q_vector->rx) 1032 clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring, 1033 per_ring_budget) 1034 < per_ring_budget); 1035 1036 #ifdef CONFIG_NET_RX_BUSY_POLL 1037 ixgbevf_qv_unlock_napi(q_vector); 1038 #endif 1039 1040 /* If all work not completed, return budget and keep polling */ 1041 if (!clean_complete) 1042 return budget; 1043 /* all work done, exit the polling mode */ 1044 napi_complete(napi); 1045 if (adapter->rx_itr_setting & 1) 1046 ixgbevf_set_itr(q_vector); 1047 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 1048 !test_bit(__IXGBEVF_REMOVING, &adapter->state)) 1049 ixgbevf_irq_enable_queues(adapter, 1050 1 << q_vector->v_idx); 1051 1052 return 0; 1053 } 1054 1055 /** 1056 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 1057 * @q_vector: structure containing interrupt and ring information 1058 **/ 1059 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) 1060 { 1061 struct ixgbevf_adapter *adapter = q_vector->adapter; 1062 struct ixgbe_hw *hw = &adapter->hw; 1063 int v_idx = q_vector->v_idx; 1064 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 1065 1066 /* set the WDIS bit to not clear the timer bits and cause an 1067 * immediate assertion of the interrupt 1068 */ 1069 itr_reg |= IXGBE_EITR_CNT_WDIS; 1070 1071 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 1072 } 1073 1074 #ifdef CONFIG_NET_RX_BUSY_POLL 1075 /* must be called with local_bh_disable()d */ 1076 static int ixgbevf_busy_poll_recv(struct napi_struct *napi) 1077 { 1078 struct ixgbevf_q_vector *q_vector = 1079 container_of(napi, struct ixgbevf_q_vector, napi); 1080 struct ixgbevf_adapter *adapter = q_vector->adapter; 1081 struct ixgbevf_ring *ring; 1082 int found = 0; 1083 1084 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 1085 return LL_FLUSH_FAILED; 1086 1087 if (!ixgbevf_qv_lock_poll(q_vector)) 1088 return LL_FLUSH_BUSY; 1089 1090 ixgbevf_for_each_ring(ring, q_vector->rx) { 1091 found = ixgbevf_clean_rx_irq(q_vector, ring, 4); 1092 #ifdef BP_EXTENDED_STATS 1093 if (found) 1094 ring->stats.cleaned += found; 1095 else 1096 ring->stats.misses++; 1097 #endif 1098 if (found) 1099 break; 1100 } 1101 1102 ixgbevf_qv_unlock_poll(q_vector); 1103 1104 return found; 1105 } 1106 #endif /* CONFIG_NET_RX_BUSY_POLL */ 1107 1108 /** 1109 * ixgbevf_configure_msix - Configure MSI-X hardware 1110 * @adapter: board private structure 1111 * 1112 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 1113 * interrupts. 1114 **/ 1115 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 1116 { 1117 struct ixgbevf_q_vector *q_vector; 1118 int q_vectors, v_idx; 1119 1120 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1121 adapter->eims_enable_mask = 0; 1122 1123 /* Populate the IVAR table and set the ITR values to the 1124 * corresponding register. 1125 */ 1126 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 1127 struct ixgbevf_ring *ring; 1128 1129 q_vector = adapter->q_vector[v_idx]; 1130 1131 ixgbevf_for_each_ring(ring, q_vector->rx) 1132 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx); 1133 1134 ixgbevf_for_each_ring(ring, q_vector->tx) 1135 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx); 1136 1137 if (q_vector->tx.ring && !q_vector->rx.ring) { 1138 /* Tx only vector */ 1139 if (adapter->tx_itr_setting == 1) 1140 q_vector->itr = IXGBE_10K_ITR; 1141 else 1142 q_vector->itr = adapter->tx_itr_setting; 1143 } else { 1144 /* Rx or Rx/Tx vector */ 1145 if (adapter->rx_itr_setting == 1) 1146 q_vector->itr = IXGBE_20K_ITR; 1147 else 1148 q_vector->itr = adapter->rx_itr_setting; 1149 } 1150 1151 /* add q_vector eims value to global eims_enable_mask */ 1152 adapter->eims_enable_mask |= 1 << v_idx; 1153 1154 ixgbevf_write_eitr(q_vector); 1155 } 1156 1157 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 1158 /* setup eims_other and add value to global eims_enable_mask */ 1159 adapter->eims_other = 1 << v_idx; 1160 adapter->eims_enable_mask |= adapter->eims_other; 1161 } 1162 1163 enum latency_range { 1164 lowest_latency = 0, 1165 low_latency = 1, 1166 bulk_latency = 2, 1167 latency_invalid = 255 1168 }; 1169 1170 /** 1171 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 1172 * @q_vector: structure containing interrupt and ring information 1173 * @ring_container: structure containing ring performance data 1174 * 1175 * Stores a new ITR value based on packets and byte 1176 * counts during the last interrupt. The advantage of per interrupt 1177 * computation is faster updates and more accurate ITR for the current 1178 * traffic pattern. Constants in this function were computed 1179 * based on theoretical maximum wire speed and thresholds were set based 1180 * on testing data as well as attempting to minimize response time 1181 * while increasing bulk throughput. 1182 **/ 1183 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector, 1184 struct ixgbevf_ring_container *ring_container) 1185 { 1186 int bytes = ring_container->total_bytes; 1187 int packets = ring_container->total_packets; 1188 u32 timepassed_us; 1189 u64 bytes_perint; 1190 u8 itr_setting = ring_container->itr; 1191 1192 if (packets == 0) 1193 return; 1194 1195 /* simple throttle rate management 1196 * 0-20MB/s lowest (100000 ints/s) 1197 * 20-100MB/s low (20000 ints/s) 1198 * 100-1249MB/s bulk (8000 ints/s) 1199 */ 1200 /* what was last interrupt timeslice? */ 1201 timepassed_us = q_vector->itr >> 2; 1202 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 1203 1204 switch (itr_setting) { 1205 case lowest_latency: 1206 if (bytes_perint > 10) 1207 itr_setting = low_latency; 1208 break; 1209 case low_latency: 1210 if (bytes_perint > 20) 1211 itr_setting = bulk_latency; 1212 else if (bytes_perint <= 10) 1213 itr_setting = lowest_latency; 1214 break; 1215 case bulk_latency: 1216 if (bytes_perint <= 20) 1217 itr_setting = low_latency; 1218 break; 1219 } 1220 1221 /* clear work counters since we have the values we need */ 1222 ring_container->total_bytes = 0; 1223 ring_container->total_packets = 0; 1224 1225 /* write updated itr to ring container */ 1226 ring_container->itr = itr_setting; 1227 } 1228 1229 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector) 1230 { 1231 u32 new_itr = q_vector->itr; 1232 u8 current_itr; 1233 1234 ixgbevf_update_itr(q_vector, &q_vector->tx); 1235 ixgbevf_update_itr(q_vector, &q_vector->rx); 1236 1237 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 1238 1239 switch (current_itr) { 1240 /* counts and packets in update_itr are dependent on these numbers */ 1241 case lowest_latency: 1242 new_itr = IXGBE_100K_ITR; 1243 break; 1244 case low_latency: 1245 new_itr = IXGBE_20K_ITR; 1246 break; 1247 case bulk_latency: 1248 default: 1249 new_itr = IXGBE_8K_ITR; 1250 break; 1251 } 1252 1253 if (new_itr != q_vector->itr) { 1254 /* do an exponential smoothing */ 1255 new_itr = (10 * new_itr * q_vector->itr) / 1256 ((9 * new_itr) + q_vector->itr); 1257 1258 /* save the algorithm value here */ 1259 q_vector->itr = new_itr; 1260 1261 ixgbevf_write_eitr(q_vector); 1262 } 1263 } 1264 1265 static irqreturn_t ixgbevf_msix_other(int irq, void *data) 1266 { 1267 struct ixgbevf_adapter *adapter = data; 1268 struct ixgbe_hw *hw = &adapter->hw; 1269 1270 hw->mac.get_link_status = 1; 1271 1272 ixgbevf_service_event_schedule(adapter); 1273 1274 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); 1275 1276 return IRQ_HANDLED; 1277 } 1278 1279 /** 1280 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) 1281 * @irq: unused 1282 * @data: pointer to our q_vector struct for this interrupt vector 1283 **/ 1284 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data) 1285 { 1286 struct ixgbevf_q_vector *q_vector = data; 1287 1288 /* EIAM disabled interrupts (on this vector) for us */ 1289 if (q_vector->rx.ring || q_vector->tx.ring) 1290 napi_schedule(&q_vector->napi); 1291 1292 return IRQ_HANDLED; 1293 } 1294 1295 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, 1296 int r_idx) 1297 { 1298 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1299 1300 a->rx_ring[r_idx]->next = q_vector->rx.ring; 1301 q_vector->rx.ring = a->rx_ring[r_idx]; 1302 q_vector->rx.count++; 1303 } 1304 1305 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, 1306 int t_idx) 1307 { 1308 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1309 1310 a->tx_ring[t_idx]->next = q_vector->tx.ring; 1311 q_vector->tx.ring = a->tx_ring[t_idx]; 1312 q_vector->tx.count++; 1313 } 1314 1315 /** 1316 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors 1317 * @adapter: board private structure to initialize 1318 * 1319 * This function maps descriptor rings to the queue-specific vectors 1320 * we were allotted through the MSI-X enabling code. Ideally, we'd have 1321 * one vector per ring/queue, but on a constrained vector budget, we 1322 * group the rings as "efficiently" as possible. You would add new 1323 * mapping configurations in here. 1324 **/ 1325 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter) 1326 { 1327 int q_vectors; 1328 int v_start = 0; 1329 int rxr_idx = 0, txr_idx = 0; 1330 int rxr_remaining = adapter->num_rx_queues; 1331 int txr_remaining = adapter->num_tx_queues; 1332 int i, j; 1333 int rqpv, tqpv; 1334 int err = 0; 1335 1336 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1337 1338 /* The ideal configuration... 1339 * We have enough vectors to map one per queue. 1340 */ 1341 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) { 1342 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) 1343 map_vector_to_rxq(adapter, v_start, rxr_idx); 1344 1345 for (; txr_idx < txr_remaining; v_start++, txr_idx++) 1346 map_vector_to_txq(adapter, v_start, txr_idx); 1347 goto out; 1348 } 1349 1350 /* If we don't have enough vectors for a 1-to-1 1351 * mapping, we'll have to group them so there are 1352 * multiple queues per vector. 1353 */ 1354 /* Re-adjusting *qpv takes care of the remainder. */ 1355 for (i = v_start; i < q_vectors; i++) { 1356 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i); 1357 for (j = 0; j < rqpv; j++) { 1358 map_vector_to_rxq(adapter, i, rxr_idx); 1359 rxr_idx++; 1360 rxr_remaining--; 1361 } 1362 } 1363 for (i = v_start; i < q_vectors; i++) { 1364 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i); 1365 for (j = 0; j < tqpv; j++) { 1366 map_vector_to_txq(adapter, i, txr_idx); 1367 txr_idx++; 1368 txr_remaining--; 1369 } 1370 } 1371 1372 out: 1373 return err; 1374 } 1375 1376 /** 1377 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 1378 * @adapter: board private structure 1379 * 1380 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 1381 * interrupts from the kernel. 1382 **/ 1383 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 1384 { 1385 struct net_device *netdev = adapter->netdev; 1386 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1387 int vector, err; 1388 int ri = 0, ti = 0; 1389 1390 for (vector = 0; vector < q_vectors; vector++) { 1391 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector]; 1392 struct msix_entry *entry = &adapter->msix_entries[vector]; 1393 1394 if (q_vector->tx.ring && q_vector->rx.ring) { 1395 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1396 "%s-%s-%d", netdev->name, "TxRx", ri++); 1397 ti++; 1398 } else if (q_vector->rx.ring) { 1399 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1400 "%s-%s-%d", netdev->name, "rx", ri++); 1401 } else if (q_vector->tx.ring) { 1402 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1403 "%s-%s-%d", netdev->name, "tx", ti++); 1404 } else { 1405 /* skip this unused q_vector */ 1406 continue; 1407 } 1408 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0, 1409 q_vector->name, q_vector); 1410 if (err) { 1411 hw_dbg(&adapter->hw, 1412 "request_irq failed for MSIX interrupt Error: %d\n", 1413 err); 1414 goto free_queue_irqs; 1415 } 1416 } 1417 1418 err = request_irq(adapter->msix_entries[vector].vector, 1419 &ixgbevf_msix_other, 0, netdev->name, adapter); 1420 if (err) { 1421 hw_dbg(&adapter->hw, "request_irq for msix_other failed: %d\n", 1422 err); 1423 goto free_queue_irqs; 1424 } 1425 1426 return 0; 1427 1428 free_queue_irqs: 1429 while (vector) { 1430 vector--; 1431 free_irq(adapter->msix_entries[vector].vector, 1432 adapter->q_vector[vector]); 1433 } 1434 /* This failure is non-recoverable - it indicates the system is 1435 * out of MSIX vector resources and the VF driver cannot run 1436 * without them. Set the number of msix vectors to zero 1437 * indicating that not enough can be allocated. The error 1438 * will be returned to the user indicating device open failed. 1439 * Any further attempts to force the driver to open will also 1440 * fail. The only way to recover is to unload the driver and 1441 * reload it again. If the system has recovered some MSIX 1442 * vectors then it may succeed. 1443 */ 1444 adapter->num_msix_vectors = 0; 1445 return err; 1446 } 1447 1448 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter) 1449 { 1450 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1451 1452 for (i = 0; i < q_vectors; i++) { 1453 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i]; 1454 1455 q_vector->rx.ring = NULL; 1456 q_vector->tx.ring = NULL; 1457 q_vector->rx.count = 0; 1458 q_vector->tx.count = 0; 1459 } 1460 } 1461 1462 /** 1463 * ixgbevf_request_irq - initialize interrupts 1464 * @adapter: board private structure 1465 * 1466 * Attempts to configure interrupts using the best available 1467 * capabilities of the hardware and kernel. 1468 **/ 1469 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 1470 { 1471 int err = 0; 1472 1473 err = ixgbevf_request_msix_irqs(adapter); 1474 1475 if (err) 1476 hw_dbg(&adapter->hw, "request_irq failed, Error %d\n", err); 1477 1478 return err; 1479 } 1480 1481 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 1482 { 1483 int i, q_vectors; 1484 1485 q_vectors = adapter->num_msix_vectors; 1486 i = q_vectors - 1; 1487 1488 free_irq(adapter->msix_entries[i].vector, adapter); 1489 i--; 1490 1491 for (; i >= 0; i--) { 1492 /* free only the irqs that were actually requested */ 1493 if (!adapter->q_vector[i]->rx.ring && 1494 !adapter->q_vector[i]->tx.ring) 1495 continue; 1496 1497 free_irq(adapter->msix_entries[i].vector, 1498 adapter->q_vector[i]); 1499 } 1500 1501 ixgbevf_reset_q_vectors(adapter); 1502 } 1503 1504 /** 1505 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1506 * @adapter: board private structure 1507 **/ 1508 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1509 { 1510 struct ixgbe_hw *hw = &adapter->hw; 1511 int i; 1512 1513 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0); 1514 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1515 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0); 1516 1517 IXGBE_WRITE_FLUSH(hw); 1518 1519 for (i = 0; i < adapter->num_msix_vectors; i++) 1520 synchronize_irq(adapter->msix_entries[i].vector); 1521 } 1522 1523 /** 1524 * ixgbevf_irq_enable - Enable default interrupt generation settings 1525 * @adapter: board private structure 1526 **/ 1527 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) 1528 { 1529 struct ixgbe_hw *hw = &adapter->hw; 1530 1531 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask); 1532 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask); 1533 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); 1534 } 1535 1536 /** 1537 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset 1538 * @adapter: board private structure 1539 * @ring: structure containing ring specific data 1540 * 1541 * Configure the Tx descriptor ring after a reset. 1542 **/ 1543 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 1544 struct ixgbevf_ring *ring) 1545 { 1546 struct ixgbe_hw *hw = &adapter->hw; 1547 u64 tdba = ring->dma; 1548 int wait_loop = 10; 1549 u32 txdctl = IXGBE_TXDCTL_ENABLE; 1550 u8 reg_idx = ring->reg_idx; 1551 1552 /* disable queue to avoid issues while updating state */ 1553 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 1554 IXGBE_WRITE_FLUSH(hw); 1555 1556 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 1557 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32); 1558 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx), 1559 ring->count * sizeof(union ixgbe_adv_tx_desc)); 1560 1561 /* disable head writeback */ 1562 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0); 1563 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0); 1564 1565 /* enable relaxed ordering */ 1566 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx), 1567 (IXGBE_DCA_TXCTRL_DESC_RRO_EN | 1568 IXGBE_DCA_TXCTRL_DATA_RRO_EN)); 1569 1570 /* reset head and tail pointers */ 1571 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0); 1572 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0); 1573 ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx); 1574 1575 /* reset ntu and ntc to place SW in sync with hardwdare */ 1576 ring->next_to_clean = 0; 1577 ring->next_to_use = 0; 1578 1579 /* In order to avoid issues WTHRESH + PTHRESH should always be equal 1580 * to or less than the number of on chip descriptors, which is 1581 * currently 40. 1582 */ 1583 txdctl |= (8 << 16); /* WTHRESH = 8 */ 1584 1585 /* Setting PTHRESH to 32 both improves performance */ 1586 txdctl |= (1 << 8) | /* HTHRESH = 1 */ 1587 32; /* PTHRESH = 32 */ 1588 1589 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); 1590 1591 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl); 1592 1593 /* poll to verify queue is enabled */ 1594 do { 1595 usleep_range(1000, 2000); 1596 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx)); 1597 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 1598 if (!wait_loop) 1599 pr_err("Could not enable Tx Queue %d\n", reg_idx); 1600 } 1601 1602 /** 1603 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1604 * @adapter: board private structure 1605 * 1606 * Configure the Tx unit of the MAC after a reset. 1607 **/ 1608 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1609 { 1610 u32 i; 1611 1612 /* Setup the HW Tx Head and Tail descriptor pointers */ 1613 for (i = 0; i < adapter->num_tx_queues; i++) 1614 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]); 1615 } 1616 1617 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1618 1619 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) 1620 { 1621 struct ixgbe_hw *hw = &adapter->hw; 1622 u32 srrctl; 1623 1624 srrctl = IXGBE_SRRCTL_DROP_EN; 1625 1626 srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; 1627 srrctl |= IXGBEVF_RX_BUFSZ >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1628 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1629 1630 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1631 } 1632 1633 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) 1634 { 1635 struct ixgbe_hw *hw = &adapter->hw; 1636 1637 /* PSRTYPE must be initialized in 82599 */ 1638 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 1639 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR | 1640 IXGBE_PSRTYPE_L2HDR; 1641 1642 if (adapter->num_rx_queues > 1) 1643 psrtype |= 1 << 29; 1644 1645 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1646 } 1647 1648 #define IXGBEVF_MAX_RX_DESC_POLL 10 1649 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, 1650 struct ixgbevf_ring *ring) 1651 { 1652 struct ixgbe_hw *hw = &adapter->hw; 1653 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1654 u32 rxdctl; 1655 u8 reg_idx = ring->reg_idx; 1656 1657 if (IXGBE_REMOVED(hw->hw_addr)) 1658 return; 1659 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1660 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1661 1662 /* write value back with RXDCTL.ENABLE bit cleared */ 1663 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1664 1665 /* the hardware may take up to 100us to really disable the Rx queue */ 1666 do { 1667 udelay(10); 1668 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1669 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 1670 1671 if (!wait_loop) 1672 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", 1673 reg_idx); 1674 } 1675 1676 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1677 struct ixgbevf_ring *ring) 1678 { 1679 struct ixgbe_hw *hw = &adapter->hw; 1680 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1681 u32 rxdctl; 1682 u8 reg_idx = ring->reg_idx; 1683 1684 if (IXGBE_REMOVED(hw->hw_addr)) 1685 return; 1686 do { 1687 usleep_range(1000, 2000); 1688 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1689 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 1690 1691 if (!wait_loop) 1692 pr_err("RXDCTL.ENABLE queue %d not set while polling\n", 1693 reg_idx); 1694 } 1695 1696 static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) 1697 { 1698 struct ixgbe_hw *hw = &adapter->hw; 1699 u32 vfmrqc = 0, vfreta = 0; 1700 u32 rss_key[10]; 1701 u16 rss_i = adapter->num_rx_queues; 1702 int i, j; 1703 1704 /* Fill out hash function seeds */ 1705 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 1706 for (i = 0; i < 10; i++) 1707 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1708 1709 /* Fill out redirection table */ 1710 for (i = 0, j = 0; i < 64; i++, j++) { 1711 if (j == rss_i) 1712 j = 0; 1713 vfreta = (vfreta << 8) | (j * 0x1); 1714 if ((i & 3) == 3) 1715 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), vfreta); 1716 } 1717 1718 /* Perform hash on these packet types */ 1719 vfmrqc |= IXGBE_VFMRQC_RSS_FIELD_IPV4 | 1720 IXGBE_VFMRQC_RSS_FIELD_IPV4_TCP | 1721 IXGBE_VFMRQC_RSS_FIELD_IPV6 | 1722 IXGBE_VFMRQC_RSS_FIELD_IPV6_TCP; 1723 1724 vfmrqc |= IXGBE_VFMRQC_RSSEN; 1725 1726 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); 1727 } 1728 1729 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, 1730 struct ixgbevf_ring *ring) 1731 { 1732 struct ixgbe_hw *hw = &adapter->hw; 1733 u64 rdba = ring->dma; 1734 u32 rxdctl; 1735 u8 reg_idx = ring->reg_idx; 1736 1737 /* disable queue to avoid issues while updating state */ 1738 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1739 ixgbevf_disable_rx_queue(adapter, ring); 1740 1741 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 1742 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32); 1743 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx), 1744 ring->count * sizeof(union ixgbe_adv_rx_desc)); 1745 1746 /* enable relaxed ordering */ 1747 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1748 IXGBE_DCA_RXCTRL_DESC_RRO_EN); 1749 1750 /* reset head and tail pointers */ 1751 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0); 1752 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); 1753 ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); 1754 1755 /* reset ntu and ntc to place SW in sync with hardwdare */ 1756 ring->next_to_clean = 0; 1757 ring->next_to_use = 0; 1758 ring->next_to_alloc = 0; 1759 1760 ixgbevf_configure_srrctl(adapter, reg_idx); 1761 1762 /* allow any size packet since we can handle overflow */ 1763 rxdctl &= ~IXGBE_RXDCTL_RLPML_EN; 1764 1765 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1766 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1767 1768 ixgbevf_rx_desc_queue_enable(adapter, ring); 1769 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); 1770 } 1771 1772 /** 1773 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1774 * @adapter: board private structure 1775 * 1776 * Configure the Rx unit of the MAC after a reset. 1777 **/ 1778 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1779 { 1780 int i; 1781 struct ixgbe_hw *hw = &adapter->hw; 1782 struct net_device *netdev = adapter->netdev; 1783 1784 ixgbevf_setup_psrtype(adapter); 1785 if (hw->mac.type >= ixgbe_mac_X550_vf) 1786 ixgbevf_setup_vfmrqc(adapter); 1787 1788 /* notify the PF of our intent to use this size of frame */ 1789 ixgbevf_rlpml_set_vf(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN); 1790 1791 /* Setup the HW Rx Head and Tail Descriptor Pointers and 1792 * the Base and Length of the Rx Descriptor Ring 1793 */ 1794 for (i = 0; i < adapter->num_rx_queues; i++) 1795 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]); 1796 } 1797 1798 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, 1799 __be16 proto, u16 vid) 1800 { 1801 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1802 struct ixgbe_hw *hw = &adapter->hw; 1803 int err; 1804 1805 spin_lock_bh(&adapter->mbx_lock); 1806 1807 /* add VID to filter table */ 1808 err = hw->mac.ops.set_vfta(hw, vid, 0, true); 1809 1810 spin_unlock_bh(&adapter->mbx_lock); 1811 1812 /* translate error return types so error makes sense */ 1813 if (err == IXGBE_ERR_MBX) 1814 return -EIO; 1815 1816 if (err == IXGBE_ERR_INVALID_ARGUMENT) 1817 return -EACCES; 1818 1819 set_bit(vid, adapter->active_vlans); 1820 1821 return err; 1822 } 1823 1824 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, 1825 __be16 proto, u16 vid) 1826 { 1827 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1828 struct ixgbe_hw *hw = &adapter->hw; 1829 int err = -EOPNOTSUPP; 1830 1831 spin_lock_bh(&adapter->mbx_lock); 1832 1833 /* remove VID from filter table */ 1834 err = hw->mac.ops.set_vfta(hw, vid, 0, false); 1835 1836 spin_unlock_bh(&adapter->mbx_lock); 1837 1838 clear_bit(vid, adapter->active_vlans); 1839 1840 return err; 1841 } 1842 1843 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 1844 { 1845 u16 vid; 1846 1847 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1848 ixgbevf_vlan_rx_add_vid(adapter->netdev, 1849 htons(ETH_P_8021Q), vid); 1850 } 1851 1852 static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 1853 { 1854 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1855 struct ixgbe_hw *hw = &adapter->hw; 1856 int count = 0; 1857 1858 if ((netdev_uc_count(netdev)) > 10) { 1859 pr_err("Too many unicast filters - No Space\n"); 1860 return -ENOSPC; 1861 } 1862 1863 if (!netdev_uc_empty(netdev)) { 1864 struct netdev_hw_addr *ha; 1865 1866 netdev_for_each_uc_addr(ha, netdev) { 1867 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 1868 udelay(200); 1869 } 1870 } else { 1871 /* If the list is empty then send message to PF driver to 1872 * clear all MAC VLANs on this VF. 1873 */ 1874 hw->mac.ops.set_uc_addr(hw, 0, NULL); 1875 } 1876 1877 return count; 1878 } 1879 1880 /** 1881 * ixgbevf_set_rx_mode - Multicast and unicast set 1882 * @netdev: network interface device structure 1883 * 1884 * The set_rx_method entry point is called whenever the multicast address 1885 * list, unicast address list or the network interface flags are updated. 1886 * This routine is responsible for configuring the hardware for proper 1887 * multicast mode and configuring requested unicast filters. 1888 **/ 1889 static void ixgbevf_set_rx_mode(struct net_device *netdev) 1890 { 1891 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1892 struct ixgbe_hw *hw = &adapter->hw; 1893 1894 spin_lock_bh(&adapter->mbx_lock); 1895 1896 /* reprogram multicast list */ 1897 hw->mac.ops.update_mc_addr_list(hw, netdev); 1898 1899 ixgbevf_write_uc_addr_list(netdev); 1900 1901 spin_unlock_bh(&adapter->mbx_lock); 1902 } 1903 1904 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 1905 { 1906 int q_idx; 1907 struct ixgbevf_q_vector *q_vector; 1908 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1909 1910 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1911 q_vector = adapter->q_vector[q_idx]; 1912 #ifdef CONFIG_NET_RX_BUSY_POLL 1913 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]); 1914 #endif 1915 napi_enable(&q_vector->napi); 1916 } 1917 } 1918 1919 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 1920 { 1921 int q_idx; 1922 struct ixgbevf_q_vector *q_vector; 1923 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1924 1925 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1926 q_vector = adapter->q_vector[q_idx]; 1927 napi_disable(&q_vector->napi); 1928 #ifdef CONFIG_NET_RX_BUSY_POLL 1929 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) { 1930 pr_info("QV %d locked\n", q_idx); 1931 usleep_range(1000, 20000); 1932 } 1933 #endif /* CONFIG_NET_RX_BUSY_POLL */ 1934 } 1935 } 1936 1937 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter) 1938 { 1939 struct ixgbe_hw *hw = &adapter->hw; 1940 unsigned int def_q = 0; 1941 unsigned int num_tcs = 0; 1942 unsigned int num_rx_queues = adapter->num_rx_queues; 1943 unsigned int num_tx_queues = adapter->num_tx_queues; 1944 int err; 1945 1946 spin_lock_bh(&adapter->mbx_lock); 1947 1948 /* fetch queue configuration from the PF */ 1949 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 1950 1951 spin_unlock_bh(&adapter->mbx_lock); 1952 1953 if (err) 1954 return err; 1955 1956 if (num_tcs > 1) { 1957 /* we need only one Tx queue */ 1958 num_tx_queues = 1; 1959 1960 /* update default Tx ring register index */ 1961 adapter->tx_ring[0]->reg_idx = def_q; 1962 1963 /* we need as many queues as traffic classes */ 1964 num_rx_queues = num_tcs; 1965 } 1966 1967 /* if we have a bad config abort request queue reset */ 1968 if ((adapter->num_rx_queues != num_rx_queues) || 1969 (adapter->num_tx_queues != num_tx_queues)) { 1970 /* force mailbox timeout to prevent further messages */ 1971 hw->mbx.timeout = 0; 1972 1973 /* wait for watchdog to come around and bail us out */ 1974 adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED; 1975 } 1976 1977 return 0; 1978 } 1979 1980 static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 1981 { 1982 ixgbevf_configure_dcb(adapter); 1983 1984 ixgbevf_set_rx_mode(adapter->netdev); 1985 1986 ixgbevf_restore_vlan(adapter); 1987 1988 ixgbevf_configure_tx(adapter); 1989 ixgbevf_configure_rx(adapter); 1990 } 1991 1992 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 1993 { 1994 /* Only save pre-reset stats if there are some */ 1995 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 1996 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 1997 adapter->stats.base_vfgprc; 1998 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 1999 adapter->stats.base_vfgptc; 2000 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 2001 adapter->stats.base_vfgorc; 2002 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 2003 adapter->stats.base_vfgotc; 2004 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 2005 adapter->stats.base_vfmprc; 2006 } 2007 } 2008 2009 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 2010 { 2011 struct ixgbe_hw *hw = &adapter->hw; 2012 2013 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2014 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2015 adapter->stats.last_vfgorc |= 2016 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2017 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2018 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2019 adapter->stats.last_vfgotc |= 2020 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2021 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2022 2023 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 2024 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 2025 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 2026 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 2027 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 2028 } 2029 2030 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) 2031 { 2032 struct ixgbe_hw *hw = &adapter->hw; 2033 int api[] = { ixgbe_mbox_api_12, 2034 ixgbe_mbox_api_11, 2035 ixgbe_mbox_api_10, 2036 ixgbe_mbox_api_unknown }; 2037 int err = 0, idx = 0; 2038 2039 spin_lock_bh(&adapter->mbx_lock); 2040 2041 while (api[idx] != ixgbe_mbox_api_unknown) { 2042 err = ixgbevf_negotiate_api_version(hw, api[idx]); 2043 if (!err) 2044 break; 2045 idx++; 2046 } 2047 2048 spin_unlock_bh(&adapter->mbx_lock); 2049 } 2050 2051 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 2052 { 2053 struct net_device *netdev = adapter->netdev; 2054 struct ixgbe_hw *hw = &adapter->hw; 2055 2056 ixgbevf_configure_msix(adapter); 2057 2058 spin_lock_bh(&adapter->mbx_lock); 2059 2060 if (is_valid_ether_addr(hw->mac.addr)) 2061 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 2062 else 2063 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 2064 2065 spin_unlock_bh(&adapter->mbx_lock); 2066 2067 smp_mb__before_atomic(); 2068 clear_bit(__IXGBEVF_DOWN, &adapter->state); 2069 ixgbevf_napi_enable_all(adapter); 2070 2071 /* clear any pending interrupts, may auto mask */ 2072 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2073 ixgbevf_irq_enable(adapter); 2074 2075 /* enable transmits */ 2076 netif_tx_start_all_queues(netdev); 2077 2078 ixgbevf_save_reset_stats(adapter); 2079 ixgbevf_init_last_counter_stats(adapter); 2080 2081 hw->mac.get_link_status = 1; 2082 mod_timer(&adapter->service_timer, jiffies); 2083 } 2084 2085 void ixgbevf_up(struct ixgbevf_adapter *adapter) 2086 { 2087 ixgbevf_configure(adapter); 2088 2089 ixgbevf_up_complete(adapter); 2090 } 2091 2092 /** 2093 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 2094 * @rx_ring: ring to free buffers from 2095 **/ 2096 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) 2097 { 2098 struct device *dev = rx_ring->dev; 2099 unsigned long size; 2100 unsigned int i; 2101 2102 /* Free Rx ring sk_buff */ 2103 if (rx_ring->skb) { 2104 dev_kfree_skb(rx_ring->skb); 2105 rx_ring->skb = NULL; 2106 } 2107 2108 /* ring already cleared, nothing to do */ 2109 if (!rx_ring->rx_buffer_info) 2110 return; 2111 2112 /* Free all the Rx ring pages */ 2113 for (i = 0; i < rx_ring->count; i++) { 2114 struct ixgbevf_rx_buffer *rx_buffer; 2115 2116 rx_buffer = &rx_ring->rx_buffer_info[i]; 2117 if (rx_buffer->dma) 2118 dma_unmap_page(dev, rx_buffer->dma, 2119 PAGE_SIZE, DMA_FROM_DEVICE); 2120 rx_buffer->dma = 0; 2121 if (rx_buffer->page) 2122 __free_page(rx_buffer->page); 2123 rx_buffer->page = NULL; 2124 } 2125 2126 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 2127 memset(rx_ring->rx_buffer_info, 0, size); 2128 2129 /* Zero out the descriptor ring */ 2130 memset(rx_ring->desc, 0, rx_ring->size); 2131 } 2132 2133 /** 2134 * ixgbevf_clean_tx_ring - Free Tx Buffers 2135 * @tx_ring: ring to be cleaned 2136 **/ 2137 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) 2138 { 2139 struct ixgbevf_tx_buffer *tx_buffer_info; 2140 unsigned long size; 2141 unsigned int i; 2142 2143 if (!tx_ring->tx_buffer_info) 2144 return; 2145 2146 /* Free all the Tx ring sk_buffs */ 2147 for (i = 0; i < tx_ring->count; i++) { 2148 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2149 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 2150 } 2151 2152 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2153 memset(tx_ring->tx_buffer_info, 0, size); 2154 2155 memset(tx_ring->desc, 0, tx_ring->size); 2156 } 2157 2158 /** 2159 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 2160 * @adapter: board private structure 2161 **/ 2162 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 2163 { 2164 int i; 2165 2166 for (i = 0; i < adapter->num_rx_queues; i++) 2167 ixgbevf_clean_rx_ring(adapter->rx_ring[i]); 2168 } 2169 2170 /** 2171 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 2172 * @adapter: board private structure 2173 **/ 2174 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 2175 { 2176 int i; 2177 2178 for (i = 0; i < adapter->num_tx_queues; i++) 2179 ixgbevf_clean_tx_ring(adapter->tx_ring[i]); 2180 } 2181 2182 void ixgbevf_down(struct ixgbevf_adapter *adapter) 2183 { 2184 struct net_device *netdev = adapter->netdev; 2185 struct ixgbe_hw *hw = &adapter->hw; 2186 int i; 2187 2188 /* signal that we are down to the interrupt handler */ 2189 if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state)) 2190 return; /* do nothing if already down */ 2191 2192 /* disable all enabled Rx queues */ 2193 for (i = 0; i < adapter->num_rx_queues; i++) 2194 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]); 2195 2196 usleep_range(10000, 20000); 2197 2198 netif_tx_stop_all_queues(netdev); 2199 2200 /* call carrier off first to avoid false dev_watchdog timeouts */ 2201 netif_carrier_off(netdev); 2202 netif_tx_disable(netdev); 2203 2204 ixgbevf_irq_disable(adapter); 2205 2206 ixgbevf_napi_disable_all(adapter); 2207 2208 del_timer_sync(&adapter->service_timer); 2209 2210 /* disable transmits in the hardware now that interrupts are off */ 2211 for (i = 0; i < adapter->num_tx_queues; i++) { 2212 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 2213 2214 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), 2215 IXGBE_TXDCTL_SWFLSH); 2216 } 2217 2218 if (!pci_channel_offline(adapter->pdev)) 2219 ixgbevf_reset(adapter); 2220 2221 ixgbevf_clean_all_tx_rings(adapter); 2222 ixgbevf_clean_all_rx_rings(adapter); 2223 } 2224 2225 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 2226 { 2227 WARN_ON(in_interrupt()); 2228 2229 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 2230 msleep(1); 2231 2232 ixgbevf_down(adapter); 2233 ixgbevf_up(adapter); 2234 2235 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 2236 } 2237 2238 void ixgbevf_reset(struct ixgbevf_adapter *adapter) 2239 { 2240 struct ixgbe_hw *hw = &adapter->hw; 2241 struct net_device *netdev = adapter->netdev; 2242 2243 if (hw->mac.ops.reset_hw(hw)) { 2244 hw_dbg(hw, "PF still resetting\n"); 2245 } else { 2246 hw->mac.ops.init_hw(hw); 2247 ixgbevf_negotiate_api(adapter); 2248 } 2249 2250 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 2251 memcpy(netdev->dev_addr, adapter->hw.mac.addr, 2252 netdev->addr_len); 2253 memcpy(netdev->perm_addr, adapter->hw.mac.addr, 2254 netdev->addr_len); 2255 } 2256 2257 adapter->last_reset = jiffies; 2258 } 2259 2260 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 2261 int vectors) 2262 { 2263 int vector_threshold; 2264 2265 /* We'll want at least 2 (vector_threshold): 2266 * 1) TxQ[0] + RxQ[0] handler 2267 * 2) Other (Link Status Change, etc.) 2268 */ 2269 vector_threshold = MIN_MSIX_COUNT; 2270 2271 /* The more we get, the more we will assign to Tx/Rx Cleanup 2272 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 2273 * Right now, we simply care about how many we'll get; we'll 2274 * set them up later while requesting irq's. 2275 */ 2276 vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 2277 vector_threshold, vectors); 2278 2279 if (vectors < 0) { 2280 dev_err(&adapter->pdev->dev, 2281 "Unable to allocate MSI-X interrupts\n"); 2282 kfree(adapter->msix_entries); 2283 adapter->msix_entries = NULL; 2284 return vectors; 2285 } 2286 2287 /* Adjust for only the vectors we'll use, which is minimum 2288 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 2289 * vectors we were allocated. 2290 */ 2291 adapter->num_msix_vectors = vectors; 2292 2293 return 0; 2294 } 2295 2296 /** 2297 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent 2298 * @adapter: board private structure to initialize 2299 * 2300 * This is the top level queue allocation routine. The order here is very 2301 * important, starting with the "most" number of features turned on at once, 2302 * and ending with the smallest set of features. This way large combinations 2303 * can be allocated if they're turned on, and smaller combinations are the 2304 * fallthrough conditions. 2305 * 2306 **/ 2307 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 2308 { 2309 struct ixgbe_hw *hw = &adapter->hw; 2310 unsigned int def_q = 0; 2311 unsigned int num_tcs = 0; 2312 int err; 2313 2314 /* Start with base case */ 2315 adapter->num_rx_queues = 1; 2316 adapter->num_tx_queues = 1; 2317 2318 spin_lock_bh(&adapter->mbx_lock); 2319 2320 /* fetch queue configuration from the PF */ 2321 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2322 2323 spin_unlock_bh(&adapter->mbx_lock); 2324 2325 if (err) 2326 return; 2327 2328 /* we need as many queues as traffic classes */ 2329 if (num_tcs > 1) { 2330 adapter->num_rx_queues = num_tcs; 2331 } else { 2332 u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES); 2333 2334 switch (hw->api_version) { 2335 case ixgbe_mbox_api_11: 2336 case ixgbe_mbox_api_12: 2337 adapter->num_rx_queues = rss; 2338 adapter->num_tx_queues = rss; 2339 default: 2340 break; 2341 } 2342 } 2343 } 2344 2345 /** 2346 * ixgbevf_alloc_queues - Allocate memory for all rings 2347 * @adapter: board private structure to initialize 2348 * 2349 * We allocate one ring per queue at run-time since we don't know the 2350 * number of queues at compile-time. The polling_netdev array is 2351 * intended for Multiqueue, but should work fine with a single queue. 2352 **/ 2353 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) 2354 { 2355 struct ixgbevf_ring *ring; 2356 int rx = 0, tx = 0; 2357 2358 for (; tx < adapter->num_tx_queues; tx++) { 2359 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 2360 if (!ring) 2361 goto err_allocation; 2362 2363 ring->dev = &adapter->pdev->dev; 2364 ring->netdev = adapter->netdev; 2365 ring->count = adapter->tx_ring_count; 2366 ring->queue_index = tx; 2367 ring->reg_idx = tx; 2368 2369 adapter->tx_ring[tx] = ring; 2370 } 2371 2372 for (; rx < adapter->num_rx_queues; rx++) { 2373 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 2374 if (!ring) 2375 goto err_allocation; 2376 2377 ring->dev = &adapter->pdev->dev; 2378 ring->netdev = adapter->netdev; 2379 2380 ring->count = adapter->rx_ring_count; 2381 ring->queue_index = rx; 2382 ring->reg_idx = rx; 2383 2384 adapter->rx_ring[rx] = ring; 2385 } 2386 2387 return 0; 2388 2389 err_allocation: 2390 while (tx) { 2391 kfree(adapter->tx_ring[--tx]); 2392 adapter->tx_ring[tx] = NULL; 2393 } 2394 2395 while (rx) { 2396 kfree(adapter->rx_ring[--rx]); 2397 adapter->rx_ring[rx] = NULL; 2398 } 2399 return -ENOMEM; 2400 } 2401 2402 /** 2403 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 2404 * @adapter: board private structure to initialize 2405 * 2406 * Attempt to configure the interrupts using the best available 2407 * capabilities of the hardware and the kernel. 2408 **/ 2409 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 2410 { 2411 struct net_device *netdev = adapter->netdev; 2412 int err = 0; 2413 int vector, v_budget; 2414 2415 /* It's easy to be greedy for MSI-X vectors, but it really 2416 * doesn't do us much good if we have a lot more vectors 2417 * than CPU's. So let's be conservative and only ask for 2418 * (roughly) the same number of vectors as there are CPU's. 2419 * The default is to use pairs of vectors. 2420 */ 2421 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); 2422 v_budget = min_t(int, v_budget, num_online_cpus()); 2423 v_budget += NON_Q_VECTORS; 2424 2425 /* A failure in MSI-X entry allocation isn't fatal, but it does 2426 * mean we disable MSI-X capabilities of the adapter. 2427 */ 2428 adapter->msix_entries = kcalloc(v_budget, 2429 sizeof(struct msix_entry), GFP_KERNEL); 2430 if (!adapter->msix_entries) { 2431 err = -ENOMEM; 2432 goto out; 2433 } 2434 2435 for (vector = 0; vector < v_budget; vector++) 2436 adapter->msix_entries[vector].entry = vector; 2437 2438 err = ixgbevf_acquire_msix_vectors(adapter, v_budget); 2439 if (err) 2440 goto out; 2441 2442 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 2443 if (err) 2444 goto out; 2445 2446 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 2447 2448 out: 2449 return err; 2450 } 2451 2452 /** 2453 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 2454 * @adapter: board private structure to initialize 2455 * 2456 * We allocate one q_vector per queue interrupt. If allocation fails we 2457 * return -ENOMEM. 2458 **/ 2459 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 2460 { 2461 int q_idx, num_q_vectors; 2462 struct ixgbevf_q_vector *q_vector; 2463 2464 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2465 2466 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2467 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL); 2468 if (!q_vector) 2469 goto err_out; 2470 q_vector->adapter = adapter; 2471 q_vector->v_idx = q_idx; 2472 netif_napi_add(adapter->netdev, &q_vector->napi, 2473 ixgbevf_poll, 64); 2474 #ifdef CONFIG_NET_RX_BUSY_POLL 2475 napi_hash_add(&q_vector->napi); 2476 #endif 2477 adapter->q_vector[q_idx] = q_vector; 2478 } 2479 2480 return 0; 2481 2482 err_out: 2483 while (q_idx) { 2484 q_idx--; 2485 q_vector = adapter->q_vector[q_idx]; 2486 #ifdef CONFIG_NET_RX_BUSY_POLL 2487 napi_hash_del(&q_vector->napi); 2488 #endif 2489 netif_napi_del(&q_vector->napi); 2490 kfree(q_vector); 2491 adapter->q_vector[q_idx] = NULL; 2492 } 2493 return -ENOMEM; 2494 } 2495 2496 /** 2497 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 2498 * @adapter: board private structure to initialize 2499 * 2500 * This function frees the memory allocated to the q_vectors. In addition if 2501 * NAPI is enabled it will delete any references to the NAPI struct prior 2502 * to freeing the q_vector. 2503 **/ 2504 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 2505 { 2506 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2507 2508 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2509 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; 2510 2511 adapter->q_vector[q_idx] = NULL; 2512 #ifdef CONFIG_NET_RX_BUSY_POLL 2513 napi_hash_del(&q_vector->napi); 2514 #endif 2515 netif_napi_del(&q_vector->napi); 2516 kfree(q_vector); 2517 } 2518 } 2519 2520 /** 2521 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 2522 * @adapter: board private structure 2523 * 2524 **/ 2525 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 2526 { 2527 pci_disable_msix(adapter->pdev); 2528 kfree(adapter->msix_entries); 2529 adapter->msix_entries = NULL; 2530 } 2531 2532 /** 2533 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 2534 * @adapter: board private structure to initialize 2535 * 2536 **/ 2537 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 2538 { 2539 int err; 2540 2541 /* Number of supported queues */ 2542 ixgbevf_set_num_queues(adapter); 2543 2544 err = ixgbevf_set_interrupt_capability(adapter); 2545 if (err) { 2546 hw_dbg(&adapter->hw, 2547 "Unable to setup interrupt capabilities\n"); 2548 goto err_set_interrupt; 2549 } 2550 2551 err = ixgbevf_alloc_q_vectors(adapter); 2552 if (err) { 2553 hw_dbg(&adapter->hw, "Unable to allocate memory for queue vectors\n"); 2554 goto err_alloc_q_vectors; 2555 } 2556 2557 err = ixgbevf_alloc_queues(adapter); 2558 if (err) { 2559 pr_err("Unable to allocate memory for queues\n"); 2560 goto err_alloc_queues; 2561 } 2562 2563 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u\n", 2564 (adapter->num_rx_queues > 1) ? "Enabled" : 2565 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues); 2566 2567 set_bit(__IXGBEVF_DOWN, &adapter->state); 2568 2569 return 0; 2570 err_alloc_queues: 2571 ixgbevf_free_q_vectors(adapter); 2572 err_alloc_q_vectors: 2573 ixgbevf_reset_interrupt_capability(adapter); 2574 err_set_interrupt: 2575 return err; 2576 } 2577 2578 /** 2579 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings 2580 * @adapter: board private structure to clear interrupt scheme on 2581 * 2582 * We go through and clear interrupt specific resources and reset the structure 2583 * to pre-load conditions 2584 **/ 2585 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) 2586 { 2587 int i; 2588 2589 for (i = 0; i < adapter->num_tx_queues; i++) { 2590 kfree(adapter->tx_ring[i]); 2591 adapter->tx_ring[i] = NULL; 2592 } 2593 for (i = 0; i < adapter->num_rx_queues; i++) { 2594 kfree(adapter->rx_ring[i]); 2595 adapter->rx_ring[i] = NULL; 2596 } 2597 2598 adapter->num_tx_queues = 0; 2599 adapter->num_rx_queues = 0; 2600 2601 ixgbevf_free_q_vectors(adapter); 2602 ixgbevf_reset_interrupt_capability(adapter); 2603 } 2604 2605 /** 2606 * ixgbevf_sw_init - Initialize general software structures 2607 * @adapter: board private structure to initialize 2608 * 2609 * ixgbevf_sw_init initializes the Adapter private data structure. 2610 * Fields are initialized based on PCI device information and 2611 * OS network device settings (MTU size). 2612 **/ 2613 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 2614 { 2615 struct ixgbe_hw *hw = &adapter->hw; 2616 struct pci_dev *pdev = adapter->pdev; 2617 struct net_device *netdev = adapter->netdev; 2618 int err; 2619 2620 /* PCI config space info */ 2621 hw->vendor_id = pdev->vendor; 2622 hw->device_id = pdev->device; 2623 hw->revision_id = pdev->revision; 2624 hw->subsystem_vendor_id = pdev->subsystem_vendor; 2625 hw->subsystem_device_id = pdev->subsystem_device; 2626 2627 hw->mbx.ops.init_params(hw); 2628 2629 /* assume legacy case in which PF would only give VF 2 queues */ 2630 hw->mac.max_tx_queues = 2; 2631 hw->mac.max_rx_queues = 2; 2632 2633 /* lock to protect mailbox accesses */ 2634 spin_lock_init(&adapter->mbx_lock); 2635 2636 err = hw->mac.ops.reset_hw(hw); 2637 if (err) { 2638 dev_info(&pdev->dev, 2639 "PF still in reset state. Is the PF interface up?\n"); 2640 } else { 2641 err = hw->mac.ops.init_hw(hw); 2642 if (err) { 2643 pr_err("init_shared_code failed: %d\n", err); 2644 goto out; 2645 } 2646 ixgbevf_negotiate_api(adapter); 2647 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr); 2648 if (err) 2649 dev_info(&pdev->dev, "Error reading MAC address\n"); 2650 else if (is_zero_ether_addr(adapter->hw.mac.addr)) 2651 dev_info(&pdev->dev, 2652 "MAC address not assigned by administrator.\n"); 2653 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 2654 } 2655 2656 if (!is_valid_ether_addr(netdev->dev_addr)) { 2657 dev_info(&pdev->dev, "Assigning random MAC address\n"); 2658 eth_hw_addr_random(netdev); 2659 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len); 2660 } 2661 2662 /* Enable dynamic interrupt throttling rates */ 2663 adapter->rx_itr_setting = 1; 2664 adapter->tx_itr_setting = 1; 2665 2666 /* set default ring sizes */ 2667 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 2668 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 2669 2670 set_bit(__IXGBEVF_DOWN, &adapter->state); 2671 return 0; 2672 2673 out: 2674 return err; 2675 } 2676 2677 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 2678 { \ 2679 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 2680 if (current_counter < last_counter) \ 2681 counter += 0x100000000LL; \ 2682 last_counter = current_counter; \ 2683 counter &= 0xFFFFFFFF00000000LL; \ 2684 counter |= current_counter; \ 2685 } 2686 2687 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 2688 { \ 2689 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 2690 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 2691 u64 current_counter = (current_counter_msb << 32) | \ 2692 current_counter_lsb; \ 2693 if (current_counter < last_counter) \ 2694 counter += 0x1000000000LL; \ 2695 last_counter = current_counter; \ 2696 counter &= 0xFFFFFFF000000000LL; \ 2697 counter |= current_counter; \ 2698 } 2699 /** 2700 * ixgbevf_update_stats - Update the board statistics counters. 2701 * @adapter: board private structure 2702 **/ 2703 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 2704 { 2705 struct ixgbe_hw *hw = &adapter->hw; 2706 int i; 2707 2708 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2709 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2710 return; 2711 2712 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 2713 adapter->stats.vfgprc); 2714 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 2715 adapter->stats.vfgptc); 2716 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2717 adapter->stats.last_vfgorc, 2718 adapter->stats.vfgorc); 2719 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2720 adapter->stats.last_vfgotc, 2721 adapter->stats.vfgotc); 2722 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 2723 adapter->stats.vfmprc); 2724 2725 for (i = 0; i < adapter->num_rx_queues; i++) { 2726 adapter->hw_csum_rx_error += 2727 adapter->rx_ring[i]->hw_csum_rx_error; 2728 adapter->rx_ring[i]->hw_csum_rx_error = 0; 2729 } 2730 } 2731 2732 /** 2733 * ixgbevf_service_timer - Timer Call-back 2734 * @data: pointer to adapter cast into an unsigned long 2735 **/ 2736 static void ixgbevf_service_timer(unsigned long data) 2737 { 2738 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data; 2739 2740 /* Reset the timer */ 2741 mod_timer(&adapter->service_timer, (HZ * 2) + jiffies); 2742 2743 ixgbevf_service_event_schedule(adapter); 2744 } 2745 2746 static void ixgbevf_reset_subtask(struct ixgbevf_adapter *adapter) 2747 { 2748 if (!(adapter->flags & IXGBEVF_FLAG_RESET_REQUESTED)) 2749 return; 2750 2751 adapter->flags &= ~IXGBEVF_FLAG_RESET_REQUESTED; 2752 2753 /* If we're already down or resetting, just bail */ 2754 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2755 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2756 return; 2757 2758 adapter->tx_timeout_count++; 2759 2760 ixgbevf_reinit_locked(adapter); 2761 } 2762 2763 /** 2764 * ixgbevf_check_hang_subtask - check for hung queues and dropped interrupts 2765 * @adapter: pointer to the device adapter structure 2766 * 2767 * This function serves two purposes. First it strobes the interrupt lines 2768 * in order to make certain interrupts are occurring. Secondly it sets the 2769 * bits needed to check for TX hangs. As a result we should immediately 2770 * determine if a hang has occurred. 2771 **/ 2772 static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter) 2773 { 2774 struct ixgbe_hw *hw = &adapter->hw; 2775 u32 eics = 0; 2776 int i; 2777 2778 /* If we're down or resetting, just bail */ 2779 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2780 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2781 return; 2782 2783 /* Force detection of hung controller */ 2784 if (netif_carrier_ok(adapter->netdev)) { 2785 for (i = 0; i < adapter->num_tx_queues; i++) 2786 set_check_for_tx_hang(adapter->tx_ring[i]); 2787 } 2788 2789 /* get one bit for every active Tx/Rx interrupt vector */ 2790 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 2791 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 2792 2793 if (qv->rx.ring || qv->tx.ring) 2794 eics |= 1 << i; 2795 } 2796 2797 /* Cause software interrupt to ensure rings are cleaned */ 2798 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics); 2799 } 2800 2801 /** 2802 * ixgbevf_watchdog_update_link - update the link status 2803 * @adapter: pointer to the device adapter structure 2804 **/ 2805 static void ixgbevf_watchdog_update_link(struct ixgbevf_adapter *adapter) 2806 { 2807 struct ixgbe_hw *hw = &adapter->hw; 2808 u32 link_speed = adapter->link_speed; 2809 bool link_up = adapter->link_up; 2810 s32 err; 2811 2812 spin_lock_bh(&adapter->mbx_lock); 2813 2814 err = hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 2815 2816 spin_unlock_bh(&adapter->mbx_lock); 2817 2818 /* if check for link returns error we will need to reset */ 2819 if (err && time_after(jiffies, adapter->last_reset + (10 * HZ))) { 2820 adapter->flags |= IXGBEVF_FLAG_RESET_REQUESTED; 2821 link_up = false; 2822 } 2823 2824 adapter->link_up = link_up; 2825 adapter->link_speed = link_speed; 2826 } 2827 2828 /** 2829 * ixgbevf_watchdog_link_is_up - update netif_carrier status and 2830 * print link up message 2831 * @adapter: pointer to the device adapter structure 2832 **/ 2833 static void ixgbevf_watchdog_link_is_up(struct ixgbevf_adapter *adapter) 2834 { 2835 struct net_device *netdev = adapter->netdev; 2836 2837 /* only continue if link was previously down */ 2838 if (netif_carrier_ok(netdev)) 2839 return; 2840 2841 dev_info(&adapter->pdev->dev, "NIC Link is Up %s\n", 2842 (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 2843 "10 Gbps" : 2844 (adapter->link_speed == IXGBE_LINK_SPEED_1GB_FULL) ? 2845 "1 Gbps" : 2846 (adapter->link_speed == IXGBE_LINK_SPEED_100_FULL) ? 2847 "100 Mbps" : 2848 "unknown speed"); 2849 2850 netif_carrier_on(netdev); 2851 } 2852 2853 /** 2854 * ixgbevf_watchdog_link_is_down - update netif_carrier status and 2855 * print link down message 2856 * @adapter: pointer to the adapter structure 2857 **/ 2858 static void ixgbevf_watchdog_link_is_down(struct ixgbevf_adapter *adapter) 2859 { 2860 struct net_device *netdev = adapter->netdev; 2861 2862 adapter->link_speed = 0; 2863 2864 /* only continue if link was up previously */ 2865 if (!netif_carrier_ok(netdev)) 2866 return; 2867 2868 dev_info(&adapter->pdev->dev, "NIC Link is Down\n"); 2869 2870 netif_carrier_off(netdev); 2871 } 2872 2873 /** 2874 * ixgbevf_watchdog_subtask - worker thread to bring link up 2875 * @work: pointer to work_struct containing our data 2876 **/ 2877 static void ixgbevf_watchdog_subtask(struct ixgbevf_adapter *adapter) 2878 { 2879 /* if interface is down do nothing */ 2880 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2881 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2882 return; 2883 2884 ixgbevf_watchdog_update_link(adapter); 2885 2886 if (adapter->link_up) 2887 ixgbevf_watchdog_link_is_up(adapter); 2888 else 2889 ixgbevf_watchdog_link_is_down(adapter); 2890 2891 ixgbevf_update_stats(adapter); 2892 } 2893 2894 /** 2895 * ixgbevf_service_task - manages and runs subtasks 2896 * @work: pointer to work_struct containing our data 2897 **/ 2898 static void ixgbevf_service_task(struct work_struct *work) 2899 { 2900 struct ixgbevf_adapter *adapter = container_of(work, 2901 struct ixgbevf_adapter, 2902 service_task); 2903 struct ixgbe_hw *hw = &adapter->hw; 2904 2905 if (IXGBE_REMOVED(hw->hw_addr)) { 2906 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 2907 rtnl_lock(); 2908 ixgbevf_down(adapter); 2909 rtnl_unlock(); 2910 } 2911 return; 2912 } 2913 2914 ixgbevf_queue_reset_subtask(adapter); 2915 ixgbevf_reset_subtask(adapter); 2916 ixgbevf_watchdog_subtask(adapter); 2917 ixgbevf_check_hang_subtask(adapter); 2918 2919 ixgbevf_service_event_complete(adapter); 2920 } 2921 2922 /** 2923 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 2924 * @tx_ring: Tx descriptor ring for a specific queue 2925 * 2926 * Free all transmit software resources 2927 **/ 2928 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) 2929 { 2930 ixgbevf_clean_tx_ring(tx_ring); 2931 2932 vfree(tx_ring->tx_buffer_info); 2933 tx_ring->tx_buffer_info = NULL; 2934 2935 /* if not set, then don't free */ 2936 if (!tx_ring->desc) 2937 return; 2938 2939 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, 2940 tx_ring->dma); 2941 2942 tx_ring->desc = NULL; 2943 } 2944 2945 /** 2946 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 2947 * @adapter: board private structure 2948 * 2949 * Free all transmit software resources 2950 **/ 2951 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 2952 { 2953 int i; 2954 2955 for (i = 0; i < adapter->num_tx_queues; i++) 2956 if (adapter->tx_ring[i]->desc) 2957 ixgbevf_free_tx_resources(adapter->tx_ring[i]); 2958 } 2959 2960 /** 2961 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 2962 * @tx_ring: Tx descriptor ring (for a specific queue) to setup 2963 * 2964 * Return 0 on success, negative on failure 2965 **/ 2966 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) 2967 { 2968 int size; 2969 2970 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2971 tx_ring->tx_buffer_info = vzalloc(size); 2972 if (!tx_ring->tx_buffer_info) 2973 goto err; 2974 2975 /* round up to nearest 4K */ 2976 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 2977 tx_ring->size = ALIGN(tx_ring->size, 4096); 2978 2979 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, 2980 &tx_ring->dma, GFP_KERNEL); 2981 if (!tx_ring->desc) 2982 goto err; 2983 2984 return 0; 2985 2986 err: 2987 vfree(tx_ring->tx_buffer_info); 2988 tx_ring->tx_buffer_info = NULL; 2989 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit descriptor ring\n"); 2990 return -ENOMEM; 2991 } 2992 2993 /** 2994 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 2995 * @adapter: board private structure 2996 * 2997 * If this function returns with an error, then it's possible one or 2998 * more of the rings is populated (while the rest are not). It is the 2999 * callers duty to clean those orphaned rings. 3000 * 3001 * Return 0 on success, negative on failure 3002 **/ 3003 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 3004 { 3005 int i, err = 0; 3006 3007 for (i = 0; i < adapter->num_tx_queues; i++) { 3008 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]); 3009 if (!err) 3010 continue; 3011 hw_dbg(&adapter->hw, "Allocation for Tx Queue %u failed\n", i); 3012 break; 3013 } 3014 3015 return err; 3016 } 3017 3018 /** 3019 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 3020 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 3021 * 3022 * Returns 0 on success, negative on failure 3023 **/ 3024 int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring) 3025 { 3026 int size; 3027 3028 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 3029 rx_ring->rx_buffer_info = vzalloc(size); 3030 if (!rx_ring->rx_buffer_info) 3031 goto err; 3032 3033 /* Round up to nearest 4K */ 3034 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 3035 rx_ring->size = ALIGN(rx_ring->size, 4096); 3036 3037 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, 3038 &rx_ring->dma, GFP_KERNEL); 3039 3040 if (!rx_ring->desc) 3041 goto err; 3042 3043 return 0; 3044 err: 3045 vfree(rx_ring->rx_buffer_info); 3046 rx_ring->rx_buffer_info = NULL; 3047 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); 3048 return -ENOMEM; 3049 } 3050 3051 /** 3052 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 3053 * @adapter: board private structure 3054 * 3055 * If this function returns with an error, then it's possible one or 3056 * more of the rings is populated (while the rest are not). It is the 3057 * callers duty to clean those orphaned rings. 3058 * 3059 * Return 0 on success, negative on failure 3060 **/ 3061 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 3062 { 3063 int i, err = 0; 3064 3065 for (i = 0; i < adapter->num_rx_queues; i++) { 3066 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]); 3067 if (!err) 3068 continue; 3069 hw_dbg(&adapter->hw, "Allocation for Rx Queue %u failed\n", i); 3070 break; 3071 } 3072 return err; 3073 } 3074 3075 /** 3076 * ixgbevf_free_rx_resources - Free Rx Resources 3077 * @rx_ring: ring to clean the resources from 3078 * 3079 * Free all receive software resources 3080 **/ 3081 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) 3082 { 3083 ixgbevf_clean_rx_ring(rx_ring); 3084 3085 vfree(rx_ring->rx_buffer_info); 3086 rx_ring->rx_buffer_info = NULL; 3087 3088 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, 3089 rx_ring->dma); 3090 3091 rx_ring->desc = NULL; 3092 } 3093 3094 /** 3095 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 3096 * @adapter: board private structure 3097 * 3098 * Free all receive software resources 3099 **/ 3100 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 3101 { 3102 int i; 3103 3104 for (i = 0; i < adapter->num_rx_queues; i++) 3105 if (adapter->rx_ring[i]->desc) 3106 ixgbevf_free_rx_resources(adapter->rx_ring[i]); 3107 } 3108 3109 /** 3110 * ixgbevf_open - Called when a network interface is made active 3111 * @netdev: network interface device structure 3112 * 3113 * Returns 0 on success, negative value on failure 3114 * 3115 * The open entry point is called when a network interface is made 3116 * active by the system (IFF_UP). At this point all resources needed 3117 * for transmit and receive operations are allocated, the interrupt 3118 * handler is registered with the OS, the watchdog timer is started, 3119 * and the stack is notified that the interface is ready. 3120 **/ 3121 static int ixgbevf_open(struct net_device *netdev) 3122 { 3123 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3124 struct ixgbe_hw *hw = &adapter->hw; 3125 int err; 3126 3127 /* A previous failure to open the device because of a lack of 3128 * available MSIX vector resources may have reset the number 3129 * of msix vectors variable to zero. The only way to recover 3130 * is to unload/reload the driver and hope that the system has 3131 * been able to recover some MSIX vector resources. 3132 */ 3133 if (!adapter->num_msix_vectors) 3134 return -ENOMEM; 3135 3136 if (hw->adapter_stopped) { 3137 ixgbevf_reset(adapter); 3138 /* if adapter is still stopped then PF isn't up and 3139 * the VF can't start. 3140 */ 3141 if (hw->adapter_stopped) { 3142 err = IXGBE_ERR_MBX; 3143 pr_err("Unable to start - perhaps the PF Driver isn't up yet\n"); 3144 goto err_setup_reset; 3145 } 3146 } 3147 3148 /* disallow open during test */ 3149 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 3150 return -EBUSY; 3151 3152 netif_carrier_off(netdev); 3153 3154 /* allocate transmit descriptors */ 3155 err = ixgbevf_setup_all_tx_resources(adapter); 3156 if (err) 3157 goto err_setup_tx; 3158 3159 /* allocate receive descriptors */ 3160 err = ixgbevf_setup_all_rx_resources(adapter); 3161 if (err) 3162 goto err_setup_rx; 3163 3164 ixgbevf_configure(adapter); 3165 3166 /* Map the Tx/Rx rings to the vectors we were allotted. 3167 * if request_irq will be called in this function map_rings 3168 * must be called *before* up_complete 3169 */ 3170 ixgbevf_map_rings_to_vectors(adapter); 3171 3172 err = ixgbevf_request_irq(adapter); 3173 if (err) 3174 goto err_req_irq; 3175 3176 ixgbevf_up_complete(adapter); 3177 3178 return 0; 3179 3180 err_req_irq: 3181 ixgbevf_down(adapter); 3182 err_setup_rx: 3183 ixgbevf_free_all_rx_resources(adapter); 3184 err_setup_tx: 3185 ixgbevf_free_all_tx_resources(adapter); 3186 ixgbevf_reset(adapter); 3187 3188 err_setup_reset: 3189 3190 return err; 3191 } 3192 3193 /** 3194 * ixgbevf_close - Disables a network interface 3195 * @netdev: network interface device structure 3196 * 3197 * Returns 0, this is not allowed to fail 3198 * 3199 * The close entry point is called when an interface is de-activated 3200 * by the OS. The hardware is still under the drivers control, but 3201 * needs to be disabled. A global MAC reset is issued to stop the 3202 * hardware, and all transmit and receive resources are freed. 3203 **/ 3204 static int ixgbevf_close(struct net_device *netdev) 3205 { 3206 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3207 3208 ixgbevf_down(adapter); 3209 ixgbevf_free_irq(adapter); 3210 3211 ixgbevf_free_all_tx_resources(adapter); 3212 ixgbevf_free_all_rx_resources(adapter); 3213 3214 return 0; 3215 } 3216 3217 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter) 3218 { 3219 struct net_device *dev = adapter->netdev; 3220 3221 if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED)) 3222 return; 3223 3224 adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED; 3225 3226 /* if interface is down do nothing */ 3227 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3228 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3229 return; 3230 3231 /* Hardware has to reinitialize queues and interrupts to 3232 * match packet buffer alignment. Unfortunately, the 3233 * hardware is not flexible enough to do this dynamically. 3234 */ 3235 if (netif_running(dev)) 3236 ixgbevf_close(dev); 3237 3238 ixgbevf_clear_interrupt_scheme(adapter); 3239 ixgbevf_init_interrupt_scheme(adapter); 3240 3241 if (netif_running(dev)) 3242 ixgbevf_open(dev); 3243 } 3244 3245 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, 3246 u32 vlan_macip_lens, u32 type_tucmd, 3247 u32 mss_l4len_idx) 3248 { 3249 struct ixgbe_adv_tx_context_desc *context_desc; 3250 u16 i = tx_ring->next_to_use; 3251 3252 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i); 3253 3254 i++; 3255 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 3256 3257 /* set bits to identify this as an advanced context descriptor */ 3258 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3259 3260 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 3261 context_desc->seqnum_seed = 0; 3262 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 3263 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 3264 } 3265 3266 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, 3267 struct ixgbevf_tx_buffer *first, 3268 u8 *hdr_len) 3269 { 3270 struct sk_buff *skb = first->skb; 3271 u32 vlan_macip_lens, type_tucmd; 3272 u32 mss_l4len_idx, l4len; 3273 int err; 3274 3275 if (skb->ip_summed != CHECKSUM_PARTIAL) 3276 return 0; 3277 3278 if (!skb_is_gso(skb)) 3279 return 0; 3280 3281 err = skb_cow_head(skb, 0); 3282 if (err < 0) 3283 return err; 3284 3285 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 3286 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3287 3288 if (first->protocol == htons(ETH_P_IP)) { 3289 struct iphdr *iph = ip_hdr(skb); 3290 3291 iph->tot_len = 0; 3292 iph->check = 0; 3293 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 3294 iph->daddr, 0, 3295 IPPROTO_TCP, 3296 0); 3297 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3298 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3299 IXGBE_TX_FLAGS_CSUM | 3300 IXGBE_TX_FLAGS_IPV4; 3301 } else if (skb_is_gso_v6(skb)) { 3302 ipv6_hdr(skb)->payload_len = 0; 3303 tcp_hdr(skb)->check = 3304 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 3305 &ipv6_hdr(skb)->daddr, 3306 0, IPPROTO_TCP, 0); 3307 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3308 IXGBE_TX_FLAGS_CSUM; 3309 } 3310 3311 /* compute header lengths */ 3312 l4len = tcp_hdrlen(skb); 3313 *hdr_len += l4len; 3314 *hdr_len = skb_transport_offset(skb) + l4len; 3315 3316 /* update GSO size and bytecount with header size */ 3317 first->gso_segs = skb_shinfo(skb)->gso_segs; 3318 first->bytecount += (first->gso_segs - 1) * *hdr_len; 3319 3320 /* mss_l4len_id: use 1 as index for TSO */ 3321 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; 3322 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 3323 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT; 3324 3325 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 3326 vlan_macip_lens = skb_network_header_len(skb); 3327 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3328 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3329 3330 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3331 type_tucmd, mss_l4len_idx); 3332 3333 return 1; 3334 } 3335 3336 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, 3337 struct ixgbevf_tx_buffer *first) 3338 { 3339 struct sk_buff *skb = first->skb; 3340 u32 vlan_macip_lens = 0; 3341 u32 mss_l4len_idx = 0; 3342 u32 type_tucmd = 0; 3343 3344 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3345 u8 l4_hdr = 0; 3346 3347 switch (first->protocol) { 3348 case htons(ETH_P_IP): 3349 vlan_macip_lens |= skb_network_header_len(skb); 3350 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3351 l4_hdr = ip_hdr(skb)->protocol; 3352 break; 3353 case htons(ETH_P_IPV6): 3354 vlan_macip_lens |= skb_network_header_len(skb); 3355 l4_hdr = ipv6_hdr(skb)->nexthdr; 3356 break; 3357 default: 3358 if (unlikely(net_ratelimit())) { 3359 dev_warn(tx_ring->dev, 3360 "partial checksum but proto=%x!\n", 3361 first->protocol); 3362 } 3363 break; 3364 } 3365 3366 switch (l4_hdr) { 3367 case IPPROTO_TCP: 3368 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3369 mss_l4len_idx = tcp_hdrlen(skb) << 3370 IXGBE_ADVTXD_L4LEN_SHIFT; 3371 break; 3372 case IPPROTO_SCTP: 3373 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 3374 mss_l4len_idx = sizeof(struct sctphdr) << 3375 IXGBE_ADVTXD_L4LEN_SHIFT; 3376 break; 3377 case IPPROTO_UDP: 3378 mss_l4len_idx = sizeof(struct udphdr) << 3379 IXGBE_ADVTXD_L4LEN_SHIFT; 3380 break; 3381 default: 3382 if (unlikely(net_ratelimit())) { 3383 dev_warn(tx_ring->dev, 3384 "partial checksum but l4 proto=%x!\n", 3385 l4_hdr); 3386 } 3387 break; 3388 } 3389 3390 /* update TX checksum flag */ 3391 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 3392 } 3393 3394 /* vlan_macip_lens: MACLEN, VLAN tag */ 3395 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3396 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3397 3398 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3399 type_tucmd, mss_l4len_idx); 3400 } 3401 3402 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags) 3403 { 3404 /* set type for advanced descriptor with frame checksum insertion */ 3405 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | 3406 IXGBE_ADVTXD_DCMD_IFCS | 3407 IXGBE_ADVTXD_DCMD_DEXT); 3408 3409 /* set HW VLAN bit if VLAN is present */ 3410 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 3411 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); 3412 3413 /* set segmentation enable bits for TSO/FSO */ 3414 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3415 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); 3416 3417 return cmd_type; 3418 } 3419 3420 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 3421 u32 tx_flags, unsigned int paylen) 3422 { 3423 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT); 3424 3425 /* enable L4 checksum for TSO and TX checksum offload */ 3426 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 3427 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM); 3428 3429 /* enble IPv4 checksum for TSO */ 3430 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 3431 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM); 3432 3433 /* use index 1 context for TSO/FSO/FCOE */ 3434 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3435 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT); 3436 3437 /* Check Context must be set if Tx switch is enabled, which it 3438 * always is for case where virtual functions are running 3439 */ 3440 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC); 3441 3442 tx_desc->read.olinfo_status = olinfo_status; 3443 } 3444 3445 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, 3446 struct ixgbevf_tx_buffer *first, 3447 const u8 hdr_len) 3448 { 3449 dma_addr_t dma; 3450 struct sk_buff *skb = first->skb; 3451 struct ixgbevf_tx_buffer *tx_buffer; 3452 union ixgbe_adv_tx_desc *tx_desc; 3453 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 3454 unsigned int data_len = skb->data_len; 3455 unsigned int size = skb_headlen(skb); 3456 unsigned int paylen = skb->len - hdr_len; 3457 u32 tx_flags = first->tx_flags; 3458 __le32 cmd_type; 3459 u16 i = tx_ring->next_to_use; 3460 3461 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 3462 3463 ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen); 3464 cmd_type = ixgbevf_tx_cmd_type(tx_flags); 3465 3466 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 3467 if (dma_mapping_error(tx_ring->dev, dma)) 3468 goto dma_error; 3469 3470 /* record length, and DMA address */ 3471 dma_unmap_len_set(first, len, size); 3472 dma_unmap_addr_set(first, dma, dma); 3473 3474 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3475 3476 for (;;) { 3477 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 3478 tx_desc->read.cmd_type_len = 3479 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); 3480 3481 i++; 3482 tx_desc++; 3483 if (i == tx_ring->count) { 3484 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3485 i = 0; 3486 } 3487 3488 dma += IXGBE_MAX_DATA_PER_TXD; 3489 size -= IXGBE_MAX_DATA_PER_TXD; 3490 3491 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3492 tx_desc->read.olinfo_status = 0; 3493 } 3494 3495 if (likely(!data_len)) 3496 break; 3497 3498 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); 3499 3500 i++; 3501 tx_desc++; 3502 if (i == tx_ring->count) { 3503 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3504 i = 0; 3505 } 3506 3507 size = skb_frag_size(frag); 3508 data_len -= size; 3509 3510 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 3511 DMA_TO_DEVICE); 3512 if (dma_mapping_error(tx_ring->dev, dma)) 3513 goto dma_error; 3514 3515 tx_buffer = &tx_ring->tx_buffer_info[i]; 3516 dma_unmap_len_set(tx_buffer, len, size); 3517 dma_unmap_addr_set(tx_buffer, dma, dma); 3518 3519 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3520 tx_desc->read.olinfo_status = 0; 3521 3522 frag++; 3523 } 3524 3525 /* write last descriptor with RS and EOP bits */ 3526 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); 3527 tx_desc->read.cmd_type_len = cmd_type; 3528 3529 /* set the timestamp */ 3530 first->time_stamp = jiffies; 3531 3532 /* Force memory writes to complete before letting h/w know there 3533 * are new descriptors to fetch. (Only applicable for weak-ordered 3534 * memory model archs, such as IA-64). 3535 * 3536 * We also need this memory barrier (wmb) to make certain all of the 3537 * status bits have been updated before next_to_watch is written. 3538 */ 3539 wmb(); 3540 3541 /* set next_to_watch value indicating a packet is present */ 3542 first->next_to_watch = tx_desc; 3543 3544 i++; 3545 if (i == tx_ring->count) 3546 i = 0; 3547 3548 tx_ring->next_to_use = i; 3549 3550 /* notify HW of packet */ 3551 ixgbevf_write_tail(tx_ring, i); 3552 3553 return; 3554 dma_error: 3555 dev_err(tx_ring->dev, "TX DMA map failed\n"); 3556 3557 /* clear dma mappings for failed tx_buffer_info map */ 3558 for (;;) { 3559 tx_buffer = &tx_ring->tx_buffer_info[i]; 3560 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer); 3561 if (tx_buffer == first) 3562 break; 3563 if (i == 0) 3564 i = tx_ring->count; 3565 i--; 3566 } 3567 3568 tx_ring->next_to_use = i; 3569 } 3570 3571 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 3572 { 3573 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 3574 /* Herbert's original patch had: 3575 * smp_mb__after_netif_stop_queue(); 3576 * but since that doesn't exist yet, just open code it. 3577 */ 3578 smp_mb(); 3579 3580 /* We need to check again in a case another CPU has just 3581 * made room available. 3582 */ 3583 if (likely(ixgbevf_desc_unused(tx_ring) < size)) 3584 return -EBUSY; 3585 3586 /* A reprieve! - use start_queue because it doesn't call schedule */ 3587 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 3588 ++tx_ring->tx_stats.restart_queue; 3589 3590 return 0; 3591 } 3592 3593 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 3594 { 3595 if (likely(ixgbevf_desc_unused(tx_ring) >= size)) 3596 return 0; 3597 return __ixgbevf_maybe_stop_tx(tx_ring, size); 3598 } 3599 3600 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 3601 { 3602 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3603 struct ixgbevf_tx_buffer *first; 3604 struct ixgbevf_ring *tx_ring; 3605 int tso; 3606 u32 tx_flags = 0; 3607 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 3608 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3609 unsigned short f; 3610 #endif 3611 u8 hdr_len = 0; 3612 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL); 3613 3614 if (!dst_mac || is_link_local_ether_addr(dst_mac)) { 3615 dev_kfree_skb_any(skb); 3616 return NETDEV_TX_OK; 3617 } 3618 3619 tx_ring = adapter->tx_ring[skb->queue_mapping]; 3620 3621 /* need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 3622 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 3623 * + 2 desc gap to keep tail from touching head, 3624 * + 1 desc for context descriptor, 3625 * otherwise try next time 3626 */ 3627 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3628 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 3629 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 3630 #else 3631 count += skb_shinfo(skb)->nr_frags; 3632 #endif 3633 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { 3634 tx_ring->tx_stats.tx_busy++; 3635 return NETDEV_TX_BUSY; 3636 } 3637 3638 /* record the location of the first descriptor for this packet */ 3639 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 3640 first->skb = skb; 3641 first->bytecount = skb->len; 3642 first->gso_segs = 1; 3643 3644 if (skb_vlan_tag_present(skb)) { 3645 tx_flags |= skb_vlan_tag_get(skb); 3646 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 3647 tx_flags |= IXGBE_TX_FLAGS_VLAN; 3648 } 3649 3650 /* record initial flags and protocol */ 3651 first->tx_flags = tx_flags; 3652 first->protocol = vlan_get_protocol(skb); 3653 3654 tso = ixgbevf_tso(tx_ring, first, &hdr_len); 3655 if (tso < 0) 3656 goto out_drop; 3657 else if (!tso) 3658 ixgbevf_tx_csum(tx_ring, first); 3659 3660 ixgbevf_tx_map(tx_ring, first, hdr_len); 3661 3662 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); 3663 3664 return NETDEV_TX_OK; 3665 3666 out_drop: 3667 dev_kfree_skb_any(first->skb); 3668 first->skb = NULL; 3669 3670 return NETDEV_TX_OK; 3671 } 3672 3673 /** 3674 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 3675 * @netdev: network interface device structure 3676 * @p: pointer to an address structure 3677 * 3678 * Returns 0 on success, negative on failure 3679 **/ 3680 static int ixgbevf_set_mac(struct net_device *netdev, void *p) 3681 { 3682 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3683 struct ixgbe_hw *hw = &adapter->hw; 3684 struct sockaddr *addr = p; 3685 3686 if (!is_valid_ether_addr(addr->sa_data)) 3687 return -EADDRNOTAVAIL; 3688 3689 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 3690 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 3691 3692 spin_lock_bh(&adapter->mbx_lock); 3693 3694 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 3695 3696 spin_unlock_bh(&adapter->mbx_lock); 3697 3698 return 0; 3699 } 3700 3701 /** 3702 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 3703 * @netdev: network interface device structure 3704 * @new_mtu: new value for maximum frame size 3705 * 3706 * Returns 0 on success, negative on failure 3707 **/ 3708 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 3709 { 3710 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3711 struct ixgbe_hw *hw = &adapter->hw; 3712 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 3713 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; 3714 3715 switch (adapter->hw.api_version) { 3716 case ixgbe_mbox_api_11: 3717 case ixgbe_mbox_api_12: 3718 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3719 break; 3720 default: 3721 if (adapter->hw.mac.type != ixgbe_mac_82599_vf) 3722 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3723 break; 3724 } 3725 3726 /* MTU < 68 is an error and causes problems on some kernels */ 3727 if ((new_mtu < 68) || (max_frame > max_possible_frame)) 3728 return -EINVAL; 3729 3730 hw_dbg(hw, "changing MTU from %d to %d\n", 3731 netdev->mtu, new_mtu); 3732 /* must set new MTU before calling down or up */ 3733 netdev->mtu = new_mtu; 3734 3735 /* notify the PF of our intent to use this size of frame */ 3736 ixgbevf_rlpml_set_vf(hw, max_frame); 3737 3738 return 0; 3739 } 3740 3741 #ifdef CONFIG_NET_POLL_CONTROLLER 3742 /* Polling 'interrupt' - used by things like netconsole to send skbs 3743 * without having to re-enable interrupts. It's not called while 3744 * the interrupt routine is executing. 3745 */ 3746 static void ixgbevf_netpoll(struct net_device *netdev) 3747 { 3748 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3749 int i; 3750 3751 /* if interface is down do nothing */ 3752 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 3753 return; 3754 for (i = 0; i < adapter->num_rx_queues; i++) 3755 ixgbevf_msix_clean_rings(0, adapter->q_vector[i]); 3756 } 3757 #endif /* CONFIG_NET_POLL_CONTROLLER */ 3758 3759 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) 3760 { 3761 struct net_device *netdev = pci_get_drvdata(pdev); 3762 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3763 #ifdef CONFIG_PM 3764 int retval = 0; 3765 #endif 3766 3767 netif_device_detach(netdev); 3768 3769 if (netif_running(netdev)) { 3770 rtnl_lock(); 3771 ixgbevf_down(adapter); 3772 ixgbevf_free_irq(adapter); 3773 ixgbevf_free_all_tx_resources(adapter); 3774 ixgbevf_free_all_rx_resources(adapter); 3775 rtnl_unlock(); 3776 } 3777 3778 ixgbevf_clear_interrupt_scheme(adapter); 3779 3780 #ifdef CONFIG_PM 3781 retval = pci_save_state(pdev); 3782 if (retval) 3783 return retval; 3784 3785 #endif 3786 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 3787 pci_disable_device(pdev); 3788 3789 return 0; 3790 } 3791 3792 #ifdef CONFIG_PM 3793 static int ixgbevf_resume(struct pci_dev *pdev) 3794 { 3795 struct net_device *netdev = pci_get_drvdata(pdev); 3796 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3797 u32 err; 3798 3799 pci_restore_state(pdev); 3800 /* pci_restore_state clears dev->state_saved so call 3801 * pci_save_state to restore it. 3802 */ 3803 pci_save_state(pdev); 3804 3805 err = pci_enable_device_mem(pdev); 3806 if (err) { 3807 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 3808 return err; 3809 } 3810 smp_mb__before_atomic(); 3811 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 3812 pci_set_master(pdev); 3813 3814 ixgbevf_reset(adapter); 3815 3816 rtnl_lock(); 3817 err = ixgbevf_init_interrupt_scheme(adapter); 3818 rtnl_unlock(); 3819 if (err) { 3820 dev_err(&pdev->dev, "Cannot initialize interrupts\n"); 3821 return err; 3822 } 3823 3824 if (netif_running(netdev)) { 3825 err = ixgbevf_open(netdev); 3826 if (err) 3827 return err; 3828 } 3829 3830 netif_device_attach(netdev); 3831 3832 return err; 3833 } 3834 3835 #endif /* CONFIG_PM */ 3836 static void ixgbevf_shutdown(struct pci_dev *pdev) 3837 { 3838 ixgbevf_suspend(pdev, PMSG_SUSPEND); 3839 } 3840 3841 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, 3842 struct rtnl_link_stats64 *stats) 3843 { 3844 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3845 unsigned int start; 3846 u64 bytes, packets; 3847 const struct ixgbevf_ring *ring; 3848 int i; 3849 3850 ixgbevf_update_stats(adapter); 3851 3852 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3853 3854 for (i = 0; i < adapter->num_rx_queues; i++) { 3855 ring = adapter->rx_ring[i]; 3856 do { 3857 start = u64_stats_fetch_begin_irq(&ring->syncp); 3858 bytes = ring->stats.bytes; 3859 packets = ring->stats.packets; 3860 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 3861 stats->rx_bytes += bytes; 3862 stats->rx_packets += packets; 3863 } 3864 3865 for (i = 0; i < adapter->num_tx_queues; i++) { 3866 ring = adapter->tx_ring[i]; 3867 do { 3868 start = u64_stats_fetch_begin_irq(&ring->syncp); 3869 bytes = ring->stats.bytes; 3870 packets = ring->stats.packets; 3871 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 3872 stats->tx_bytes += bytes; 3873 stats->tx_packets += packets; 3874 } 3875 3876 return stats; 3877 } 3878 3879 static const struct net_device_ops ixgbevf_netdev_ops = { 3880 .ndo_open = ixgbevf_open, 3881 .ndo_stop = ixgbevf_close, 3882 .ndo_start_xmit = ixgbevf_xmit_frame, 3883 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 3884 .ndo_get_stats64 = ixgbevf_get_stats, 3885 .ndo_validate_addr = eth_validate_addr, 3886 .ndo_set_mac_address = ixgbevf_set_mac, 3887 .ndo_change_mtu = ixgbevf_change_mtu, 3888 .ndo_tx_timeout = ixgbevf_tx_timeout, 3889 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 3890 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 3891 #ifdef CONFIG_NET_RX_BUSY_POLL 3892 .ndo_busy_poll = ixgbevf_busy_poll_recv, 3893 #endif 3894 #ifdef CONFIG_NET_POLL_CONTROLLER 3895 .ndo_poll_controller = ixgbevf_netpoll, 3896 #endif 3897 }; 3898 3899 static void ixgbevf_assign_netdev_ops(struct net_device *dev) 3900 { 3901 dev->netdev_ops = &ixgbevf_netdev_ops; 3902 ixgbevf_set_ethtool_ops(dev); 3903 dev->watchdog_timeo = 5 * HZ; 3904 } 3905 3906 /** 3907 * ixgbevf_probe - Device Initialization Routine 3908 * @pdev: PCI device information struct 3909 * @ent: entry in ixgbevf_pci_tbl 3910 * 3911 * Returns 0 on success, negative on failure 3912 * 3913 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 3914 * The OS initialization, configuring of the adapter private structure, 3915 * and a hardware reset occur. 3916 **/ 3917 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3918 { 3919 struct net_device *netdev; 3920 struct ixgbevf_adapter *adapter = NULL; 3921 struct ixgbe_hw *hw = NULL; 3922 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 3923 int err, pci_using_dac; 3924 bool disable_dev = false; 3925 3926 err = pci_enable_device(pdev); 3927 if (err) 3928 return err; 3929 3930 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 3931 pci_using_dac = 1; 3932 } else { 3933 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 3934 if (err) { 3935 dev_err(&pdev->dev, "No usable DMA configuration, aborting\n"); 3936 goto err_dma; 3937 } 3938 pci_using_dac = 0; 3939 } 3940 3941 err = pci_request_regions(pdev, ixgbevf_driver_name); 3942 if (err) { 3943 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 3944 goto err_pci_reg; 3945 } 3946 3947 pci_set_master(pdev); 3948 3949 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 3950 MAX_TX_QUEUES); 3951 if (!netdev) { 3952 err = -ENOMEM; 3953 goto err_alloc_etherdev; 3954 } 3955 3956 SET_NETDEV_DEV(netdev, &pdev->dev); 3957 3958 adapter = netdev_priv(netdev); 3959 3960 adapter->netdev = netdev; 3961 adapter->pdev = pdev; 3962 hw = &adapter->hw; 3963 hw->back = adapter; 3964 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 3965 3966 /* call save state here in standalone driver because it relies on 3967 * adapter struct to exist, and needs to call netdev_priv 3968 */ 3969 pci_save_state(pdev); 3970 3971 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3972 pci_resource_len(pdev, 0)); 3973 adapter->io_addr = hw->hw_addr; 3974 if (!hw->hw_addr) { 3975 err = -EIO; 3976 goto err_ioremap; 3977 } 3978 3979 ixgbevf_assign_netdev_ops(netdev); 3980 3981 /* Setup HW API */ 3982 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 3983 hw->mac.type = ii->mac; 3984 3985 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 3986 sizeof(struct ixgbe_mbx_operations)); 3987 3988 /* setup the private structure */ 3989 err = ixgbevf_sw_init(adapter); 3990 if (err) 3991 goto err_sw_init; 3992 3993 /* The HW MAC address was set and/or determined in sw_init */ 3994 if (!is_valid_ether_addr(netdev->dev_addr)) { 3995 pr_err("invalid MAC address\n"); 3996 err = -EIO; 3997 goto err_sw_init; 3998 } 3999 4000 netdev->hw_features = NETIF_F_SG | 4001 NETIF_F_IP_CSUM | 4002 NETIF_F_IPV6_CSUM | 4003 NETIF_F_TSO | 4004 NETIF_F_TSO6 | 4005 NETIF_F_RXCSUM; 4006 4007 netdev->features = netdev->hw_features | 4008 NETIF_F_HW_VLAN_CTAG_TX | 4009 NETIF_F_HW_VLAN_CTAG_RX | 4010 NETIF_F_HW_VLAN_CTAG_FILTER; 4011 4012 netdev->vlan_features |= NETIF_F_TSO | 4013 NETIF_F_TSO6 | 4014 NETIF_F_IP_CSUM | 4015 NETIF_F_IPV6_CSUM | 4016 NETIF_F_SG; 4017 4018 if (pci_using_dac) 4019 netdev->features |= NETIF_F_HIGHDMA; 4020 4021 netdev->priv_flags |= IFF_UNICAST_FLT; 4022 4023 if (IXGBE_REMOVED(hw->hw_addr)) { 4024 err = -EIO; 4025 goto err_sw_init; 4026 } 4027 4028 setup_timer(&adapter->service_timer, &ixgbevf_service_timer, 4029 (unsigned long)adapter); 4030 4031 INIT_WORK(&adapter->service_task, ixgbevf_service_task); 4032 set_bit(__IXGBEVF_SERVICE_INITED, &adapter->state); 4033 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 4034 4035 err = ixgbevf_init_interrupt_scheme(adapter); 4036 if (err) 4037 goto err_sw_init; 4038 4039 strcpy(netdev->name, "eth%d"); 4040 4041 err = register_netdev(netdev); 4042 if (err) 4043 goto err_register; 4044 4045 pci_set_drvdata(pdev, netdev); 4046 netif_carrier_off(netdev); 4047 4048 ixgbevf_init_last_counter_stats(adapter); 4049 4050 /* print the VF info */ 4051 dev_info(&pdev->dev, "%pM\n", netdev->dev_addr); 4052 dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type); 4053 4054 switch (hw->mac.type) { 4055 case ixgbe_mac_X550_vf: 4056 dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n"); 4057 break; 4058 case ixgbe_mac_X540_vf: 4059 dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n"); 4060 break; 4061 case ixgbe_mac_82599_vf: 4062 default: 4063 dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n"); 4064 break; 4065 } 4066 4067 return 0; 4068 4069 err_register: 4070 ixgbevf_clear_interrupt_scheme(adapter); 4071 err_sw_init: 4072 ixgbevf_reset_interrupt_capability(adapter); 4073 iounmap(adapter->io_addr); 4074 err_ioremap: 4075 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4076 free_netdev(netdev); 4077 err_alloc_etherdev: 4078 pci_release_regions(pdev); 4079 err_pci_reg: 4080 err_dma: 4081 if (!adapter || disable_dev) 4082 pci_disable_device(pdev); 4083 return err; 4084 } 4085 4086 /** 4087 * ixgbevf_remove - Device Removal Routine 4088 * @pdev: PCI device information struct 4089 * 4090 * ixgbevf_remove is called by the PCI subsystem to alert the driver 4091 * that it should release a PCI device. The could be caused by a 4092 * Hot-Plug event, or because the driver is going to be removed from 4093 * memory. 4094 **/ 4095 static void ixgbevf_remove(struct pci_dev *pdev) 4096 { 4097 struct net_device *netdev = pci_get_drvdata(pdev); 4098 struct ixgbevf_adapter *adapter; 4099 bool disable_dev; 4100 4101 if (!netdev) 4102 return; 4103 4104 adapter = netdev_priv(netdev); 4105 4106 set_bit(__IXGBEVF_REMOVING, &adapter->state); 4107 cancel_work_sync(&adapter->service_task); 4108 4109 if (netdev->reg_state == NETREG_REGISTERED) 4110 unregister_netdev(netdev); 4111 4112 ixgbevf_clear_interrupt_scheme(adapter); 4113 ixgbevf_reset_interrupt_capability(adapter); 4114 4115 iounmap(adapter->io_addr); 4116 pci_release_regions(pdev); 4117 4118 hw_dbg(&adapter->hw, "Remove complete\n"); 4119 4120 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4121 free_netdev(netdev); 4122 4123 if (disable_dev) 4124 pci_disable_device(pdev); 4125 } 4126 4127 /** 4128 * ixgbevf_io_error_detected - called when PCI error is detected 4129 * @pdev: Pointer to PCI device 4130 * @state: The current pci connection state 4131 * 4132 * This function is called after a PCI bus error affecting 4133 * this device has been detected. 4134 **/ 4135 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, 4136 pci_channel_state_t state) 4137 { 4138 struct net_device *netdev = pci_get_drvdata(pdev); 4139 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4140 4141 if (!test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 4142 return PCI_ERS_RESULT_DISCONNECT; 4143 4144 rtnl_lock(); 4145 netif_device_detach(netdev); 4146 4147 if (state == pci_channel_io_perm_failure) { 4148 rtnl_unlock(); 4149 return PCI_ERS_RESULT_DISCONNECT; 4150 } 4151 4152 if (netif_running(netdev)) 4153 ixgbevf_down(adapter); 4154 4155 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 4156 pci_disable_device(pdev); 4157 rtnl_unlock(); 4158 4159 /* Request a slot slot reset. */ 4160 return PCI_ERS_RESULT_NEED_RESET; 4161 } 4162 4163 /** 4164 * ixgbevf_io_slot_reset - called after the pci bus has been reset. 4165 * @pdev: Pointer to PCI device 4166 * 4167 * Restart the card from scratch, as if from a cold-boot. Implementation 4168 * resembles the first-half of the ixgbevf_resume routine. 4169 **/ 4170 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) 4171 { 4172 struct net_device *netdev = pci_get_drvdata(pdev); 4173 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4174 4175 if (pci_enable_device_mem(pdev)) { 4176 dev_err(&pdev->dev, 4177 "Cannot re-enable PCI device after reset.\n"); 4178 return PCI_ERS_RESULT_DISCONNECT; 4179 } 4180 4181 smp_mb__before_atomic(); 4182 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 4183 pci_set_master(pdev); 4184 4185 ixgbevf_reset(adapter); 4186 4187 return PCI_ERS_RESULT_RECOVERED; 4188 } 4189 4190 /** 4191 * ixgbevf_io_resume - called when traffic can start flowing again. 4192 * @pdev: Pointer to PCI device 4193 * 4194 * This callback is called when the error recovery driver tells us that 4195 * its OK to resume normal operation. Implementation resembles the 4196 * second-half of the ixgbevf_resume routine. 4197 **/ 4198 static void ixgbevf_io_resume(struct pci_dev *pdev) 4199 { 4200 struct net_device *netdev = pci_get_drvdata(pdev); 4201 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4202 4203 if (netif_running(netdev)) 4204 ixgbevf_up(adapter); 4205 4206 netif_device_attach(netdev); 4207 } 4208 4209 /* PCI Error Recovery (ERS) */ 4210 static const struct pci_error_handlers ixgbevf_err_handler = { 4211 .error_detected = ixgbevf_io_error_detected, 4212 .slot_reset = ixgbevf_io_slot_reset, 4213 .resume = ixgbevf_io_resume, 4214 }; 4215 4216 static struct pci_driver ixgbevf_driver = { 4217 .name = ixgbevf_driver_name, 4218 .id_table = ixgbevf_pci_tbl, 4219 .probe = ixgbevf_probe, 4220 .remove = ixgbevf_remove, 4221 #ifdef CONFIG_PM 4222 /* Power Management Hooks */ 4223 .suspend = ixgbevf_suspend, 4224 .resume = ixgbevf_resume, 4225 #endif 4226 .shutdown = ixgbevf_shutdown, 4227 .err_handler = &ixgbevf_err_handler 4228 }; 4229 4230 /** 4231 * ixgbevf_init_module - Driver Registration Routine 4232 * 4233 * ixgbevf_init_module is the first routine called when the driver is 4234 * loaded. All it does is register with the PCI subsystem. 4235 **/ 4236 static int __init ixgbevf_init_module(void) 4237 { 4238 int ret; 4239 4240 pr_info("%s - version %s\n", ixgbevf_driver_string, 4241 ixgbevf_driver_version); 4242 4243 pr_info("%s\n", ixgbevf_copyright); 4244 4245 ret = pci_register_driver(&ixgbevf_driver); 4246 return ret; 4247 } 4248 4249 module_init(ixgbevf_init_module); 4250 4251 /** 4252 * ixgbevf_exit_module - Driver Exit Cleanup Routine 4253 * 4254 * ixgbevf_exit_module is called just before the driver is removed 4255 * from memory. 4256 **/ 4257 static void __exit ixgbevf_exit_module(void) 4258 { 4259 pci_unregister_driver(&ixgbevf_driver); 4260 } 4261 4262 #ifdef DEBUG 4263 /** 4264 * ixgbevf_get_hw_dev_name - return device name string 4265 * used by hardware layer to print debugging information 4266 **/ 4267 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 4268 { 4269 struct ixgbevf_adapter *adapter = hw->back; 4270 4271 return adapter->netdev->name; 4272 } 4273 4274 #endif 4275 module_exit(ixgbevf_exit_module); 4276 4277 /* ixgbevf_main.c */ 4278