1 /******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2015 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, see <http://www.gnu.org/licenses/>. 17 18 The full GNU General Public License is included in this distribution in 19 the file called "COPYING". 20 21 Contact Information: 22 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 25 *******************************************************************************/ 26 27 /****************************************************************************** 28 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 29 ******************************************************************************/ 30 31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 32 33 #include <linux/types.h> 34 #include <linux/bitops.h> 35 #include <linux/module.h> 36 #include <linux/pci.h> 37 #include <linux/netdevice.h> 38 #include <linux/vmalloc.h> 39 #include <linux/string.h> 40 #include <linux/in.h> 41 #include <linux/ip.h> 42 #include <linux/tcp.h> 43 #include <linux/sctp.h> 44 #include <linux/ipv6.h> 45 #include <linux/slab.h> 46 #include <net/checksum.h> 47 #include <net/ip6_checksum.h> 48 #include <linux/ethtool.h> 49 #include <linux/if.h> 50 #include <linux/if_vlan.h> 51 #include <linux/prefetch.h> 52 53 #include "ixgbevf.h" 54 55 const char ixgbevf_driver_name[] = "ixgbevf"; 56 static const char ixgbevf_driver_string[] = 57 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 58 59 #define DRV_VERSION "2.12.1-k" 60 const char ixgbevf_driver_version[] = DRV_VERSION; 61 static char ixgbevf_copyright[] = 62 "Copyright (c) 2009 - 2012 Intel Corporation."; 63 64 static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 65 [board_82599_vf] = &ixgbevf_82599_vf_info, 66 [board_X540_vf] = &ixgbevf_X540_vf_info, 67 [board_X550_vf] = &ixgbevf_X550_vf_info, 68 [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info, 69 }; 70 71 /* ixgbevf_pci_tbl - PCI Device ID Table 72 * 73 * Wildcard entries (PCI_ANY_ID) should come last 74 * Last entry must be all 0s 75 * 76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 77 * Class, Class Mask, private data (not used) } 78 */ 79 static const struct pci_device_id ixgbevf_pci_tbl[] = { 80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf }, 81 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf }, 82 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf }, 83 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf }, 84 /* required last entry */ 85 {0, } 86 }; 87 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 88 89 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 90 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); 91 MODULE_LICENSE("GPL"); 92 MODULE_VERSION(DRV_VERSION); 93 94 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 95 static int debug = -1; 96 module_param(debug, int, 0); 97 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 98 99 static void ixgbevf_service_event_schedule(struct ixgbevf_adapter *adapter) 100 { 101 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 102 !test_bit(__IXGBEVF_REMOVING, &adapter->state) && 103 !test_and_set_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)) 104 schedule_work(&adapter->service_task); 105 } 106 107 static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter) 108 { 109 BUG_ON(!test_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)); 110 111 /* flush memory to make sure state is correct before next watchdog */ 112 smp_mb__before_atomic(); 113 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 114 } 115 116 /* forward decls */ 117 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); 118 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); 119 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); 120 121 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) 122 { 123 struct ixgbevf_adapter *adapter = hw->back; 124 125 if (!hw->hw_addr) 126 return; 127 hw->hw_addr = NULL; 128 dev_err(&adapter->pdev->dev, "Adapter removed\n"); 129 if (test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 130 ixgbevf_service_event_schedule(adapter); 131 } 132 133 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg) 134 { 135 u32 value; 136 137 /* The following check not only optimizes a bit by not 138 * performing a read on the status register when the 139 * register just read was a status register read that 140 * returned IXGBE_FAILED_READ_REG. It also blocks any 141 * potential recursion. 142 */ 143 if (reg == IXGBE_VFSTATUS) { 144 ixgbevf_remove_adapter(hw); 145 return; 146 } 147 value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS); 148 if (value == IXGBE_FAILED_READ_REG) 149 ixgbevf_remove_adapter(hw); 150 } 151 152 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg) 153 { 154 u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr); 155 u32 value; 156 157 if (IXGBE_REMOVED(reg_addr)) 158 return IXGBE_FAILED_READ_REG; 159 value = readl(reg_addr + reg); 160 if (unlikely(value == IXGBE_FAILED_READ_REG)) 161 ixgbevf_check_remove(hw, reg); 162 return value; 163 } 164 165 /** 166 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 167 * @adapter: pointer to adapter struct 168 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 169 * @queue: queue to map the corresponding interrupt to 170 * @msix_vector: the vector to map to the corresponding queue 171 **/ 172 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 173 u8 queue, u8 msix_vector) 174 { 175 u32 ivar, index; 176 struct ixgbe_hw *hw = &adapter->hw; 177 178 if (direction == -1) { 179 /* other causes */ 180 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 181 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 182 ivar &= ~0xFF; 183 ivar |= msix_vector; 184 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 185 } else { 186 /* Tx or Rx causes */ 187 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 188 index = ((16 * (queue & 1)) + (8 * direction)); 189 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 190 ivar &= ~(0xFF << index); 191 ivar |= (msix_vector << index); 192 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 193 } 194 } 195 196 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring, 197 struct ixgbevf_tx_buffer *tx_buffer) 198 { 199 if (tx_buffer->skb) { 200 dev_kfree_skb_any(tx_buffer->skb); 201 if (dma_unmap_len(tx_buffer, len)) 202 dma_unmap_single(tx_ring->dev, 203 dma_unmap_addr(tx_buffer, dma), 204 dma_unmap_len(tx_buffer, len), 205 DMA_TO_DEVICE); 206 } else if (dma_unmap_len(tx_buffer, len)) { 207 dma_unmap_page(tx_ring->dev, 208 dma_unmap_addr(tx_buffer, dma), 209 dma_unmap_len(tx_buffer, len), 210 DMA_TO_DEVICE); 211 } 212 tx_buffer->next_to_watch = NULL; 213 tx_buffer->skb = NULL; 214 dma_unmap_len_set(tx_buffer, len, 0); 215 /* tx_buffer must be completely set up in the transmit path */ 216 } 217 218 static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring) 219 { 220 return ring->stats.packets; 221 } 222 223 static u32 ixgbevf_get_tx_pending(struct ixgbevf_ring *ring) 224 { 225 struct ixgbevf_adapter *adapter = netdev_priv(ring->netdev); 226 struct ixgbe_hw *hw = &adapter->hw; 227 228 u32 head = IXGBE_READ_REG(hw, IXGBE_VFTDH(ring->reg_idx)); 229 u32 tail = IXGBE_READ_REG(hw, IXGBE_VFTDT(ring->reg_idx)); 230 231 if (head != tail) 232 return (head < tail) ? 233 tail - head : (tail + ring->count - head); 234 235 return 0; 236 } 237 238 static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring) 239 { 240 u32 tx_done = ixgbevf_get_tx_completed(tx_ring); 241 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 242 u32 tx_pending = ixgbevf_get_tx_pending(tx_ring); 243 244 clear_check_for_tx_hang(tx_ring); 245 246 /* Check for a hung queue, but be thorough. This verifies 247 * that a transmit has been completed since the previous 248 * check AND there is at least one packet pending. The 249 * ARMED bit is set to indicate a potential hang. 250 */ 251 if ((tx_done_old == tx_done) && tx_pending) { 252 /* make sure it is true for two checks in a row */ 253 return test_and_set_bit(__IXGBEVF_HANG_CHECK_ARMED, 254 &tx_ring->state); 255 } 256 /* reset the countdown */ 257 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &tx_ring->state); 258 259 /* update completed stats and continue */ 260 tx_ring->tx_stats.tx_done_old = tx_done; 261 262 return false; 263 } 264 265 static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter) 266 { 267 /* Do the reset outside of interrupt context */ 268 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 269 adapter->flags |= IXGBEVF_FLAG_RESET_REQUESTED; 270 ixgbevf_service_event_schedule(adapter); 271 } 272 } 273 274 /** 275 * ixgbevf_tx_timeout - Respond to a Tx Hang 276 * @netdev: network interface device structure 277 **/ 278 static void ixgbevf_tx_timeout(struct net_device *netdev) 279 { 280 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 281 282 ixgbevf_tx_timeout_reset(adapter); 283 } 284 285 /** 286 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 287 * @q_vector: board private structure 288 * @tx_ring: tx ring to clean 289 **/ 290 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, 291 struct ixgbevf_ring *tx_ring) 292 { 293 struct ixgbevf_adapter *adapter = q_vector->adapter; 294 struct ixgbevf_tx_buffer *tx_buffer; 295 union ixgbe_adv_tx_desc *tx_desc; 296 unsigned int total_bytes = 0, total_packets = 0; 297 unsigned int budget = tx_ring->count / 2; 298 unsigned int i = tx_ring->next_to_clean; 299 300 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 301 return true; 302 303 tx_buffer = &tx_ring->tx_buffer_info[i]; 304 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 305 i -= tx_ring->count; 306 307 do { 308 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 309 310 /* if next_to_watch is not set then there is no work pending */ 311 if (!eop_desc) 312 break; 313 314 /* prevent any other reads prior to eop_desc */ 315 read_barrier_depends(); 316 317 /* if DD is not set pending work has not been completed */ 318 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 319 break; 320 321 /* clear next_to_watch to prevent false hangs */ 322 tx_buffer->next_to_watch = NULL; 323 324 /* update the statistics for this packet */ 325 total_bytes += tx_buffer->bytecount; 326 total_packets += tx_buffer->gso_segs; 327 328 /* free the skb */ 329 dev_kfree_skb_any(tx_buffer->skb); 330 331 /* unmap skb header data */ 332 dma_unmap_single(tx_ring->dev, 333 dma_unmap_addr(tx_buffer, dma), 334 dma_unmap_len(tx_buffer, len), 335 DMA_TO_DEVICE); 336 337 /* clear tx_buffer data */ 338 tx_buffer->skb = NULL; 339 dma_unmap_len_set(tx_buffer, len, 0); 340 341 /* unmap remaining buffers */ 342 while (tx_desc != eop_desc) { 343 tx_buffer++; 344 tx_desc++; 345 i++; 346 if (unlikely(!i)) { 347 i -= tx_ring->count; 348 tx_buffer = tx_ring->tx_buffer_info; 349 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 350 } 351 352 /* unmap any remaining paged data */ 353 if (dma_unmap_len(tx_buffer, len)) { 354 dma_unmap_page(tx_ring->dev, 355 dma_unmap_addr(tx_buffer, dma), 356 dma_unmap_len(tx_buffer, len), 357 DMA_TO_DEVICE); 358 dma_unmap_len_set(tx_buffer, len, 0); 359 } 360 } 361 362 /* move us one more past the eop_desc for start of next pkt */ 363 tx_buffer++; 364 tx_desc++; 365 i++; 366 if (unlikely(!i)) { 367 i -= tx_ring->count; 368 tx_buffer = tx_ring->tx_buffer_info; 369 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 370 } 371 372 /* issue prefetch for next Tx descriptor */ 373 prefetch(tx_desc); 374 375 /* update budget accounting */ 376 budget--; 377 } while (likely(budget)); 378 379 i += tx_ring->count; 380 tx_ring->next_to_clean = i; 381 u64_stats_update_begin(&tx_ring->syncp); 382 tx_ring->stats.bytes += total_bytes; 383 tx_ring->stats.packets += total_packets; 384 u64_stats_update_end(&tx_ring->syncp); 385 q_vector->tx.total_bytes += total_bytes; 386 q_vector->tx.total_packets += total_packets; 387 388 if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) { 389 struct ixgbe_hw *hw = &adapter->hw; 390 union ixgbe_adv_tx_desc *eop_desc; 391 392 eop_desc = tx_ring->tx_buffer_info[i].next_to_watch; 393 394 pr_err("Detected Tx Unit Hang\n" 395 " Tx Queue <%d>\n" 396 " TDH, TDT <%x>, <%x>\n" 397 " next_to_use <%x>\n" 398 " next_to_clean <%x>\n" 399 "tx_buffer_info[next_to_clean]\n" 400 " next_to_watch <%p>\n" 401 " eop_desc->wb.status <%x>\n" 402 " time_stamp <%lx>\n" 403 " jiffies <%lx>\n", 404 tx_ring->queue_index, 405 IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)), 406 IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)), 407 tx_ring->next_to_use, i, 408 eop_desc, (eop_desc ? eop_desc->wb.status : 0), 409 tx_ring->tx_buffer_info[i].time_stamp, jiffies); 410 411 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 412 413 /* schedule immediate reset if we believe we hung */ 414 ixgbevf_tx_timeout_reset(adapter); 415 416 return true; 417 } 418 419 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 420 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 421 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 422 /* Make sure that anybody stopping the queue after this 423 * sees the new next_to_clean. 424 */ 425 smp_mb(); 426 427 if (__netif_subqueue_stopped(tx_ring->netdev, 428 tx_ring->queue_index) && 429 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 430 netif_wake_subqueue(tx_ring->netdev, 431 tx_ring->queue_index); 432 ++tx_ring->tx_stats.restart_queue; 433 } 434 } 435 436 return !!budget; 437 } 438 439 /** 440 * ixgbevf_rx_skb - Helper function to determine proper Rx method 441 * @q_vector: structure containing interrupt and ring information 442 * @skb: packet to send up 443 **/ 444 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, 445 struct sk_buff *skb) 446 { 447 #ifdef CONFIG_NET_RX_BUSY_POLL 448 skb_mark_napi_id(skb, &q_vector->napi); 449 450 if (ixgbevf_qv_busy_polling(q_vector)) { 451 netif_receive_skb(skb); 452 /* exit early if we busy polled */ 453 return; 454 } 455 #endif /* CONFIG_NET_RX_BUSY_POLL */ 456 457 napi_gro_receive(&q_vector->napi, skb); 458 } 459 460 #define IXGBE_RSS_L4_TYPES_MASK \ 461 ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ 462 (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ 463 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ 464 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) 465 466 static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring, 467 union ixgbe_adv_rx_desc *rx_desc, 468 struct sk_buff *skb) 469 { 470 u16 rss_type; 471 472 if (!(ring->netdev->features & NETIF_F_RXHASH)) 473 return; 474 475 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & 476 IXGBE_RXDADV_RSSTYPE_MASK; 477 478 if (!rss_type) 479 return; 480 481 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 482 (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? 483 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); 484 } 485 486 /** 487 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 488 * @ring: structure containig ring specific data 489 * @rx_desc: current Rx descriptor being processed 490 * @skb: skb currently being received and modified 491 **/ 492 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, 493 union ixgbe_adv_rx_desc *rx_desc, 494 struct sk_buff *skb) 495 { 496 skb_checksum_none_assert(skb); 497 498 /* Rx csum disabled */ 499 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 500 return; 501 502 /* if IP and error */ 503 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 504 ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 505 ring->rx_stats.csum_err++; 506 return; 507 } 508 509 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 510 return; 511 512 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 513 ring->rx_stats.csum_err++; 514 return; 515 } 516 517 /* It must be a TCP or UDP packet with a valid checksum */ 518 skb->ip_summed = CHECKSUM_UNNECESSARY; 519 } 520 521 /** 522 * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor 523 * @rx_ring: rx descriptor ring packet is being transacted on 524 * @rx_desc: pointer to the EOP Rx descriptor 525 * @skb: pointer to current skb being populated 526 * 527 * This function checks the ring, descriptor, and packet information in 528 * order to populate the checksum, VLAN, protocol, and other fields within 529 * the skb. 530 **/ 531 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, 532 union ixgbe_adv_rx_desc *rx_desc, 533 struct sk_buff *skb) 534 { 535 ixgbevf_rx_hash(rx_ring, rx_desc, skb); 536 ixgbevf_rx_checksum(rx_ring, rx_desc, skb); 537 538 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 539 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 540 unsigned long *active_vlans = netdev_priv(rx_ring->netdev); 541 542 if (test_bit(vid & VLAN_VID_MASK, active_vlans)) 543 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 544 } 545 546 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 547 } 548 549 /** 550 * ixgbevf_is_non_eop - process handling of non-EOP buffers 551 * @rx_ring: Rx ring being processed 552 * @rx_desc: Rx descriptor for current buffer 553 * @skb: current socket buffer containing buffer in progress 554 * 555 * This function updates next to clean. If the buffer is an EOP buffer 556 * this function exits returning false, otherwise it will place the 557 * sk_buff in the next buffer to be chained and return true indicating 558 * that this is in fact a non-EOP buffer. 559 **/ 560 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, 561 union ixgbe_adv_rx_desc *rx_desc) 562 { 563 u32 ntc = rx_ring->next_to_clean + 1; 564 565 /* fetch, update, and store next to clean */ 566 ntc = (ntc < rx_ring->count) ? ntc : 0; 567 rx_ring->next_to_clean = ntc; 568 569 prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); 570 571 if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 572 return false; 573 574 return true; 575 } 576 577 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, 578 struct ixgbevf_rx_buffer *bi) 579 { 580 struct page *page = bi->page; 581 dma_addr_t dma = bi->dma; 582 583 /* since we are recycling buffers we should seldom need to alloc */ 584 if (likely(page)) 585 return true; 586 587 /* alloc new page for storage */ 588 page = dev_alloc_page(); 589 if (unlikely(!page)) { 590 rx_ring->rx_stats.alloc_rx_page_failed++; 591 return false; 592 } 593 594 /* map page for use */ 595 dma = dma_map_page(rx_ring->dev, page, 0, 596 PAGE_SIZE, DMA_FROM_DEVICE); 597 598 /* if mapping failed free memory back to system since 599 * there isn't much point in holding memory we can't use 600 */ 601 if (dma_mapping_error(rx_ring->dev, dma)) { 602 __free_page(page); 603 604 rx_ring->rx_stats.alloc_rx_buff_failed++; 605 return false; 606 } 607 608 bi->dma = dma; 609 bi->page = page; 610 bi->page_offset = 0; 611 612 return true; 613 } 614 615 /** 616 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 617 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on 618 * @cleaned_count: number of buffers to replace 619 **/ 620 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, 621 u16 cleaned_count) 622 { 623 union ixgbe_adv_rx_desc *rx_desc; 624 struct ixgbevf_rx_buffer *bi; 625 unsigned int i = rx_ring->next_to_use; 626 627 /* nothing to do or no valid netdev defined */ 628 if (!cleaned_count || !rx_ring->netdev) 629 return; 630 631 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 632 bi = &rx_ring->rx_buffer_info[i]; 633 i -= rx_ring->count; 634 635 do { 636 if (!ixgbevf_alloc_mapped_page(rx_ring, bi)) 637 break; 638 639 /* Refresh the desc even if pkt_addr didn't change 640 * because each write-back erases this info. 641 */ 642 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 643 644 rx_desc++; 645 bi++; 646 i++; 647 if (unlikely(!i)) { 648 rx_desc = IXGBEVF_RX_DESC(rx_ring, 0); 649 bi = rx_ring->rx_buffer_info; 650 i -= rx_ring->count; 651 } 652 653 /* clear the hdr_addr for the next_to_use descriptor */ 654 rx_desc->read.hdr_addr = 0; 655 656 cleaned_count--; 657 } while (cleaned_count); 658 659 i += rx_ring->count; 660 661 if (rx_ring->next_to_use != i) { 662 /* record the next descriptor to use */ 663 rx_ring->next_to_use = i; 664 665 /* update next to alloc since we have filled the ring */ 666 rx_ring->next_to_alloc = i; 667 668 /* Force memory writes to complete before letting h/w 669 * know there are new descriptors to fetch. (Only 670 * applicable for weak-ordered memory model archs, 671 * such as IA-64). 672 */ 673 wmb(); 674 ixgbevf_write_tail(rx_ring, i); 675 } 676 } 677 678 /** 679 * ixgbevf_cleanup_headers - Correct corrupted or empty headers 680 * @rx_ring: rx descriptor ring packet is being transacted on 681 * @rx_desc: pointer to the EOP Rx descriptor 682 * @skb: pointer to current skb being fixed 683 * 684 * Check for corrupted packet headers caused by senders on the local L2 685 * embedded NIC switch not setting up their Tx Descriptors right. These 686 * should be very rare. 687 * 688 * Also address the case where we are pulling data in on pages only 689 * and as such no data is present in the skb header. 690 * 691 * In addition if skb is not at least 60 bytes we need to pad it so that 692 * it is large enough to qualify as a valid Ethernet frame. 693 * 694 * Returns true if an error was encountered and skb was freed. 695 **/ 696 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, 697 union ixgbe_adv_rx_desc *rx_desc, 698 struct sk_buff *skb) 699 { 700 /* verify that the packet does not have any known errors */ 701 if (unlikely(ixgbevf_test_staterr(rx_desc, 702 IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { 703 struct net_device *netdev = rx_ring->netdev; 704 705 if (!(netdev->features & NETIF_F_RXALL)) { 706 dev_kfree_skb_any(skb); 707 return true; 708 } 709 } 710 711 /* if eth_skb_pad returns an error the skb was freed */ 712 if (eth_skb_pad(skb)) 713 return true; 714 715 return false; 716 } 717 718 /** 719 * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring 720 * @rx_ring: rx descriptor ring to store buffers on 721 * @old_buff: donor buffer to have page reused 722 * 723 * Synchronizes page for reuse by the adapter 724 **/ 725 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, 726 struct ixgbevf_rx_buffer *old_buff) 727 { 728 struct ixgbevf_rx_buffer *new_buff; 729 u16 nta = rx_ring->next_to_alloc; 730 731 new_buff = &rx_ring->rx_buffer_info[nta]; 732 733 /* update, and store next to alloc */ 734 nta++; 735 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 736 737 /* transfer page from old buffer to new buffer */ 738 new_buff->page = old_buff->page; 739 new_buff->dma = old_buff->dma; 740 new_buff->page_offset = old_buff->page_offset; 741 742 /* sync the buffer for use by the device */ 743 dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma, 744 new_buff->page_offset, 745 IXGBEVF_RX_BUFSZ, 746 DMA_FROM_DEVICE); 747 } 748 749 static inline bool ixgbevf_page_is_reserved(struct page *page) 750 { 751 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 752 } 753 754 /** 755 * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff 756 * @rx_ring: rx descriptor ring to transact packets on 757 * @rx_buffer: buffer containing page to add 758 * @rx_desc: descriptor containing length of buffer written by hardware 759 * @skb: sk_buff to place the data into 760 * 761 * This function will add the data contained in rx_buffer->page to the skb. 762 * This is done either through a direct copy if the data in the buffer is 763 * less than the skb header size, otherwise it will just attach the page as 764 * a frag to the skb. 765 * 766 * The function will then update the page offset if necessary and return 767 * true if the buffer can be reused by the adapter. 768 **/ 769 static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, 770 struct ixgbevf_rx_buffer *rx_buffer, 771 union ixgbe_adv_rx_desc *rx_desc, 772 struct sk_buff *skb) 773 { 774 struct page *page = rx_buffer->page; 775 unsigned char *va = page_address(page) + rx_buffer->page_offset; 776 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length); 777 #if (PAGE_SIZE < 8192) 778 unsigned int truesize = IXGBEVF_RX_BUFSZ; 779 #else 780 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES); 781 #endif 782 unsigned int pull_len; 783 784 if (unlikely(skb_is_nonlinear(skb))) 785 goto add_tail_frag; 786 787 if (likely(size <= IXGBEVF_RX_HDR_SIZE)) { 788 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 789 790 /* page is not reserved, we can reuse buffer as is */ 791 if (likely(!ixgbevf_page_is_reserved(page))) 792 return true; 793 794 /* this page cannot be reused so discard it */ 795 put_page(page); 796 return false; 797 } 798 799 /* we need the header to contain the greater of either ETH_HLEN or 800 * 60 bytes if the skb->len is less than 60 for skb_pad. 801 */ 802 pull_len = eth_get_headlen(va, IXGBEVF_RX_HDR_SIZE); 803 804 /* align pull length to size of long to optimize memcpy performance */ 805 memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long))); 806 807 /* update all of the pointers */ 808 va += pull_len; 809 size -= pull_len; 810 811 add_tail_frag: 812 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 813 (unsigned long)va & ~PAGE_MASK, size, truesize); 814 815 /* avoid re-using remote pages */ 816 if (unlikely(ixgbevf_page_is_reserved(page))) 817 return false; 818 819 #if (PAGE_SIZE < 8192) 820 /* if we are only owner of page we can reuse it */ 821 if (unlikely(page_count(page) != 1)) 822 return false; 823 824 /* flip page offset to other buffer */ 825 rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ; 826 827 #else 828 /* move offset up to the next cache line */ 829 rx_buffer->page_offset += truesize; 830 831 if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ)) 832 return false; 833 834 #endif 835 /* Even if we own the page, we are not allowed to use atomic_set() 836 * This would break get_page_unless_zero() users. 837 */ 838 atomic_inc(&page->_count); 839 840 return true; 841 } 842 843 static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring, 844 union ixgbe_adv_rx_desc *rx_desc, 845 struct sk_buff *skb) 846 { 847 struct ixgbevf_rx_buffer *rx_buffer; 848 struct page *page; 849 850 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 851 page = rx_buffer->page; 852 prefetchw(page); 853 854 if (likely(!skb)) { 855 void *page_addr = page_address(page) + 856 rx_buffer->page_offset; 857 858 /* prefetch first cache line of first page */ 859 prefetch(page_addr); 860 #if L1_CACHE_BYTES < 128 861 prefetch(page_addr + L1_CACHE_BYTES); 862 #endif 863 864 /* allocate a skb to store the frags */ 865 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 866 IXGBEVF_RX_HDR_SIZE); 867 if (unlikely(!skb)) { 868 rx_ring->rx_stats.alloc_rx_buff_failed++; 869 return NULL; 870 } 871 872 /* we will be copying header into skb->data in 873 * pskb_may_pull so it is in our interest to prefetch 874 * it now to avoid a possible cache miss 875 */ 876 prefetchw(skb->data); 877 } 878 879 /* we are reusing so sync this buffer for CPU use */ 880 dma_sync_single_range_for_cpu(rx_ring->dev, 881 rx_buffer->dma, 882 rx_buffer->page_offset, 883 IXGBEVF_RX_BUFSZ, 884 DMA_FROM_DEVICE); 885 886 /* pull page into skb */ 887 if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) { 888 /* hand second half of page back to the ring */ 889 ixgbevf_reuse_rx_page(rx_ring, rx_buffer); 890 } else { 891 /* we are not reusing the buffer so unmap it */ 892 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 893 PAGE_SIZE, DMA_FROM_DEVICE); 894 } 895 896 /* clear contents of buffer_info */ 897 rx_buffer->dma = 0; 898 rx_buffer->page = NULL; 899 900 return skb; 901 } 902 903 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 904 u32 qmask) 905 { 906 struct ixgbe_hw *hw = &adapter->hw; 907 908 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); 909 } 910 911 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 912 struct ixgbevf_ring *rx_ring, 913 int budget) 914 { 915 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 916 u16 cleaned_count = ixgbevf_desc_unused(rx_ring); 917 struct sk_buff *skb = rx_ring->skb; 918 919 while (likely(total_rx_packets < budget)) { 920 union ixgbe_adv_rx_desc *rx_desc; 921 922 /* return some buffers to hardware, one at a time is too slow */ 923 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 924 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count); 925 cleaned_count = 0; 926 } 927 928 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); 929 930 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD)) 931 break; 932 933 /* This memory barrier is needed to keep us from reading 934 * any other fields out of the rx_desc until we know the 935 * RXD_STAT_DD bit is set 936 */ 937 rmb(); 938 939 /* retrieve a buffer from the ring */ 940 skb = ixgbevf_fetch_rx_buffer(rx_ring, rx_desc, skb); 941 942 /* exit if we failed to retrieve a buffer */ 943 if (!skb) 944 break; 945 946 cleaned_count++; 947 948 /* fetch next buffer in frame if non-eop */ 949 if (ixgbevf_is_non_eop(rx_ring, rx_desc)) 950 continue; 951 952 /* verify the packet layout is correct */ 953 if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) { 954 skb = NULL; 955 continue; 956 } 957 958 /* probably a little skewed due to removing CRC */ 959 total_rx_bytes += skb->len; 960 961 /* Workaround hardware that can't do proper VEPA multicast 962 * source pruning. 963 */ 964 if ((skb->pkt_type == PACKET_BROADCAST || 965 skb->pkt_type == PACKET_MULTICAST) && 966 ether_addr_equal(rx_ring->netdev->dev_addr, 967 eth_hdr(skb)->h_source)) { 968 dev_kfree_skb_irq(skb); 969 continue; 970 } 971 972 /* populate checksum, VLAN, and protocol */ 973 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); 974 975 ixgbevf_rx_skb(q_vector, skb); 976 977 /* reset skb pointer */ 978 skb = NULL; 979 980 /* update budget accounting */ 981 total_rx_packets++; 982 } 983 984 /* place incomplete frames back on ring for completion */ 985 rx_ring->skb = skb; 986 987 u64_stats_update_begin(&rx_ring->syncp); 988 rx_ring->stats.packets += total_rx_packets; 989 rx_ring->stats.bytes += total_rx_bytes; 990 u64_stats_update_end(&rx_ring->syncp); 991 q_vector->rx.total_packets += total_rx_packets; 992 q_vector->rx.total_bytes += total_rx_bytes; 993 994 return total_rx_packets; 995 } 996 997 /** 998 * ixgbevf_poll - NAPI polling calback 999 * @napi: napi struct with our devices info in it 1000 * @budget: amount of work driver is allowed to do this pass, in packets 1001 * 1002 * This function will clean more than one or more rings associated with a 1003 * q_vector. 1004 **/ 1005 static int ixgbevf_poll(struct napi_struct *napi, int budget) 1006 { 1007 struct ixgbevf_q_vector *q_vector = 1008 container_of(napi, struct ixgbevf_q_vector, napi); 1009 struct ixgbevf_adapter *adapter = q_vector->adapter; 1010 struct ixgbevf_ring *ring; 1011 int per_ring_budget, work_done = 0; 1012 bool clean_complete = true; 1013 1014 ixgbevf_for_each_ring(ring, q_vector->tx) 1015 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring); 1016 1017 #ifdef CONFIG_NET_RX_BUSY_POLL 1018 if (!ixgbevf_qv_lock_napi(q_vector)) 1019 return budget; 1020 #endif 1021 1022 /* attempt to distribute budget to each queue fairly, but don't allow 1023 * the budget to go below 1 because we'll exit polling 1024 */ 1025 if (q_vector->rx.count > 1) 1026 per_ring_budget = max(budget/q_vector->rx.count, 1); 1027 else 1028 per_ring_budget = budget; 1029 1030 ixgbevf_for_each_ring(ring, q_vector->rx) { 1031 int cleaned = ixgbevf_clean_rx_irq(q_vector, ring, 1032 per_ring_budget); 1033 work_done += cleaned; 1034 clean_complete &= (cleaned < per_ring_budget); 1035 } 1036 1037 #ifdef CONFIG_NET_RX_BUSY_POLL 1038 ixgbevf_qv_unlock_napi(q_vector); 1039 #endif 1040 1041 /* If all work not completed, return budget and keep polling */ 1042 if (!clean_complete) 1043 return budget; 1044 /* all work done, exit the polling mode */ 1045 napi_complete_done(napi, work_done); 1046 if (adapter->rx_itr_setting & 1) 1047 ixgbevf_set_itr(q_vector); 1048 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 1049 !test_bit(__IXGBEVF_REMOVING, &adapter->state)) 1050 ixgbevf_irq_enable_queues(adapter, 1051 1 << q_vector->v_idx); 1052 1053 return 0; 1054 } 1055 1056 /** 1057 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 1058 * @q_vector: structure containing interrupt and ring information 1059 **/ 1060 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) 1061 { 1062 struct ixgbevf_adapter *adapter = q_vector->adapter; 1063 struct ixgbe_hw *hw = &adapter->hw; 1064 int v_idx = q_vector->v_idx; 1065 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 1066 1067 /* set the WDIS bit to not clear the timer bits and cause an 1068 * immediate assertion of the interrupt 1069 */ 1070 itr_reg |= IXGBE_EITR_CNT_WDIS; 1071 1072 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 1073 } 1074 1075 #ifdef CONFIG_NET_RX_BUSY_POLL 1076 /* must be called with local_bh_disable()d */ 1077 static int ixgbevf_busy_poll_recv(struct napi_struct *napi) 1078 { 1079 struct ixgbevf_q_vector *q_vector = 1080 container_of(napi, struct ixgbevf_q_vector, napi); 1081 struct ixgbevf_adapter *adapter = q_vector->adapter; 1082 struct ixgbevf_ring *ring; 1083 int found = 0; 1084 1085 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 1086 return LL_FLUSH_FAILED; 1087 1088 if (!ixgbevf_qv_lock_poll(q_vector)) 1089 return LL_FLUSH_BUSY; 1090 1091 ixgbevf_for_each_ring(ring, q_vector->rx) { 1092 found = ixgbevf_clean_rx_irq(q_vector, ring, 4); 1093 #ifdef BP_EXTENDED_STATS 1094 if (found) 1095 ring->stats.cleaned += found; 1096 else 1097 ring->stats.misses++; 1098 #endif 1099 if (found) 1100 break; 1101 } 1102 1103 ixgbevf_qv_unlock_poll(q_vector); 1104 1105 return found; 1106 } 1107 #endif /* CONFIG_NET_RX_BUSY_POLL */ 1108 1109 /** 1110 * ixgbevf_configure_msix - Configure MSI-X hardware 1111 * @adapter: board private structure 1112 * 1113 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 1114 * interrupts. 1115 **/ 1116 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 1117 { 1118 struct ixgbevf_q_vector *q_vector; 1119 int q_vectors, v_idx; 1120 1121 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1122 adapter->eims_enable_mask = 0; 1123 1124 /* Populate the IVAR table and set the ITR values to the 1125 * corresponding register. 1126 */ 1127 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 1128 struct ixgbevf_ring *ring; 1129 1130 q_vector = adapter->q_vector[v_idx]; 1131 1132 ixgbevf_for_each_ring(ring, q_vector->rx) 1133 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx); 1134 1135 ixgbevf_for_each_ring(ring, q_vector->tx) 1136 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx); 1137 1138 if (q_vector->tx.ring && !q_vector->rx.ring) { 1139 /* Tx only vector */ 1140 if (adapter->tx_itr_setting == 1) 1141 q_vector->itr = IXGBE_10K_ITR; 1142 else 1143 q_vector->itr = adapter->tx_itr_setting; 1144 } else { 1145 /* Rx or Rx/Tx vector */ 1146 if (adapter->rx_itr_setting == 1) 1147 q_vector->itr = IXGBE_20K_ITR; 1148 else 1149 q_vector->itr = adapter->rx_itr_setting; 1150 } 1151 1152 /* add q_vector eims value to global eims_enable_mask */ 1153 adapter->eims_enable_mask |= 1 << v_idx; 1154 1155 ixgbevf_write_eitr(q_vector); 1156 } 1157 1158 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 1159 /* setup eims_other and add value to global eims_enable_mask */ 1160 adapter->eims_other = 1 << v_idx; 1161 adapter->eims_enable_mask |= adapter->eims_other; 1162 } 1163 1164 enum latency_range { 1165 lowest_latency = 0, 1166 low_latency = 1, 1167 bulk_latency = 2, 1168 latency_invalid = 255 1169 }; 1170 1171 /** 1172 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 1173 * @q_vector: structure containing interrupt and ring information 1174 * @ring_container: structure containing ring performance data 1175 * 1176 * Stores a new ITR value based on packets and byte 1177 * counts during the last interrupt. The advantage of per interrupt 1178 * computation is faster updates and more accurate ITR for the current 1179 * traffic pattern. Constants in this function were computed 1180 * based on theoretical maximum wire speed and thresholds were set based 1181 * on testing data as well as attempting to minimize response time 1182 * while increasing bulk throughput. 1183 **/ 1184 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector, 1185 struct ixgbevf_ring_container *ring_container) 1186 { 1187 int bytes = ring_container->total_bytes; 1188 int packets = ring_container->total_packets; 1189 u32 timepassed_us; 1190 u64 bytes_perint; 1191 u8 itr_setting = ring_container->itr; 1192 1193 if (packets == 0) 1194 return; 1195 1196 /* simple throttle rate management 1197 * 0-20MB/s lowest (100000 ints/s) 1198 * 20-100MB/s low (20000 ints/s) 1199 * 100-1249MB/s bulk (8000 ints/s) 1200 */ 1201 /* what was last interrupt timeslice? */ 1202 timepassed_us = q_vector->itr >> 2; 1203 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 1204 1205 switch (itr_setting) { 1206 case lowest_latency: 1207 if (bytes_perint > 10) 1208 itr_setting = low_latency; 1209 break; 1210 case low_latency: 1211 if (bytes_perint > 20) 1212 itr_setting = bulk_latency; 1213 else if (bytes_perint <= 10) 1214 itr_setting = lowest_latency; 1215 break; 1216 case bulk_latency: 1217 if (bytes_perint <= 20) 1218 itr_setting = low_latency; 1219 break; 1220 } 1221 1222 /* clear work counters since we have the values we need */ 1223 ring_container->total_bytes = 0; 1224 ring_container->total_packets = 0; 1225 1226 /* write updated itr to ring container */ 1227 ring_container->itr = itr_setting; 1228 } 1229 1230 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector) 1231 { 1232 u32 new_itr = q_vector->itr; 1233 u8 current_itr; 1234 1235 ixgbevf_update_itr(q_vector, &q_vector->tx); 1236 ixgbevf_update_itr(q_vector, &q_vector->rx); 1237 1238 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 1239 1240 switch (current_itr) { 1241 /* counts and packets in update_itr are dependent on these numbers */ 1242 case lowest_latency: 1243 new_itr = IXGBE_100K_ITR; 1244 break; 1245 case low_latency: 1246 new_itr = IXGBE_20K_ITR; 1247 break; 1248 case bulk_latency: 1249 default: 1250 new_itr = IXGBE_8K_ITR; 1251 break; 1252 } 1253 1254 if (new_itr != q_vector->itr) { 1255 /* do an exponential smoothing */ 1256 new_itr = (10 * new_itr * q_vector->itr) / 1257 ((9 * new_itr) + q_vector->itr); 1258 1259 /* save the algorithm value here */ 1260 q_vector->itr = new_itr; 1261 1262 ixgbevf_write_eitr(q_vector); 1263 } 1264 } 1265 1266 static irqreturn_t ixgbevf_msix_other(int irq, void *data) 1267 { 1268 struct ixgbevf_adapter *adapter = data; 1269 struct ixgbe_hw *hw = &adapter->hw; 1270 1271 hw->mac.get_link_status = 1; 1272 1273 ixgbevf_service_event_schedule(adapter); 1274 1275 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); 1276 1277 return IRQ_HANDLED; 1278 } 1279 1280 /** 1281 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) 1282 * @irq: unused 1283 * @data: pointer to our q_vector struct for this interrupt vector 1284 **/ 1285 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data) 1286 { 1287 struct ixgbevf_q_vector *q_vector = data; 1288 1289 /* EIAM disabled interrupts (on this vector) for us */ 1290 if (q_vector->rx.ring || q_vector->tx.ring) 1291 napi_schedule(&q_vector->napi); 1292 1293 return IRQ_HANDLED; 1294 } 1295 1296 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, 1297 int r_idx) 1298 { 1299 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1300 1301 a->rx_ring[r_idx]->next = q_vector->rx.ring; 1302 q_vector->rx.ring = a->rx_ring[r_idx]; 1303 q_vector->rx.count++; 1304 } 1305 1306 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, 1307 int t_idx) 1308 { 1309 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1310 1311 a->tx_ring[t_idx]->next = q_vector->tx.ring; 1312 q_vector->tx.ring = a->tx_ring[t_idx]; 1313 q_vector->tx.count++; 1314 } 1315 1316 /** 1317 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors 1318 * @adapter: board private structure to initialize 1319 * 1320 * This function maps descriptor rings to the queue-specific vectors 1321 * we were allotted through the MSI-X enabling code. Ideally, we'd have 1322 * one vector per ring/queue, but on a constrained vector budget, we 1323 * group the rings as "efficiently" as possible. You would add new 1324 * mapping configurations in here. 1325 **/ 1326 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter) 1327 { 1328 int q_vectors; 1329 int v_start = 0; 1330 int rxr_idx = 0, txr_idx = 0; 1331 int rxr_remaining = adapter->num_rx_queues; 1332 int txr_remaining = adapter->num_tx_queues; 1333 int i, j; 1334 int rqpv, tqpv; 1335 int err = 0; 1336 1337 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1338 1339 /* The ideal configuration... 1340 * We have enough vectors to map one per queue. 1341 */ 1342 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) { 1343 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) 1344 map_vector_to_rxq(adapter, v_start, rxr_idx); 1345 1346 for (; txr_idx < txr_remaining; v_start++, txr_idx++) 1347 map_vector_to_txq(adapter, v_start, txr_idx); 1348 goto out; 1349 } 1350 1351 /* If we don't have enough vectors for a 1-to-1 1352 * mapping, we'll have to group them so there are 1353 * multiple queues per vector. 1354 */ 1355 /* Re-adjusting *qpv takes care of the remainder. */ 1356 for (i = v_start; i < q_vectors; i++) { 1357 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i); 1358 for (j = 0; j < rqpv; j++) { 1359 map_vector_to_rxq(adapter, i, rxr_idx); 1360 rxr_idx++; 1361 rxr_remaining--; 1362 } 1363 } 1364 for (i = v_start; i < q_vectors; i++) { 1365 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i); 1366 for (j = 0; j < tqpv; j++) { 1367 map_vector_to_txq(adapter, i, txr_idx); 1368 txr_idx++; 1369 txr_remaining--; 1370 } 1371 } 1372 1373 out: 1374 return err; 1375 } 1376 1377 /** 1378 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 1379 * @adapter: board private structure 1380 * 1381 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 1382 * interrupts from the kernel. 1383 **/ 1384 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 1385 { 1386 struct net_device *netdev = adapter->netdev; 1387 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1388 int vector, err; 1389 int ri = 0, ti = 0; 1390 1391 for (vector = 0; vector < q_vectors; vector++) { 1392 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector]; 1393 struct msix_entry *entry = &adapter->msix_entries[vector]; 1394 1395 if (q_vector->tx.ring && q_vector->rx.ring) { 1396 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1397 "%s-%s-%d", netdev->name, "TxRx", ri++); 1398 ti++; 1399 } else if (q_vector->rx.ring) { 1400 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1401 "%s-%s-%d", netdev->name, "rx", ri++); 1402 } else if (q_vector->tx.ring) { 1403 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1404 "%s-%s-%d", netdev->name, "tx", ti++); 1405 } else { 1406 /* skip this unused q_vector */ 1407 continue; 1408 } 1409 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0, 1410 q_vector->name, q_vector); 1411 if (err) { 1412 hw_dbg(&adapter->hw, 1413 "request_irq failed for MSIX interrupt Error: %d\n", 1414 err); 1415 goto free_queue_irqs; 1416 } 1417 } 1418 1419 err = request_irq(adapter->msix_entries[vector].vector, 1420 &ixgbevf_msix_other, 0, netdev->name, adapter); 1421 if (err) { 1422 hw_dbg(&adapter->hw, "request_irq for msix_other failed: %d\n", 1423 err); 1424 goto free_queue_irqs; 1425 } 1426 1427 return 0; 1428 1429 free_queue_irqs: 1430 while (vector) { 1431 vector--; 1432 free_irq(adapter->msix_entries[vector].vector, 1433 adapter->q_vector[vector]); 1434 } 1435 /* This failure is non-recoverable - it indicates the system is 1436 * out of MSIX vector resources and the VF driver cannot run 1437 * without them. Set the number of msix vectors to zero 1438 * indicating that not enough can be allocated. The error 1439 * will be returned to the user indicating device open failed. 1440 * Any further attempts to force the driver to open will also 1441 * fail. The only way to recover is to unload the driver and 1442 * reload it again. If the system has recovered some MSIX 1443 * vectors then it may succeed. 1444 */ 1445 adapter->num_msix_vectors = 0; 1446 return err; 1447 } 1448 1449 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter) 1450 { 1451 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1452 1453 for (i = 0; i < q_vectors; i++) { 1454 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i]; 1455 1456 q_vector->rx.ring = NULL; 1457 q_vector->tx.ring = NULL; 1458 q_vector->rx.count = 0; 1459 q_vector->tx.count = 0; 1460 } 1461 } 1462 1463 /** 1464 * ixgbevf_request_irq - initialize interrupts 1465 * @adapter: board private structure 1466 * 1467 * Attempts to configure interrupts using the best available 1468 * capabilities of the hardware and kernel. 1469 **/ 1470 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 1471 { 1472 int err = 0; 1473 1474 err = ixgbevf_request_msix_irqs(adapter); 1475 1476 if (err) 1477 hw_dbg(&adapter->hw, "request_irq failed, Error %d\n", err); 1478 1479 return err; 1480 } 1481 1482 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 1483 { 1484 int i, q_vectors; 1485 1486 q_vectors = adapter->num_msix_vectors; 1487 i = q_vectors - 1; 1488 1489 free_irq(adapter->msix_entries[i].vector, adapter); 1490 i--; 1491 1492 for (; i >= 0; i--) { 1493 /* free only the irqs that were actually requested */ 1494 if (!adapter->q_vector[i]->rx.ring && 1495 !adapter->q_vector[i]->tx.ring) 1496 continue; 1497 1498 free_irq(adapter->msix_entries[i].vector, 1499 adapter->q_vector[i]); 1500 } 1501 1502 ixgbevf_reset_q_vectors(adapter); 1503 } 1504 1505 /** 1506 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1507 * @adapter: board private structure 1508 **/ 1509 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1510 { 1511 struct ixgbe_hw *hw = &adapter->hw; 1512 int i; 1513 1514 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0); 1515 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1516 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0); 1517 1518 IXGBE_WRITE_FLUSH(hw); 1519 1520 for (i = 0; i < adapter->num_msix_vectors; i++) 1521 synchronize_irq(adapter->msix_entries[i].vector); 1522 } 1523 1524 /** 1525 * ixgbevf_irq_enable - Enable default interrupt generation settings 1526 * @adapter: board private structure 1527 **/ 1528 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) 1529 { 1530 struct ixgbe_hw *hw = &adapter->hw; 1531 1532 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask); 1533 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask); 1534 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); 1535 } 1536 1537 /** 1538 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset 1539 * @adapter: board private structure 1540 * @ring: structure containing ring specific data 1541 * 1542 * Configure the Tx descriptor ring after a reset. 1543 **/ 1544 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 1545 struct ixgbevf_ring *ring) 1546 { 1547 struct ixgbe_hw *hw = &adapter->hw; 1548 u64 tdba = ring->dma; 1549 int wait_loop = 10; 1550 u32 txdctl = IXGBE_TXDCTL_ENABLE; 1551 u8 reg_idx = ring->reg_idx; 1552 1553 /* disable queue to avoid issues while updating state */ 1554 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 1555 IXGBE_WRITE_FLUSH(hw); 1556 1557 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 1558 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32); 1559 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx), 1560 ring->count * sizeof(union ixgbe_adv_tx_desc)); 1561 1562 /* disable head writeback */ 1563 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0); 1564 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0); 1565 1566 /* enable relaxed ordering */ 1567 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx), 1568 (IXGBE_DCA_TXCTRL_DESC_RRO_EN | 1569 IXGBE_DCA_TXCTRL_DATA_RRO_EN)); 1570 1571 /* reset head and tail pointers */ 1572 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0); 1573 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0); 1574 ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx); 1575 1576 /* reset ntu and ntc to place SW in sync with hardwdare */ 1577 ring->next_to_clean = 0; 1578 ring->next_to_use = 0; 1579 1580 /* In order to avoid issues WTHRESH + PTHRESH should always be equal 1581 * to or less than the number of on chip descriptors, which is 1582 * currently 40. 1583 */ 1584 txdctl |= (8 << 16); /* WTHRESH = 8 */ 1585 1586 /* Setting PTHRESH to 32 both improves performance */ 1587 txdctl |= (1 << 8) | /* HTHRESH = 1 */ 1588 32; /* PTHRESH = 32 */ 1589 1590 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); 1591 1592 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl); 1593 1594 /* poll to verify queue is enabled */ 1595 do { 1596 usleep_range(1000, 2000); 1597 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx)); 1598 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 1599 if (!wait_loop) 1600 pr_err("Could not enable Tx Queue %d\n", reg_idx); 1601 } 1602 1603 /** 1604 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1605 * @adapter: board private structure 1606 * 1607 * Configure the Tx unit of the MAC after a reset. 1608 **/ 1609 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1610 { 1611 u32 i; 1612 1613 /* Setup the HW Tx Head and Tail descriptor pointers */ 1614 for (i = 0; i < adapter->num_tx_queues; i++) 1615 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]); 1616 } 1617 1618 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1619 1620 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) 1621 { 1622 struct ixgbe_hw *hw = &adapter->hw; 1623 u32 srrctl; 1624 1625 srrctl = IXGBE_SRRCTL_DROP_EN; 1626 1627 srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; 1628 srrctl |= IXGBEVF_RX_BUFSZ >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1629 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1630 1631 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1632 } 1633 1634 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) 1635 { 1636 struct ixgbe_hw *hw = &adapter->hw; 1637 1638 /* PSRTYPE must be initialized in 82599 */ 1639 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 1640 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR | 1641 IXGBE_PSRTYPE_L2HDR; 1642 1643 if (adapter->num_rx_queues > 1) 1644 psrtype |= 1 << 29; 1645 1646 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1647 } 1648 1649 #define IXGBEVF_MAX_RX_DESC_POLL 10 1650 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, 1651 struct ixgbevf_ring *ring) 1652 { 1653 struct ixgbe_hw *hw = &adapter->hw; 1654 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1655 u32 rxdctl; 1656 u8 reg_idx = ring->reg_idx; 1657 1658 if (IXGBE_REMOVED(hw->hw_addr)) 1659 return; 1660 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1661 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1662 1663 /* write value back with RXDCTL.ENABLE bit cleared */ 1664 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1665 1666 /* the hardware may take up to 100us to really disable the Rx queue */ 1667 do { 1668 udelay(10); 1669 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1670 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 1671 1672 if (!wait_loop) 1673 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", 1674 reg_idx); 1675 } 1676 1677 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1678 struct ixgbevf_ring *ring) 1679 { 1680 struct ixgbe_hw *hw = &adapter->hw; 1681 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1682 u32 rxdctl; 1683 u8 reg_idx = ring->reg_idx; 1684 1685 if (IXGBE_REMOVED(hw->hw_addr)) 1686 return; 1687 do { 1688 usleep_range(1000, 2000); 1689 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1690 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 1691 1692 if (!wait_loop) 1693 pr_err("RXDCTL.ENABLE queue %d not set while polling\n", 1694 reg_idx); 1695 } 1696 1697 static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) 1698 { 1699 struct ixgbe_hw *hw = &adapter->hw; 1700 u32 vfmrqc = 0, vfreta = 0; 1701 u16 rss_i = adapter->num_rx_queues; 1702 u8 i, j; 1703 1704 /* Fill out hash function seeds */ 1705 netdev_rss_key_fill(adapter->rss_key, sizeof(adapter->rss_key)); 1706 for (i = 0; i < IXGBEVF_VFRSSRK_REGS; i++) 1707 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), adapter->rss_key[i]); 1708 1709 for (i = 0, j = 0; i < IXGBEVF_X550_VFRETA_SIZE; i++, j++) { 1710 if (j == rss_i) 1711 j = 0; 1712 1713 adapter->rss_indir_tbl[i] = j; 1714 1715 vfreta |= j << (i & 0x3) * 8; 1716 if ((i & 3) == 3) { 1717 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), vfreta); 1718 vfreta = 0; 1719 } 1720 } 1721 1722 /* Perform hash on these packet types */ 1723 vfmrqc |= IXGBE_VFMRQC_RSS_FIELD_IPV4 | 1724 IXGBE_VFMRQC_RSS_FIELD_IPV4_TCP | 1725 IXGBE_VFMRQC_RSS_FIELD_IPV6 | 1726 IXGBE_VFMRQC_RSS_FIELD_IPV6_TCP; 1727 1728 vfmrqc |= IXGBE_VFMRQC_RSSEN; 1729 1730 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); 1731 } 1732 1733 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, 1734 struct ixgbevf_ring *ring) 1735 { 1736 struct ixgbe_hw *hw = &adapter->hw; 1737 u64 rdba = ring->dma; 1738 u32 rxdctl; 1739 u8 reg_idx = ring->reg_idx; 1740 1741 /* disable queue to avoid issues while updating state */ 1742 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1743 ixgbevf_disable_rx_queue(adapter, ring); 1744 1745 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 1746 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32); 1747 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx), 1748 ring->count * sizeof(union ixgbe_adv_rx_desc)); 1749 1750 /* enable relaxed ordering */ 1751 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1752 IXGBE_DCA_RXCTRL_DESC_RRO_EN); 1753 1754 /* reset head and tail pointers */ 1755 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0); 1756 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); 1757 ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); 1758 1759 /* reset ntu and ntc to place SW in sync with hardwdare */ 1760 ring->next_to_clean = 0; 1761 ring->next_to_use = 0; 1762 ring->next_to_alloc = 0; 1763 1764 ixgbevf_configure_srrctl(adapter, reg_idx); 1765 1766 /* allow any size packet since we can handle overflow */ 1767 rxdctl &= ~IXGBE_RXDCTL_RLPML_EN; 1768 1769 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1770 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1771 1772 ixgbevf_rx_desc_queue_enable(adapter, ring); 1773 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); 1774 } 1775 1776 /** 1777 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1778 * @adapter: board private structure 1779 * 1780 * Configure the Rx unit of the MAC after a reset. 1781 **/ 1782 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1783 { 1784 int i; 1785 struct ixgbe_hw *hw = &adapter->hw; 1786 struct net_device *netdev = adapter->netdev; 1787 1788 ixgbevf_setup_psrtype(adapter); 1789 if (hw->mac.type >= ixgbe_mac_X550_vf) 1790 ixgbevf_setup_vfmrqc(adapter); 1791 1792 /* notify the PF of our intent to use this size of frame */ 1793 ixgbevf_rlpml_set_vf(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN); 1794 1795 /* Setup the HW Rx Head and Tail Descriptor Pointers and 1796 * the Base and Length of the Rx Descriptor Ring 1797 */ 1798 for (i = 0; i < adapter->num_rx_queues; i++) 1799 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]); 1800 } 1801 1802 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, 1803 __be16 proto, u16 vid) 1804 { 1805 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1806 struct ixgbe_hw *hw = &adapter->hw; 1807 int err; 1808 1809 spin_lock_bh(&adapter->mbx_lock); 1810 1811 /* add VID to filter table */ 1812 err = hw->mac.ops.set_vfta(hw, vid, 0, true); 1813 1814 spin_unlock_bh(&adapter->mbx_lock); 1815 1816 /* translate error return types so error makes sense */ 1817 if (err == IXGBE_ERR_MBX) 1818 return -EIO; 1819 1820 if (err == IXGBE_ERR_INVALID_ARGUMENT) 1821 return -EACCES; 1822 1823 set_bit(vid, adapter->active_vlans); 1824 1825 return err; 1826 } 1827 1828 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, 1829 __be16 proto, u16 vid) 1830 { 1831 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1832 struct ixgbe_hw *hw = &adapter->hw; 1833 int err = -EOPNOTSUPP; 1834 1835 spin_lock_bh(&adapter->mbx_lock); 1836 1837 /* remove VID from filter table */ 1838 err = hw->mac.ops.set_vfta(hw, vid, 0, false); 1839 1840 spin_unlock_bh(&adapter->mbx_lock); 1841 1842 clear_bit(vid, adapter->active_vlans); 1843 1844 return err; 1845 } 1846 1847 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 1848 { 1849 u16 vid; 1850 1851 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1852 ixgbevf_vlan_rx_add_vid(adapter->netdev, 1853 htons(ETH_P_8021Q), vid); 1854 } 1855 1856 static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 1857 { 1858 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1859 struct ixgbe_hw *hw = &adapter->hw; 1860 int count = 0; 1861 1862 if ((netdev_uc_count(netdev)) > 10) { 1863 pr_err("Too many unicast filters - No Space\n"); 1864 return -ENOSPC; 1865 } 1866 1867 if (!netdev_uc_empty(netdev)) { 1868 struct netdev_hw_addr *ha; 1869 1870 netdev_for_each_uc_addr(ha, netdev) { 1871 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 1872 udelay(200); 1873 } 1874 } else { 1875 /* If the list is empty then send message to PF driver to 1876 * clear all MAC VLANs on this VF. 1877 */ 1878 hw->mac.ops.set_uc_addr(hw, 0, NULL); 1879 } 1880 1881 return count; 1882 } 1883 1884 /** 1885 * ixgbevf_set_rx_mode - Multicast and unicast set 1886 * @netdev: network interface device structure 1887 * 1888 * The set_rx_method entry point is called whenever the multicast address 1889 * list, unicast address list or the network interface flags are updated. 1890 * This routine is responsible for configuring the hardware for proper 1891 * multicast mode and configuring requested unicast filters. 1892 **/ 1893 static void ixgbevf_set_rx_mode(struct net_device *netdev) 1894 { 1895 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1896 struct ixgbe_hw *hw = &adapter->hw; 1897 unsigned int flags = netdev->flags; 1898 int xcast_mode; 1899 1900 xcast_mode = (flags & IFF_ALLMULTI) ? IXGBEVF_XCAST_MODE_ALLMULTI : 1901 (flags & (IFF_BROADCAST | IFF_MULTICAST)) ? 1902 IXGBEVF_XCAST_MODE_MULTI : IXGBEVF_XCAST_MODE_NONE; 1903 1904 spin_lock_bh(&adapter->mbx_lock); 1905 1906 hw->mac.ops.update_xcast_mode(hw, netdev, xcast_mode); 1907 1908 /* reprogram multicast list */ 1909 hw->mac.ops.update_mc_addr_list(hw, netdev); 1910 1911 ixgbevf_write_uc_addr_list(netdev); 1912 1913 spin_unlock_bh(&adapter->mbx_lock); 1914 } 1915 1916 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 1917 { 1918 int q_idx; 1919 struct ixgbevf_q_vector *q_vector; 1920 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1921 1922 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1923 q_vector = adapter->q_vector[q_idx]; 1924 #ifdef CONFIG_NET_RX_BUSY_POLL 1925 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]); 1926 #endif 1927 napi_enable(&q_vector->napi); 1928 } 1929 } 1930 1931 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 1932 { 1933 int q_idx; 1934 struct ixgbevf_q_vector *q_vector; 1935 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1936 1937 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1938 q_vector = adapter->q_vector[q_idx]; 1939 napi_disable(&q_vector->napi); 1940 #ifdef CONFIG_NET_RX_BUSY_POLL 1941 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) { 1942 pr_info("QV %d locked\n", q_idx); 1943 usleep_range(1000, 20000); 1944 } 1945 #endif /* CONFIG_NET_RX_BUSY_POLL */ 1946 } 1947 } 1948 1949 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter) 1950 { 1951 struct ixgbe_hw *hw = &adapter->hw; 1952 unsigned int def_q = 0; 1953 unsigned int num_tcs = 0; 1954 unsigned int num_rx_queues = adapter->num_rx_queues; 1955 unsigned int num_tx_queues = adapter->num_tx_queues; 1956 int err; 1957 1958 spin_lock_bh(&adapter->mbx_lock); 1959 1960 /* fetch queue configuration from the PF */ 1961 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 1962 1963 spin_unlock_bh(&adapter->mbx_lock); 1964 1965 if (err) 1966 return err; 1967 1968 if (num_tcs > 1) { 1969 /* we need only one Tx queue */ 1970 num_tx_queues = 1; 1971 1972 /* update default Tx ring register index */ 1973 adapter->tx_ring[0]->reg_idx = def_q; 1974 1975 /* we need as many queues as traffic classes */ 1976 num_rx_queues = num_tcs; 1977 } 1978 1979 /* if we have a bad config abort request queue reset */ 1980 if ((adapter->num_rx_queues != num_rx_queues) || 1981 (adapter->num_tx_queues != num_tx_queues)) { 1982 /* force mailbox timeout to prevent further messages */ 1983 hw->mbx.timeout = 0; 1984 1985 /* wait for watchdog to come around and bail us out */ 1986 adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED; 1987 } 1988 1989 return 0; 1990 } 1991 1992 static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 1993 { 1994 ixgbevf_configure_dcb(adapter); 1995 1996 ixgbevf_set_rx_mode(adapter->netdev); 1997 1998 ixgbevf_restore_vlan(adapter); 1999 2000 ixgbevf_configure_tx(adapter); 2001 ixgbevf_configure_rx(adapter); 2002 } 2003 2004 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 2005 { 2006 /* Only save pre-reset stats if there are some */ 2007 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 2008 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 2009 adapter->stats.base_vfgprc; 2010 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 2011 adapter->stats.base_vfgptc; 2012 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 2013 adapter->stats.base_vfgorc; 2014 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 2015 adapter->stats.base_vfgotc; 2016 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 2017 adapter->stats.base_vfmprc; 2018 } 2019 } 2020 2021 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 2022 { 2023 struct ixgbe_hw *hw = &adapter->hw; 2024 2025 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2026 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2027 adapter->stats.last_vfgorc |= 2028 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2029 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2030 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2031 adapter->stats.last_vfgotc |= 2032 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2033 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2034 2035 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 2036 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 2037 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 2038 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 2039 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 2040 } 2041 2042 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) 2043 { 2044 struct ixgbe_hw *hw = &adapter->hw; 2045 int api[] = { ixgbe_mbox_api_12, 2046 ixgbe_mbox_api_11, 2047 ixgbe_mbox_api_10, 2048 ixgbe_mbox_api_unknown }; 2049 int err = 0, idx = 0; 2050 2051 spin_lock_bh(&adapter->mbx_lock); 2052 2053 while (api[idx] != ixgbe_mbox_api_unknown) { 2054 err = ixgbevf_negotiate_api_version(hw, api[idx]); 2055 if (!err) 2056 break; 2057 idx++; 2058 } 2059 2060 spin_unlock_bh(&adapter->mbx_lock); 2061 } 2062 2063 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 2064 { 2065 struct net_device *netdev = adapter->netdev; 2066 struct ixgbe_hw *hw = &adapter->hw; 2067 2068 ixgbevf_configure_msix(adapter); 2069 2070 spin_lock_bh(&adapter->mbx_lock); 2071 2072 if (is_valid_ether_addr(hw->mac.addr)) 2073 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 2074 else 2075 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 2076 2077 spin_unlock_bh(&adapter->mbx_lock); 2078 2079 smp_mb__before_atomic(); 2080 clear_bit(__IXGBEVF_DOWN, &adapter->state); 2081 ixgbevf_napi_enable_all(adapter); 2082 2083 /* clear any pending interrupts, may auto mask */ 2084 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2085 ixgbevf_irq_enable(adapter); 2086 2087 /* enable transmits */ 2088 netif_tx_start_all_queues(netdev); 2089 2090 ixgbevf_save_reset_stats(adapter); 2091 ixgbevf_init_last_counter_stats(adapter); 2092 2093 hw->mac.get_link_status = 1; 2094 mod_timer(&adapter->service_timer, jiffies); 2095 } 2096 2097 void ixgbevf_up(struct ixgbevf_adapter *adapter) 2098 { 2099 ixgbevf_configure(adapter); 2100 2101 ixgbevf_up_complete(adapter); 2102 } 2103 2104 /** 2105 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 2106 * @rx_ring: ring to free buffers from 2107 **/ 2108 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) 2109 { 2110 struct device *dev = rx_ring->dev; 2111 unsigned long size; 2112 unsigned int i; 2113 2114 /* Free Rx ring sk_buff */ 2115 if (rx_ring->skb) { 2116 dev_kfree_skb(rx_ring->skb); 2117 rx_ring->skb = NULL; 2118 } 2119 2120 /* ring already cleared, nothing to do */ 2121 if (!rx_ring->rx_buffer_info) 2122 return; 2123 2124 /* Free all the Rx ring pages */ 2125 for (i = 0; i < rx_ring->count; i++) { 2126 struct ixgbevf_rx_buffer *rx_buffer; 2127 2128 rx_buffer = &rx_ring->rx_buffer_info[i]; 2129 if (rx_buffer->dma) 2130 dma_unmap_page(dev, rx_buffer->dma, 2131 PAGE_SIZE, DMA_FROM_DEVICE); 2132 rx_buffer->dma = 0; 2133 if (rx_buffer->page) 2134 __free_page(rx_buffer->page); 2135 rx_buffer->page = NULL; 2136 } 2137 2138 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 2139 memset(rx_ring->rx_buffer_info, 0, size); 2140 2141 /* Zero out the descriptor ring */ 2142 memset(rx_ring->desc, 0, rx_ring->size); 2143 } 2144 2145 /** 2146 * ixgbevf_clean_tx_ring - Free Tx Buffers 2147 * @tx_ring: ring to be cleaned 2148 **/ 2149 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) 2150 { 2151 struct ixgbevf_tx_buffer *tx_buffer_info; 2152 unsigned long size; 2153 unsigned int i; 2154 2155 if (!tx_ring->tx_buffer_info) 2156 return; 2157 2158 /* Free all the Tx ring sk_buffs */ 2159 for (i = 0; i < tx_ring->count; i++) { 2160 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2161 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 2162 } 2163 2164 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2165 memset(tx_ring->tx_buffer_info, 0, size); 2166 2167 memset(tx_ring->desc, 0, tx_ring->size); 2168 } 2169 2170 /** 2171 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 2172 * @adapter: board private structure 2173 **/ 2174 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 2175 { 2176 int i; 2177 2178 for (i = 0; i < adapter->num_rx_queues; i++) 2179 ixgbevf_clean_rx_ring(adapter->rx_ring[i]); 2180 } 2181 2182 /** 2183 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 2184 * @adapter: board private structure 2185 **/ 2186 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 2187 { 2188 int i; 2189 2190 for (i = 0; i < adapter->num_tx_queues; i++) 2191 ixgbevf_clean_tx_ring(adapter->tx_ring[i]); 2192 } 2193 2194 void ixgbevf_down(struct ixgbevf_adapter *adapter) 2195 { 2196 struct net_device *netdev = adapter->netdev; 2197 struct ixgbe_hw *hw = &adapter->hw; 2198 int i; 2199 2200 /* signal that we are down to the interrupt handler */ 2201 if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state)) 2202 return; /* do nothing if already down */ 2203 2204 /* disable all enabled Rx queues */ 2205 for (i = 0; i < adapter->num_rx_queues; i++) 2206 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]); 2207 2208 usleep_range(10000, 20000); 2209 2210 netif_tx_stop_all_queues(netdev); 2211 2212 /* call carrier off first to avoid false dev_watchdog timeouts */ 2213 netif_carrier_off(netdev); 2214 netif_tx_disable(netdev); 2215 2216 ixgbevf_irq_disable(adapter); 2217 2218 ixgbevf_napi_disable_all(adapter); 2219 2220 del_timer_sync(&adapter->service_timer); 2221 2222 /* disable transmits in the hardware now that interrupts are off */ 2223 for (i = 0; i < adapter->num_tx_queues; i++) { 2224 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 2225 2226 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), 2227 IXGBE_TXDCTL_SWFLSH); 2228 } 2229 2230 if (!pci_channel_offline(adapter->pdev)) 2231 ixgbevf_reset(adapter); 2232 2233 ixgbevf_clean_all_tx_rings(adapter); 2234 ixgbevf_clean_all_rx_rings(adapter); 2235 } 2236 2237 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 2238 { 2239 WARN_ON(in_interrupt()); 2240 2241 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 2242 msleep(1); 2243 2244 ixgbevf_down(adapter); 2245 ixgbevf_up(adapter); 2246 2247 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 2248 } 2249 2250 void ixgbevf_reset(struct ixgbevf_adapter *adapter) 2251 { 2252 struct ixgbe_hw *hw = &adapter->hw; 2253 struct net_device *netdev = adapter->netdev; 2254 2255 if (hw->mac.ops.reset_hw(hw)) { 2256 hw_dbg(hw, "PF still resetting\n"); 2257 } else { 2258 hw->mac.ops.init_hw(hw); 2259 ixgbevf_negotiate_api(adapter); 2260 } 2261 2262 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 2263 memcpy(netdev->dev_addr, adapter->hw.mac.addr, 2264 netdev->addr_len); 2265 memcpy(netdev->perm_addr, adapter->hw.mac.addr, 2266 netdev->addr_len); 2267 } 2268 2269 adapter->last_reset = jiffies; 2270 } 2271 2272 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 2273 int vectors) 2274 { 2275 int vector_threshold; 2276 2277 /* We'll want at least 2 (vector_threshold): 2278 * 1) TxQ[0] + RxQ[0] handler 2279 * 2) Other (Link Status Change, etc.) 2280 */ 2281 vector_threshold = MIN_MSIX_COUNT; 2282 2283 /* The more we get, the more we will assign to Tx/Rx Cleanup 2284 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 2285 * Right now, we simply care about how many we'll get; we'll 2286 * set them up later while requesting irq's. 2287 */ 2288 vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 2289 vector_threshold, vectors); 2290 2291 if (vectors < 0) { 2292 dev_err(&adapter->pdev->dev, 2293 "Unable to allocate MSI-X interrupts\n"); 2294 kfree(adapter->msix_entries); 2295 adapter->msix_entries = NULL; 2296 return vectors; 2297 } 2298 2299 /* Adjust for only the vectors we'll use, which is minimum 2300 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 2301 * vectors we were allocated. 2302 */ 2303 adapter->num_msix_vectors = vectors; 2304 2305 return 0; 2306 } 2307 2308 /** 2309 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent 2310 * @adapter: board private structure to initialize 2311 * 2312 * This is the top level queue allocation routine. The order here is very 2313 * important, starting with the "most" number of features turned on at once, 2314 * and ending with the smallest set of features. This way large combinations 2315 * can be allocated if they're turned on, and smaller combinations are the 2316 * fallthrough conditions. 2317 * 2318 **/ 2319 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 2320 { 2321 struct ixgbe_hw *hw = &adapter->hw; 2322 unsigned int def_q = 0; 2323 unsigned int num_tcs = 0; 2324 int err; 2325 2326 /* Start with base case */ 2327 adapter->num_rx_queues = 1; 2328 adapter->num_tx_queues = 1; 2329 2330 spin_lock_bh(&adapter->mbx_lock); 2331 2332 /* fetch queue configuration from the PF */ 2333 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2334 2335 spin_unlock_bh(&adapter->mbx_lock); 2336 2337 if (err) 2338 return; 2339 2340 /* we need as many queues as traffic classes */ 2341 if (num_tcs > 1) { 2342 adapter->num_rx_queues = num_tcs; 2343 } else { 2344 u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES); 2345 2346 switch (hw->api_version) { 2347 case ixgbe_mbox_api_11: 2348 case ixgbe_mbox_api_12: 2349 adapter->num_rx_queues = rss; 2350 adapter->num_tx_queues = rss; 2351 default: 2352 break; 2353 } 2354 } 2355 } 2356 2357 /** 2358 * ixgbevf_alloc_queues - Allocate memory for all rings 2359 * @adapter: board private structure to initialize 2360 * 2361 * We allocate one ring per queue at run-time since we don't know the 2362 * number of queues at compile-time. The polling_netdev array is 2363 * intended for Multiqueue, but should work fine with a single queue. 2364 **/ 2365 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) 2366 { 2367 struct ixgbevf_ring *ring; 2368 int rx = 0, tx = 0; 2369 2370 for (; tx < adapter->num_tx_queues; tx++) { 2371 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 2372 if (!ring) 2373 goto err_allocation; 2374 2375 ring->dev = &adapter->pdev->dev; 2376 ring->netdev = adapter->netdev; 2377 ring->count = adapter->tx_ring_count; 2378 ring->queue_index = tx; 2379 ring->reg_idx = tx; 2380 2381 adapter->tx_ring[tx] = ring; 2382 } 2383 2384 for (; rx < adapter->num_rx_queues; rx++) { 2385 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 2386 if (!ring) 2387 goto err_allocation; 2388 2389 ring->dev = &adapter->pdev->dev; 2390 ring->netdev = adapter->netdev; 2391 2392 ring->count = adapter->rx_ring_count; 2393 ring->queue_index = rx; 2394 ring->reg_idx = rx; 2395 2396 adapter->rx_ring[rx] = ring; 2397 } 2398 2399 return 0; 2400 2401 err_allocation: 2402 while (tx) { 2403 kfree(adapter->tx_ring[--tx]); 2404 adapter->tx_ring[tx] = NULL; 2405 } 2406 2407 while (rx) { 2408 kfree(adapter->rx_ring[--rx]); 2409 adapter->rx_ring[rx] = NULL; 2410 } 2411 return -ENOMEM; 2412 } 2413 2414 /** 2415 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 2416 * @adapter: board private structure to initialize 2417 * 2418 * Attempt to configure the interrupts using the best available 2419 * capabilities of the hardware and the kernel. 2420 **/ 2421 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 2422 { 2423 struct net_device *netdev = adapter->netdev; 2424 int err = 0; 2425 int vector, v_budget; 2426 2427 /* It's easy to be greedy for MSI-X vectors, but it really 2428 * doesn't do us much good if we have a lot more vectors 2429 * than CPU's. So let's be conservative and only ask for 2430 * (roughly) the same number of vectors as there are CPU's. 2431 * The default is to use pairs of vectors. 2432 */ 2433 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); 2434 v_budget = min_t(int, v_budget, num_online_cpus()); 2435 v_budget += NON_Q_VECTORS; 2436 2437 /* A failure in MSI-X entry allocation isn't fatal, but it does 2438 * mean we disable MSI-X capabilities of the adapter. 2439 */ 2440 adapter->msix_entries = kcalloc(v_budget, 2441 sizeof(struct msix_entry), GFP_KERNEL); 2442 if (!adapter->msix_entries) { 2443 err = -ENOMEM; 2444 goto out; 2445 } 2446 2447 for (vector = 0; vector < v_budget; vector++) 2448 adapter->msix_entries[vector].entry = vector; 2449 2450 err = ixgbevf_acquire_msix_vectors(adapter, v_budget); 2451 if (err) 2452 goto out; 2453 2454 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 2455 if (err) 2456 goto out; 2457 2458 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 2459 2460 out: 2461 return err; 2462 } 2463 2464 /** 2465 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 2466 * @adapter: board private structure to initialize 2467 * 2468 * We allocate one q_vector per queue interrupt. If allocation fails we 2469 * return -ENOMEM. 2470 **/ 2471 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 2472 { 2473 int q_idx, num_q_vectors; 2474 struct ixgbevf_q_vector *q_vector; 2475 2476 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2477 2478 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2479 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL); 2480 if (!q_vector) 2481 goto err_out; 2482 q_vector->adapter = adapter; 2483 q_vector->v_idx = q_idx; 2484 netif_napi_add(adapter->netdev, &q_vector->napi, 2485 ixgbevf_poll, 64); 2486 adapter->q_vector[q_idx] = q_vector; 2487 } 2488 2489 return 0; 2490 2491 err_out: 2492 while (q_idx) { 2493 q_idx--; 2494 q_vector = adapter->q_vector[q_idx]; 2495 #ifdef CONFIG_NET_RX_BUSY_POLL 2496 napi_hash_del(&q_vector->napi); 2497 #endif 2498 netif_napi_del(&q_vector->napi); 2499 kfree(q_vector); 2500 adapter->q_vector[q_idx] = NULL; 2501 } 2502 return -ENOMEM; 2503 } 2504 2505 /** 2506 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 2507 * @adapter: board private structure to initialize 2508 * 2509 * This function frees the memory allocated to the q_vectors. In addition if 2510 * NAPI is enabled it will delete any references to the NAPI struct prior 2511 * to freeing the q_vector. 2512 **/ 2513 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 2514 { 2515 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2516 2517 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2518 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; 2519 2520 adapter->q_vector[q_idx] = NULL; 2521 #ifdef CONFIG_NET_RX_BUSY_POLL 2522 napi_hash_del(&q_vector->napi); 2523 #endif 2524 netif_napi_del(&q_vector->napi); 2525 kfree(q_vector); 2526 } 2527 } 2528 2529 /** 2530 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 2531 * @adapter: board private structure 2532 * 2533 **/ 2534 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 2535 { 2536 pci_disable_msix(adapter->pdev); 2537 kfree(adapter->msix_entries); 2538 adapter->msix_entries = NULL; 2539 } 2540 2541 /** 2542 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 2543 * @adapter: board private structure to initialize 2544 * 2545 **/ 2546 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 2547 { 2548 int err; 2549 2550 /* Number of supported queues */ 2551 ixgbevf_set_num_queues(adapter); 2552 2553 err = ixgbevf_set_interrupt_capability(adapter); 2554 if (err) { 2555 hw_dbg(&adapter->hw, 2556 "Unable to setup interrupt capabilities\n"); 2557 goto err_set_interrupt; 2558 } 2559 2560 err = ixgbevf_alloc_q_vectors(adapter); 2561 if (err) { 2562 hw_dbg(&adapter->hw, "Unable to allocate memory for queue vectors\n"); 2563 goto err_alloc_q_vectors; 2564 } 2565 2566 err = ixgbevf_alloc_queues(adapter); 2567 if (err) { 2568 pr_err("Unable to allocate memory for queues\n"); 2569 goto err_alloc_queues; 2570 } 2571 2572 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u\n", 2573 (adapter->num_rx_queues > 1) ? "Enabled" : 2574 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues); 2575 2576 set_bit(__IXGBEVF_DOWN, &adapter->state); 2577 2578 return 0; 2579 err_alloc_queues: 2580 ixgbevf_free_q_vectors(adapter); 2581 err_alloc_q_vectors: 2582 ixgbevf_reset_interrupt_capability(adapter); 2583 err_set_interrupt: 2584 return err; 2585 } 2586 2587 /** 2588 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings 2589 * @adapter: board private structure to clear interrupt scheme on 2590 * 2591 * We go through and clear interrupt specific resources and reset the structure 2592 * to pre-load conditions 2593 **/ 2594 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) 2595 { 2596 int i; 2597 2598 for (i = 0; i < adapter->num_tx_queues; i++) { 2599 kfree(adapter->tx_ring[i]); 2600 adapter->tx_ring[i] = NULL; 2601 } 2602 for (i = 0; i < adapter->num_rx_queues; i++) { 2603 kfree(adapter->rx_ring[i]); 2604 adapter->rx_ring[i] = NULL; 2605 } 2606 2607 adapter->num_tx_queues = 0; 2608 adapter->num_rx_queues = 0; 2609 2610 ixgbevf_free_q_vectors(adapter); 2611 ixgbevf_reset_interrupt_capability(adapter); 2612 } 2613 2614 /** 2615 * ixgbevf_sw_init - Initialize general software structures 2616 * @adapter: board private structure to initialize 2617 * 2618 * ixgbevf_sw_init initializes the Adapter private data structure. 2619 * Fields are initialized based on PCI device information and 2620 * OS network device settings (MTU size). 2621 **/ 2622 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 2623 { 2624 struct ixgbe_hw *hw = &adapter->hw; 2625 struct pci_dev *pdev = adapter->pdev; 2626 struct net_device *netdev = adapter->netdev; 2627 int err; 2628 2629 /* PCI config space info */ 2630 hw->vendor_id = pdev->vendor; 2631 hw->device_id = pdev->device; 2632 hw->revision_id = pdev->revision; 2633 hw->subsystem_vendor_id = pdev->subsystem_vendor; 2634 hw->subsystem_device_id = pdev->subsystem_device; 2635 2636 hw->mbx.ops.init_params(hw); 2637 2638 /* assume legacy case in which PF would only give VF 2 queues */ 2639 hw->mac.max_tx_queues = 2; 2640 hw->mac.max_rx_queues = 2; 2641 2642 /* lock to protect mailbox accesses */ 2643 spin_lock_init(&adapter->mbx_lock); 2644 2645 err = hw->mac.ops.reset_hw(hw); 2646 if (err) { 2647 dev_info(&pdev->dev, 2648 "PF still in reset state. Is the PF interface up?\n"); 2649 } else { 2650 err = hw->mac.ops.init_hw(hw); 2651 if (err) { 2652 pr_err("init_shared_code failed: %d\n", err); 2653 goto out; 2654 } 2655 ixgbevf_negotiate_api(adapter); 2656 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr); 2657 if (err) 2658 dev_info(&pdev->dev, "Error reading MAC address\n"); 2659 else if (is_zero_ether_addr(adapter->hw.mac.addr)) 2660 dev_info(&pdev->dev, 2661 "MAC address not assigned by administrator.\n"); 2662 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 2663 } 2664 2665 if (!is_valid_ether_addr(netdev->dev_addr)) { 2666 dev_info(&pdev->dev, "Assigning random MAC address\n"); 2667 eth_hw_addr_random(netdev); 2668 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len); 2669 } 2670 2671 /* Enable dynamic interrupt throttling rates */ 2672 adapter->rx_itr_setting = 1; 2673 adapter->tx_itr_setting = 1; 2674 2675 /* set default ring sizes */ 2676 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 2677 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 2678 2679 set_bit(__IXGBEVF_DOWN, &adapter->state); 2680 return 0; 2681 2682 out: 2683 return err; 2684 } 2685 2686 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 2687 { \ 2688 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 2689 if (current_counter < last_counter) \ 2690 counter += 0x100000000LL; \ 2691 last_counter = current_counter; \ 2692 counter &= 0xFFFFFFFF00000000LL; \ 2693 counter |= current_counter; \ 2694 } 2695 2696 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 2697 { \ 2698 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 2699 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 2700 u64 current_counter = (current_counter_msb << 32) | \ 2701 current_counter_lsb; \ 2702 if (current_counter < last_counter) \ 2703 counter += 0x1000000000LL; \ 2704 last_counter = current_counter; \ 2705 counter &= 0xFFFFFFF000000000LL; \ 2706 counter |= current_counter; \ 2707 } 2708 /** 2709 * ixgbevf_update_stats - Update the board statistics counters. 2710 * @adapter: board private structure 2711 **/ 2712 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 2713 { 2714 struct ixgbe_hw *hw = &adapter->hw; 2715 int i; 2716 2717 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2718 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2719 return; 2720 2721 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 2722 adapter->stats.vfgprc); 2723 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 2724 adapter->stats.vfgptc); 2725 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2726 adapter->stats.last_vfgorc, 2727 adapter->stats.vfgorc); 2728 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2729 adapter->stats.last_vfgotc, 2730 adapter->stats.vfgotc); 2731 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 2732 adapter->stats.vfmprc); 2733 2734 for (i = 0; i < adapter->num_rx_queues; i++) { 2735 adapter->hw_csum_rx_error += 2736 adapter->rx_ring[i]->hw_csum_rx_error; 2737 adapter->rx_ring[i]->hw_csum_rx_error = 0; 2738 } 2739 } 2740 2741 /** 2742 * ixgbevf_service_timer - Timer Call-back 2743 * @data: pointer to adapter cast into an unsigned long 2744 **/ 2745 static void ixgbevf_service_timer(unsigned long data) 2746 { 2747 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data; 2748 2749 /* Reset the timer */ 2750 mod_timer(&adapter->service_timer, (HZ * 2) + jiffies); 2751 2752 ixgbevf_service_event_schedule(adapter); 2753 } 2754 2755 static void ixgbevf_reset_subtask(struct ixgbevf_adapter *adapter) 2756 { 2757 if (!(adapter->flags & IXGBEVF_FLAG_RESET_REQUESTED)) 2758 return; 2759 2760 adapter->flags &= ~IXGBEVF_FLAG_RESET_REQUESTED; 2761 2762 /* If we're already down or resetting, just bail */ 2763 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2764 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2765 return; 2766 2767 adapter->tx_timeout_count++; 2768 2769 ixgbevf_reinit_locked(adapter); 2770 } 2771 2772 /** 2773 * ixgbevf_check_hang_subtask - check for hung queues and dropped interrupts 2774 * @adapter: pointer to the device adapter structure 2775 * 2776 * This function serves two purposes. First it strobes the interrupt lines 2777 * in order to make certain interrupts are occurring. Secondly it sets the 2778 * bits needed to check for TX hangs. As a result we should immediately 2779 * determine if a hang has occurred. 2780 **/ 2781 static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter) 2782 { 2783 struct ixgbe_hw *hw = &adapter->hw; 2784 u32 eics = 0; 2785 int i; 2786 2787 /* If we're down or resetting, just bail */ 2788 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2789 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2790 return; 2791 2792 /* Force detection of hung controller */ 2793 if (netif_carrier_ok(adapter->netdev)) { 2794 for (i = 0; i < adapter->num_tx_queues; i++) 2795 set_check_for_tx_hang(adapter->tx_ring[i]); 2796 } 2797 2798 /* get one bit for every active Tx/Rx interrupt vector */ 2799 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 2800 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 2801 2802 if (qv->rx.ring || qv->tx.ring) 2803 eics |= 1 << i; 2804 } 2805 2806 /* Cause software interrupt to ensure rings are cleaned */ 2807 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics); 2808 } 2809 2810 /** 2811 * ixgbevf_watchdog_update_link - update the link status 2812 * @adapter: pointer to the device adapter structure 2813 **/ 2814 static void ixgbevf_watchdog_update_link(struct ixgbevf_adapter *adapter) 2815 { 2816 struct ixgbe_hw *hw = &adapter->hw; 2817 u32 link_speed = adapter->link_speed; 2818 bool link_up = adapter->link_up; 2819 s32 err; 2820 2821 spin_lock_bh(&adapter->mbx_lock); 2822 2823 err = hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 2824 2825 spin_unlock_bh(&adapter->mbx_lock); 2826 2827 /* if check for link returns error we will need to reset */ 2828 if (err && time_after(jiffies, adapter->last_reset + (10 * HZ))) { 2829 adapter->flags |= IXGBEVF_FLAG_RESET_REQUESTED; 2830 link_up = false; 2831 } 2832 2833 adapter->link_up = link_up; 2834 adapter->link_speed = link_speed; 2835 } 2836 2837 /** 2838 * ixgbevf_watchdog_link_is_up - update netif_carrier status and 2839 * print link up message 2840 * @adapter: pointer to the device adapter structure 2841 **/ 2842 static void ixgbevf_watchdog_link_is_up(struct ixgbevf_adapter *adapter) 2843 { 2844 struct net_device *netdev = adapter->netdev; 2845 2846 /* only continue if link was previously down */ 2847 if (netif_carrier_ok(netdev)) 2848 return; 2849 2850 dev_info(&adapter->pdev->dev, "NIC Link is Up %s\n", 2851 (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 2852 "10 Gbps" : 2853 (adapter->link_speed == IXGBE_LINK_SPEED_1GB_FULL) ? 2854 "1 Gbps" : 2855 (adapter->link_speed == IXGBE_LINK_SPEED_100_FULL) ? 2856 "100 Mbps" : 2857 "unknown speed"); 2858 2859 netif_carrier_on(netdev); 2860 } 2861 2862 /** 2863 * ixgbevf_watchdog_link_is_down - update netif_carrier status and 2864 * print link down message 2865 * @adapter: pointer to the adapter structure 2866 **/ 2867 static void ixgbevf_watchdog_link_is_down(struct ixgbevf_adapter *adapter) 2868 { 2869 struct net_device *netdev = adapter->netdev; 2870 2871 adapter->link_speed = 0; 2872 2873 /* only continue if link was up previously */ 2874 if (!netif_carrier_ok(netdev)) 2875 return; 2876 2877 dev_info(&adapter->pdev->dev, "NIC Link is Down\n"); 2878 2879 netif_carrier_off(netdev); 2880 } 2881 2882 /** 2883 * ixgbevf_watchdog_subtask - worker thread to bring link up 2884 * @work: pointer to work_struct containing our data 2885 **/ 2886 static void ixgbevf_watchdog_subtask(struct ixgbevf_adapter *adapter) 2887 { 2888 /* if interface is down do nothing */ 2889 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2890 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2891 return; 2892 2893 ixgbevf_watchdog_update_link(adapter); 2894 2895 if (adapter->link_up) 2896 ixgbevf_watchdog_link_is_up(adapter); 2897 else 2898 ixgbevf_watchdog_link_is_down(adapter); 2899 2900 ixgbevf_update_stats(adapter); 2901 } 2902 2903 /** 2904 * ixgbevf_service_task - manages and runs subtasks 2905 * @work: pointer to work_struct containing our data 2906 **/ 2907 static void ixgbevf_service_task(struct work_struct *work) 2908 { 2909 struct ixgbevf_adapter *adapter = container_of(work, 2910 struct ixgbevf_adapter, 2911 service_task); 2912 struct ixgbe_hw *hw = &adapter->hw; 2913 2914 if (IXGBE_REMOVED(hw->hw_addr)) { 2915 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 2916 rtnl_lock(); 2917 ixgbevf_down(adapter); 2918 rtnl_unlock(); 2919 } 2920 return; 2921 } 2922 2923 ixgbevf_queue_reset_subtask(adapter); 2924 ixgbevf_reset_subtask(adapter); 2925 ixgbevf_watchdog_subtask(adapter); 2926 ixgbevf_check_hang_subtask(adapter); 2927 2928 ixgbevf_service_event_complete(adapter); 2929 } 2930 2931 /** 2932 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 2933 * @tx_ring: Tx descriptor ring for a specific queue 2934 * 2935 * Free all transmit software resources 2936 **/ 2937 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) 2938 { 2939 ixgbevf_clean_tx_ring(tx_ring); 2940 2941 vfree(tx_ring->tx_buffer_info); 2942 tx_ring->tx_buffer_info = NULL; 2943 2944 /* if not set, then don't free */ 2945 if (!tx_ring->desc) 2946 return; 2947 2948 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, 2949 tx_ring->dma); 2950 2951 tx_ring->desc = NULL; 2952 } 2953 2954 /** 2955 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 2956 * @adapter: board private structure 2957 * 2958 * Free all transmit software resources 2959 **/ 2960 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 2961 { 2962 int i; 2963 2964 for (i = 0; i < adapter->num_tx_queues; i++) 2965 if (adapter->tx_ring[i]->desc) 2966 ixgbevf_free_tx_resources(adapter->tx_ring[i]); 2967 } 2968 2969 /** 2970 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 2971 * @tx_ring: Tx descriptor ring (for a specific queue) to setup 2972 * 2973 * Return 0 on success, negative on failure 2974 **/ 2975 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) 2976 { 2977 int size; 2978 2979 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2980 tx_ring->tx_buffer_info = vzalloc(size); 2981 if (!tx_ring->tx_buffer_info) 2982 goto err; 2983 2984 /* round up to nearest 4K */ 2985 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 2986 tx_ring->size = ALIGN(tx_ring->size, 4096); 2987 2988 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, 2989 &tx_ring->dma, GFP_KERNEL); 2990 if (!tx_ring->desc) 2991 goto err; 2992 2993 return 0; 2994 2995 err: 2996 vfree(tx_ring->tx_buffer_info); 2997 tx_ring->tx_buffer_info = NULL; 2998 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit descriptor ring\n"); 2999 return -ENOMEM; 3000 } 3001 3002 /** 3003 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 3004 * @adapter: board private structure 3005 * 3006 * If this function returns with an error, then it's possible one or 3007 * more of the rings is populated (while the rest are not). It is the 3008 * callers duty to clean those orphaned rings. 3009 * 3010 * Return 0 on success, negative on failure 3011 **/ 3012 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 3013 { 3014 int i, err = 0; 3015 3016 for (i = 0; i < adapter->num_tx_queues; i++) { 3017 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]); 3018 if (!err) 3019 continue; 3020 hw_dbg(&adapter->hw, "Allocation for Tx Queue %u failed\n", i); 3021 break; 3022 } 3023 3024 return err; 3025 } 3026 3027 /** 3028 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 3029 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 3030 * 3031 * Returns 0 on success, negative on failure 3032 **/ 3033 int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring) 3034 { 3035 int size; 3036 3037 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 3038 rx_ring->rx_buffer_info = vzalloc(size); 3039 if (!rx_ring->rx_buffer_info) 3040 goto err; 3041 3042 /* Round up to nearest 4K */ 3043 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 3044 rx_ring->size = ALIGN(rx_ring->size, 4096); 3045 3046 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, 3047 &rx_ring->dma, GFP_KERNEL); 3048 3049 if (!rx_ring->desc) 3050 goto err; 3051 3052 return 0; 3053 err: 3054 vfree(rx_ring->rx_buffer_info); 3055 rx_ring->rx_buffer_info = NULL; 3056 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); 3057 return -ENOMEM; 3058 } 3059 3060 /** 3061 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 3062 * @adapter: board private structure 3063 * 3064 * If this function returns with an error, then it's possible one or 3065 * more of the rings is populated (while the rest are not). It is the 3066 * callers duty to clean those orphaned rings. 3067 * 3068 * Return 0 on success, negative on failure 3069 **/ 3070 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 3071 { 3072 int i, err = 0; 3073 3074 for (i = 0; i < adapter->num_rx_queues; i++) { 3075 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]); 3076 if (!err) 3077 continue; 3078 hw_dbg(&adapter->hw, "Allocation for Rx Queue %u failed\n", i); 3079 break; 3080 } 3081 return err; 3082 } 3083 3084 /** 3085 * ixgbevf_free_rx_resources - Free Rx Resources 3086 * @rx_ring: ring to clean the resources from 3087 * 3088 * Free all receive software resources 3089 **/ 3090 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) 3091 { 3092 ixgbevf_clean_rx_ring(rx_ring); 3093 3094 vfree(rx_ring->rx_buffer_info); 3095 rx_ring->rx_buffer_info = NULL; 3096 3097 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, 3098 rx_ring->dma); 3099 3100 rx_ring->desc = NULL; 3101 } 3102 3103 /** 3104 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 3105 * @adapter: board private structure 3106 * 3107 * Free all receive software resources 3108 **/ 3109 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 3110 { 3111 int i; 3112 3113 for (i = 0; i < adapter->num_rx_queues; i++) 3114 if (adapter->rx_ring[i]->desc) 3115 ixgbevf_free_rx_resources(adapter->rx_ring[i]); 3116 } 3117 3118 /** 3119 * ixgbevf_open - Called when a network interface is made active 3120 * @netdev: network interface device structure 3121 * 3122 * Returns 0 on success, negative value on failure 3123 * 3124 * The open entry point is called when a network interface is made 3125 * active by the system (IFF_UP). At this point all resources needed 3126 * for transmit and receive operations are allocated, the interrupt 3127 * handler is registered with the OS, the watchdog timer is started, 3128 * and the stack is notified that the interface is ready. 3129 **/ 3130 static int ixgbevf_open(struct net_device *netdev) 3131 { 3132 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3133 struct ixgbe_hw *hw = &adapter->hw; 3134 int err; 3135 3136 /* A previous failure to open the device because of a lack of 3137 * available MSIX vector resources may have reset the number 3138 * of msix vectors variable to zero. The only way to recover 3139 * is to unload/reload the driver and hope that the system has 3140 * been able to recover some MSIX vector resources. 3141 */ 3142 if (!adapter->num_msix_vectors) 3143 return -ENOMEM; 3144 3145 if (hw->adapter_stopped) { 3146 ixgbevf_reset(adapter); 3147 /* if adapter is still stopped then PF isn't up and 3148 * the VF can't start. 3149 */ 3150 if (hw->adapter_stopped) { 3151 err = IXGBE_ERR_MBX; 3152 pr_err("Unable to start - perhaps the PF Driver isn't up yet\n"); 3153 goto err_setup_reset; 3154 } 3155 } 3156 3157 /* disallow open during test */ 3158 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 3159 return -EBUSY; 3160 3161 netif_carrier_off(netdev); 3162 3163 /* allocate transmit descriptors */ 3164 err = ixgbevf_setup_all_tx_resources(adapter); 3165 if (err) 3166 goto err_setup_tx; 3167 3168 /* allocate receive descriptors */ 3169 err = ixgbevf_setup_all_rx_resources(adapter); 3170 if (err) 3171 goto err_setup_rx; 3172 3173 ixgbevf_configure(adapter); 3174 3175 /* Map the Tx/Rx rings to the vectors we were allotted. 3176 * if request_irq will be called in this function map_rings 3177 * must be called *before* up_complete 3178 */ 3179 ixgbevf_map_rings_to_vectors(adapter); 3180 3181 err = ixgbevf_request_irq(adapter); 3182 if (err) 3183 goto err_req_irq; 3184 3185 ixgbevf_up_complete(adapter); 3186 3187 return 0; 3188 3189 err_req_irq: 3190 ixgbevf_down(adapter); 3191 err_setup_rx: 3192 ixgbevf_free_all_rx_resources(adapter); 3193 err_setup_tx: 3194 ixgbevf_free_all_tx_resources(adapter); 3195 ixgbevf_reset(adapter); 3196 3197 err_setup_reset: 3198 3199 return err; 3200 } 3201 3202 /** 3203 * ixgbevf_close - Disables a network interface 3204 * @netdev: network interface device structure 3205 * 3206 * Returns 0, this is not allowed to fail 3207 * 3208 * The close entry point is called when an interface is de-activated 3209 * by the OS. The hardware is still under the drivers control, but 3210 * needs to be disabled. A global MAC reset is issued to stop the 3211 * hardware, and all transmit and receive resources are freed. 3212 **/ 3213 static int ixgbevf_close(struct net_device *netdev) 3214 { 3215 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3216 3217 ixgbevf_down(adapter); 3218 ixgbevf_free_irq(adapter); 3219 3220 ixgbevf_free_all_tx_resources(adapter); 3221 ixgbevf_free_all_rx_resources(adapter); 3222 3223 return 0; 3224 } 3225 3226 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter) 3227 { 3228 struct net_device *dev = adapter->netdev; 3229 3230 if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED)) 3231 return; 3232 3233 adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED; 3234 3235 /* if interface is down do nothing */ 3236 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3237 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3238 return; 3239 3240 /* Hardware has to reinitialize queues and interrupts to 3241 * match packet buffer alignment. Unfortunately, the 3242 * hardware is not flexible enough to do this dynamically. 3243 */ 3244 if (netif_running(dev)) 3245 ixgbevf_close(dev); 3246 3247 ixgbevf_clear_interrupt_scheme(adapter); 3248 ixgbevf_init_interrupt_scheme(adapter); 3249 3250 if (netif_running(dev)) 3251 ixgbevf_open(dev); 3252 } 3253 3254 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, 3255 u32 vlan_macip_lens, u32 type_tucmd, 3256 u32 mss_l4len_idx) 3257 { 3258 struct ixgbe_adv_tx_context_desc *context_desc; 3259 u16 i = tx_ring->next_to_use; 3260 3261 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i); 3262 3263 i++; 3264 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 3265 3266 /* set bits to identify this as an advanced context descriptor */ 3267 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3268 3269 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 3270 context_desc->seqnum_seed = 0; 3271 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 3272 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 3273 } 3274 3275 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, 3276 struct ixgbevf_tx_buffer *first, 3277 u8 *hdr_len) 3278 { 3279 struct sk_buff *skb = first->skb; 3280 u32 vlan_macip_lens, type_tucmd; 3281 u32 mss_l4len_idx, l4len; 3282 int err; 3283 3284 if (skb->ip_summed != CHECKSUM_PARTIAL) 3285 return 0; 3286 3287 if (!skb_is_gso(skb)) 3288 return 0; 3289 3290 err = skb_cow_head(skb, 0); 3291 if (err < 0) 3292 return err; 3293 3294 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 3295 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3296 3297 if (first->protocol == htons(ETH_P_IP)) { 3298 struct iphdr *iph = ip_hdr(skb); 3299 3300 iph->tot_len = 0; 3301 iph->check = 0; 3302 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 3303 iph->daddr, 0, 3304 IPPROTO_TCP, 3305 0); 3306 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3307 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3308 IXGBE_TX_FLAGS_CSUM | 3309 IXGBE_TX_FLAGS_IPV4; 3310 } else if (skb_is_gso_v6(skb)) { 3311 ipv6_hdr(skb)->payload_len = 0; 3312 tcp_hdr(skb)->check = 3313 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 3314 &ipv6_hdr(skb)->daddr, 3315 0, IPPROTO_TCP, 0); 3316 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3317 IXGBE_TX_FLAGS_CSUM; 3318 } 3319 3320 /* compute header lengths */ 3321 l4len = tcp_hdrlen(skb); 3322 *hdr_len += l4len; 3323 *hdr_len = skb_transport_offset(skb) + l4len; 3324 3325 /* update GSO size and bytecount with header size */ 3326 first->gso_segs = skb_shinfo(skb)->gso_segs; 3327 first->bytecount += (first->gso_segs - 1) * *hdr_len; 3328 3329 /* mss_l4len_id: use 1 as index for TSO */ 3330 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; 3331 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 3332 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT; 3333 3334 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 3335 vlan_macip_lens = skb_network_header_len(skb); 3336 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3337 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3338 3339 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3340 type_tucmd, mss_l4len_idx); 3341 3342 return 1; 3343 } 3344 3345 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, 3346 struct ixgbevf_tx_buffer *first) 3347 { 3348 struct sk_buff *skb = first->skb; 3349 u32 vlan_macip_lens = 0; 3350 u32 mss_l4len_idx = 0; 3351 u32 type_tucmd = 0; 3352 3353 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3354 u8 l4_hdr = 0; 3355 3356 switch (first->protocol) { 3357 case htons(ETH_P_IP): 3358 vlan_macip_lens |= skb_network_header_len(skb); 3359 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3360 l4_hdr = ip_hdr(skb)->protocol; 3361 break; 3362 case htons(ETH_P_IPV6): 3363 vlan_macip_lens |= skb_network_header_len(skb); 3364 l4_hdr = ipv6_hdr(skb)->nexthdr; 3365 break; 3366 default: 3367 if (unlikely(net_ratelimit())) { 3368 dev_warn(tx_ring->dev, 3369 "partial checksum but proto=%x!\n", 3370 first->protocol); 3371 } 3372 break; 3373 } 3374 3375 switch (l4_hdr) { 3376 case IPPROTO_TCP: 3377 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3378 mss_l4len_idx = tcp_hdrlen(skb) << 3379 IXGBE_ADVTXD_L4LEN_SHIFT; 3380 break; 3381 case IPPROTO_SCTP: 3382 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 3383 mss_l4len_idx = sizeof(struct sctphdr) << 3384 IXGBE_ADVTXD_L4LEN_SHIFT; 3385 break; 3386 case IPPROTO_UDP: 3387 mss_l4len_idx = sizeof(struct udphdr) << 3388 IXGBE_ADVTXD_L4LEN_SHIFT; 3389 break; 3390 default: 3391 if (unlikely(net_ratelimit())) { 3392 dev_warn(tx_ring->dev, 3393 "partial checksum but l4 proto=%x!\n", 3394 l4_hdr); 3395 } 3396 break; 3397 } 3398 3399 /* update TX checksum flag */ 3400 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 3401 } 3402 3403 /* vlan_macip_lens: MACLEN, VLAN tag */ 3404 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3405 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3406 3407 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3408 type_tucmd, mss_l4len_idx); 3409 } 3410 3411 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags) 3412 { 3413 /* set type for advanced descriptor with frame checksum insertion */ 3414 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | 3415 IXGBE_ADVTXD_DCMD_IFCS | 3416 IXGBE_ADVTXD_DCMD_DEXT); 3417 3418 /* set HW VLAN bit if VLAN is present */ 3419 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 3420 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); 3421 3422 /* set segmentation enable bits for TSO/FSO */ 3423 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3424 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); 3425 3426 return cmd_type; 3427 } 3428 3429 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 3430 u32 tx_flags, unsigned int paylen) 3431 { 3432 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT); 3433 3434 /* enable L4 checksum for TSO and TX checksum offload */ 3435 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 3436 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM); 3437 3438 /* enble IPv4 checksum for TSO */ 3439 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 3440 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM); 3441 3442 /* use index 1 context for TSO/FSO/FCOE */ 3443 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3444 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT); 3445 3446 /* Check Context must be set if Tx switch is enabled, which it 3447 * always is for case where virtual functions are running 3448 */ 3449 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC); 3450 3451 tx_desc->read.olinfo_status = olinfo_status; 3452 } 3453 3454 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, 3455 struct ixgbevf_tx_buffer *first, 3456 const u8 hdr_len) 3457 { 3458 dma_addr_t dma; 3459 struct sk_buff *skb = first->skb; 3460 struct ixgbevf_tx_buffer *tx_buffer; 3461 union ixgbe_adv_tx_desc *tx_desc; 3462 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 3463 unsigned int data_len = skb->data_len; 3464 unsigned int size = skb_headlen(skb); 3465 unsigned int paylen = skb->len - hdr_len; 3466 u32 tx_flags = first->tx_flags; 3467 __le32 cmd_type; 3468 u16 i = tx_ring->next_to_use; 3469 3470 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 3471 3472 ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen); 3473 cmd_type = ixgbevf_tx_cmd_type(tx_flags); 3474 3475 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 3476 if (dma_mapping_error(tx_ring->dev, dma)) 3477 goto dma_error; 3478 3479 /* record length, and DMA address */ 3480 dma_unmap_len_set(first, len, size); 3481 dma_unmap_addr_set(first, dma, dma); 3482 3483 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3484 3485 for (;;) { 3486 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 3487 tx_desc->read.cmd_type_len = 3488 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); 3489 3490 i++; 3491 tx_desc++; 3492 if (i == tx_ring->count) { 3493 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3494 i = 0; 3495 } 3496 3497 dma += IXGBE_MAX_DATA_PER_TXD; 3498 size -= IXGBE_MAX_DATA_PER_TXD; 3499 3500 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3501 tx_desc->read.olinfo_status = 0; 3502 } 3503 3504 if (likely(!data_len)) 3505 break; 3506 3507 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); 3508 3509 i++; 3510 tx_desc++; 3511 if (i == tx_ring->count) { 3512 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3513 i = 0; 3514 } 3515 3516 size = skb_frag_size(frag); 3517 data_len -= size; 3518 3519 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 3520 DMA_TO_DEVICE); 3521 if (dma_mapping_error(tx_ring->dev, dma)) 3522 goto dma_error; 3523 3524 tx_buffer = &tx_ring->tx_buffer_info[i]; 3525 dma_unmap_len_set(tx_buffer, len, size); 3526 dma_unmap_addr_set(tx_buffer, dma, dma); 3527 3528 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3529 tx_desc->read.olinfo_status = 0; 3530 3531 frag++; 3532 } 3533 3534 /* write last descriptor with RS and EOP bits */ 3535 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); 3536 tx_desc->read.cmd_type_len = cmd_type; 3537 3538 /* set the timestamp */ 3539 first->time_stamp = jiffies; 3540 3541 /* Force memory writes to complete before letting h/w know there 3542 * are new descriptors to fetch. (Only applicable for weak-ordered 3543 * memory model archs, such as IA-64). 3544 * 3545 * We also need this memory barrier (wmb) to make certain all of the 3546 * status bits have been updated before next_to_watch is written. 3547 */ 3548 wmb(); 3549 3550 /* set next_to_watch value indicating a packet is present */ 3551 first->next_to_watch = tx_desc; 3552 3553 i++; 3554 if (i == tx_ring->count) 3555 i = 0; 3556 3557 tx_ring->next_to_use = i; 3558 3559 /* notify HW of packet */ 3560 ixgbevf_write_tail(tx_ring, i); 3561 3562 return; 3563 dma_error: 3564 dev_err(tx_ring->dev, "TX DMA map failed\n"); 3565 3566 /* clear dma mappings for failed tx_buffer_info map */ 3567 for (;;) { 3568 tx_buffer = &tx_ring->tx_buffer_info[i]; 3569 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer); 3570 if (tx_buffer == first) 3571 break; 3572 if (i == 0) 3573 i = tx_ring->count; 3574 i--; 3575 } 3576 3577 tx_ring->next_to_use = i; 3578 } 3579 3580 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 3581 { 3582 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 3583 /* Herbert's original patch had: 3584 * smp_mb__after_netif_stop_queue(); 3585 * but since that doesn't exist yet, just open code it. 3586 */ 3587 smp_mb(); 3588 3589 /* We need to check again in a case another CPU has just 3590 * made room available. 3591 */ 3592 if (likely(ixgbevf_desc_unused(tx_ring) < size)) 3593 return -EBUSY; 3594 3595 /* A reprieve! - use start_queue because it doesn't call schedule */ 3596 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 3597 ++tx_ring->tx_stats.restart_queue; 3598 3599 return 0; 3600 } 3601 3602 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 3603 { 3604 if (likely(ixgbevf_desc_unused(tx_ring) >= size)) 3605 return 0; 3606 return __ixgbevf_maybe_stop_tx(tx_ring, size); 3607 } 3608 3609 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 3610 { 3611 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3612 struct ixgbevf_tx_buffer *first; 3613 struct ixgbevf_ring *tx_ring; 3614 int tso; 3615 u32 tx_flags = 0; 3616 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 3617 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3618 unsigned short f; 3619 #endif 3620 u8 hdr_len = 0; 3621 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL); 3622 3623 if (!dst_mac || is_link_local_ether_addr(dst_mac)) { 3624 dev_kfree_skb_any(skb); 3625 return NETDEV_TX_OK; 3626 } 3627 3628 tx_ring = adapter->tx_ring[skb->queue_mapping]; 3629 3630 /* need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 3631 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 3632 * + 2 desc gap to keep tail from touching head, 3633 * + 1 desc for context descriptor, 3634 * otherwise try next time 3635 */ 3636 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3637 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 3638 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 3639 #else 3640 count += skb_shinfo(skb)->nr_frags; 3641 #endif 3642 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { 3643 tx_ring->tx_stats.tx_busy++; 3644 return NETDEV_TX_BUSY; 3645 } 3646 3647 /* record the location of the first descriptor for this packet */ 3648 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 3649 first->skb = skb; 3650 first->bytecount = skb->len; 3651 first->gso_segs = 1; 3652 3653 if (skb_vlan_tag_present(skb)) { 3654 tx_flags |= skb_vlan_tag_get(skb); 3655 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 3656 tx_flags |= IXGBE_TX_FLAGS_VLAN; 3657 } 3658 3659 /* record initial flags and protocol */ 3660 first->tx_flags = tx_flags; 3661 first->protocol = vlan_get_protocol(skb); 3662 3663 tso = ixgbevf_tso(tx_ring, first, &hdr_len); 3664 if (tso < 0) 3665 goto out_drop; 3666 else if (!tso) 3667 ixgbevf_tx_csum(tx_ring, first); 3668 3669 ixgbevf_tx_map(tx_ring, first, hdr_len); 3670 3671 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); 3672 3673 return NETDEV_TX_OK; 3674 3675 out_drop: 3676 dev_kfree_skb_any(first->skb); 3677 first->skb = NULL; 3678 3679 return NETDEV_TX_OK; 3680 } 3681 3682 /** 3683 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 3684 * @netdev: network interface device structure 3685 * @p: pointer to an address structure 3686 * 3687 * Returns 0 on success, negative on failure 3688 **/ 3689 static int ixgbevf_set_mac(struct net_device *netdev, void *p) 3690 { 3691 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3692 struct ixgbe_hw *hw = &adapter->hw; 3693 struct sockaddr *addr = p; 3694 3695 if (!is_valid_ether_addr(addr->sa_data)) 3696 return -EADDRNOTAVAIL; 3697 3698 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 3699 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 3700 3701 spin_lock_bh(&adapter->mbx_lock); 3702 3703 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 3704 3705 spin_unlock_bh(&adapter->mbx_lock); 3706 3707 return 0; 3708 } 3709 3710 /** 3711 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 3712 * @netdev: network interface device structure 3713 * @new_mtu: new value for maximum frame size 3714 * 3715 * Returns 0 on success, negative on failure 3716 **/ 3717 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 3718 { 3719 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3720 struct ixgbe_hw *hw = &adapter->hw; 3721 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 3722 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; 3723 3724 switch (adapter->hw.api_version) { 3725 case ixgbe_mbox_api_11: 3726 case ixgbe_mbox_api_12: 3727 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3728 break; 3729 default: 3730 if (adapter->hw.mac.type != ixgbe_mac_82599_vf) 3731 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3732 break; 3733 } 3734 3735 /* MTU < 68 is an error and causes problems on some kernels */ 3736 if ((new_mtu < 68) || (max_frame > max_possible_frame)) 3737 return -EINVAL; 3738 3739 hw_dbg(hw, "changing MTU from %d to %d\n", 3740 netdev->mtu, new_mtu); 3741 /* must set new MTU before calling down or up */ 3742 netdev->mtu = new_mtu; 3743 3744 /* notify the PF of our intent to use this size of frame */ 3745 ixgbevf_rlpml_set_vf(hw, max_frame); 3746 3747 return 0; 3748 } 3749 3750 #ifdef CONFIG_NET_POLL_CONTROLLER 3751 /* Polling 'interrupt' - used by things like netconsole to send skbs 3752 * without having to re-enable interrupts. It's not called while 3753 * the interrupt routine is executing. 3754 */ 3755 static void ixgbevf_netpoll(struct net_device *netdev) 3756 { 3757 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3758 int i; 3759 3760 /* if interface is down do nothing */ 3761 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 3762 return; 3763 for (i = 0; i < adapter->num_rx_queues; i++) 3764 ixgbevf_msix_clean_rings(0, adapter->q_vector[i]); 3765 } 3766 #endif /* CONFIG_NET_POLL_CONTROLLER */ 3767 3768 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) 3769 { 3770 struct net_device *netdev = pci_get_drvdata(pdev); 3771 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3772 #ifdef CONFIG_PM 3773 int retval = 0; 3774 #endif 3775 3776 netif_device_detach(netdev); 3777 3778 if (netif_running(netdev)) { 3779 rtnl_lock(); 3780 ixgbevf_down(adapter); 3781 ixgbevf_free_irq(adapter); 3782 ixgbevf_free_all_tx_resources(adapter); 3783 ixgbevf_free_all_rx_resources(adapter); 3784 rtnl_unlock(); 3785 } 3786 3787 ixgbevf_clear_interrupt_scheme(adapter); 3788 3789 #ifdef CONFIG_PM 3790 retval = pci_save_state(pdev); 3791 if (retval) 3792 return retval; 3793 3794 #endif 3795 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 3796 pci_disable_device(pdev); 3797 3798 return 0; 3799 } 3800 3801 #ifdef CONFIG_PM 3802 static int ixgbevf_resume(struct pci_dev *pdev) 3803 { 3804 struct net_device *netdev = pci_get_drvdata(pdev); 3805 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3806 u32 err; 3807 3808 pci_restore_state(pdev); 3809 /* pci_restore_state clears dev->state_saved so call 3810 * pci_save_state to restore it. 3811 */ 3812 pci_save_state(pdev); 3813 3814 err = pci_enable_device_mem(pdev); 3815 if (err) { 3816 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 3817 return err; 3818 } 3819 smp_mb__before_atomic(); 3820 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 3821 pci_set_master(pdev); 3822 3823 ixgbevf_reset(adapter); 3824 3825 rtnl_lock(); 3826 err = ixgbevf_init_interrupt_scheme(adapter); 3827 rtnl_unlock(); 3828 if (err) { 3829 dev_err(&pdev->dev, "Cannot initialize interrupts\n"); 3830 return err; 3831 } 3832 3833 if (netif_running(netdev)) { 3834 err = ixgbevf_open(netdev); 3835 if (err) 3836 return err; 3837 } 3838 3839 netif_device_attach(netdev); 3840 3841 return err; 3842 } 3843 3844 #endif /* CONFIG_PM */ 3845 static void ixgbevf_shutdown(struct pci_dev *pdev) 3846 { 3847 ixgbevf_suspend(pdev, PMSG_SUSPEND); 3848 } 3849 3850 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, 3851 struct rtnl_link_stats64 *stats) 3852 { 3853 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3854 unsigned int start; 3855 u64 bytes, packets; 3856 const struct ixgbevf_ring *ring; 3857 int i; 3858 3859 ixgbevf_update_stats(adapter); 3860 3861 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3862 3863 for (i = 0; i < adapter->num_rx_queues; i++) { 3864 ring = adapter->rx_ring[i]; 3865 do { 3866 start = u64_stats_fetch_begin_irq(&ring->syncp); 3867 bytes = ring->stats.bytes; 3868 packets = ring->stats.packets; 3869 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 3870 stats->rx_bytes += bytes; 3871 stats->rx_packets += packets; 3872 } 3873 3874 for (i = 0; i < adapter->num_tx_queues; i++) { 3875 ring = adapter->tx_ring[i]; 3876 do { 3877 start = u64_stats_fetch_begin_irq(&ring->syncp); 3878 bytes = ring->stats.bytes; 3879 packets = ring->stats.packets; 3880 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 3881 stats->tx_bytes += bytes; 3882 stats->tx_packets += packets; 3883 } 3884 3885 return stats; 3886 } 3887 3888 static const struct net_device_ops ixgbevf_netdev_ops = { 3889 .ndo_open = ixgbevf_open, 3890 .ndo_stop = ixgbevf_close, 3891 .ndo_start_xmit = ixgbevf_xmit_frame, 3892 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 3893 .ndo_get_stats64 = ixgbevf_get_stats, 3894 .ndo_validate_addr = eth_validate_addr, 3895 .ndo_set_mac_address = ixgbevf_set_mac, 3896 .ndo_change_mtu = ixgbevf_change_mtu, 3897 .ndo_tx_timeout = ixgbevf_tx_timeout, 3898 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 3899 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 3900 #ifdef CONFIG_NET_RX_BUSY_POLL 3901 .ndo_busy_poll = ixgbevf_busy_poll_recv, 3902 #endif 3903 #ifdef CONFIG_NET_POLL_CONTROLLER 3904 .ndo_poll_controller = ixgbevf_netpoll, 3905 #endif 3906 .ndo_features_check = passthru_features_check, 3907 }; 3908 3909 static void ixgbevf_assign_netdev_ops(struct net_device *dev) 3910 { 3911 dev->netdev_ops = &ixgbevf_netdev_ops; 3912 ixgbevf_set_ethtool_ops(dev); 3913 dev->watchdog_timeo = 5 * HZ; 3914 } 3915 3916 /** 3917 * ixgbevf_probe - Device Initialization Routine 3918 * @pdev: PCI device information struct 3919 * @ent: entry in ixgbevf_pci_tbl 3920 * 3921 * Returns 0 on success, negative on failure 3922 * 3923 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 3924 * The OS initialization, configuring of the adapter private structure, 3925 * and a hardware reset occur. 3926 **/ 3927 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3928 { 3929 struct net_device *netdev; 3930 struct ixgbevf_adapter *adapter = NULL; 3931 struct ixgbe_hw *hw = NULL; 3932 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 3933 int err, pci_using_dac; 3934 bool disable_dev = false; 3935 3936 err = pci_enable_device(pdev); 3937 if (err) 3938 return err; 3939 3940 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 3941 pci_using_dac = 1; 3942 } else { 3943 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 3944 if (err) { 3945 dev_err(&pdev->dev, "No usable DMA configuration, aborting\n"); 3946 goto err_dma; 3947 } 3948 pci_using_dac = 0; 3949 } 3950 3951 err = pci_request_regions(pdev, ixgbevf_driver_name); 3952 if (err) { 3953 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 3954 goto err_pci_reg; 3955 } 3956 3957 pci_set_master(pdev); 3958 3959 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 3960 MAX_TX_QUEUES); 3961 if (!netdev) { 3962 err = -ENOMEM; 3963 goto err_alloc_etherdev; 3964 } 3965 3966 SET_NETDEV_DEV(netdev, &pdev->dev); 3967 3968 adapter = netdev_priv(netdev); 3969 3970 adapter->netdev = netdev; 3971 adapter->pdev = pdev; 3972 hw = &adapter->hw; 3973 hw->back = adapter; 3974 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 3975 3976 /* call save state here in standalone driver because it relies on 3977 * adapter struct to exist, and needs to call netdev_priv 3978 */ 3979 pci_save_state(pdev); 3980 3981 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3982 pci_resource_len(pdev, 0)); 3983 adapter->io_addr = hw->hw_addr; 3984 if (!hw->hw_addr) { 3985 err = -EIO; 3986 goto err_ioremap; 3987 } 3988 3989 ixgbevf_assign_netdev_ops(netdev); 3990 3991 /* Setup HW API */ 3992 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 3993 hw->mac.type = ii->mac; 3994 3995 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 3996 sizeof(struct ixgbe_mbx_operations)); 3997 3998 /* setup the private structure */ 3999 err = ixgbevf_sw_init(adapter); 4000 if (err) 4001 goto err_sw_init; 4002 4003 /* The HW MAC address was set and/or determined in sw_init */ 4004 if (!is_valid_ether_addr(netdev->dev_addr)) { 4005 pr_err("invalid MAC address\n"); 4006 err = -EIO; 4007 goto err_sw_init; 4008 } 4009 4010 netdev->hw_features = NETIF_F_SG | 4011 NETIF_F_IP_CSUM | 4012 NETIF_F_IPV6_CSUM | 4013 NETIF_F_TSO | 4014 NETIF_F_TSO6 | 4015 NETIF_F_RXCSUM; 4016 4017 netdev->features = netdev->hw_features | 4018 NETIF_F_HW_VLAN_CTAG_TX | 4019 NETIF_F_HW_VLAN_CTAG_RX | 4020 NETIF_F_HW_VLAN_CTAG_FILTER; 4021 4022 netdev->vlan_features |= NETIF_F_TSO | 4023 NETIF_F_TSO6 | 4024 NETIF_F_IP_CSUM | 4025 NETIF_F_IPV6_CSUM | 4026 NETIF_F_SG; 4027 4028 if (pci_using_dac) 4029 netdev->features |= NETIF_F_HIGHDMA; 4030 4031 netdev->priv_flags |= IFF_UNICAST_FLT; 4032 4033 if (IXGBE_REMOVED(hw->hw_addr)) { 4034 err = -EIO; 4035 goto err_sw_init; 4036 } 4037 4038 setup_timer(&adapter->service_timer, &ixgbevf_service_timer, 4039 (unsigned long)adapter); 4040 4041 INIT_WORK(&adapter->service_task, ixgbevf_service_task); 4042 set_bit(__IXGBEVF_SERVICE_INITED, &adapter->state); 4043 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 4044 4045 err = ixgbevf_init_interrupt_scheme(adapter); 4046 if (err) 4047 goto err_sw_init; 4048 4049 strcpy(netdev->name, "eth%d"); 4050 4051 err = register_netdev(netdev); 4052 if (err) 4053 goto err_register; 4054 4055 pci_set_drvdata(pdev, netdev); 4056 netif_carrier_off(netdev); 4057 4058 ixgbevf_init_last_counter_stats(adapter); 4059 4060 /* print the VF info */ 4061 dev_info(&pdev->dev, "%pM\n", netdev->dev_addr); 4062 dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type); 4063 4064 switch (hw->mac.type) { 4065 case ixgbe_mac_X550_vf: 4066 dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n"); 4067 break; 4068 case ixgbe_mac_X540_vf: 4069 dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n"); 4070 break; 4071 case ixgbe_mac_82599_vf: 4072 default: 4073 dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n"); 4074 break; 4075 } 4076 4077 return 0; 4078 4079 err_register: 4080 ixgbevf_clear_interrupt_scheme(adapter); 4081 err_sw_init: 4082 ixgbevf_reset_interrupt_capability(adapter); 4083 iounmap(adapter->io_addr); 4084 err_ioremap: 4085 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4086 free_netdev(netdev); 4087 err_alloc_etherdev: 4088 pci_release_regions(pdev); 4089 err_pci_reg: 4090 err_dma: 4091 if (!adapter || disable_dev) 4092 pci_disable_device(pdev); 4093 return err; 4094 } 4095 4096 /** 4097 * ixgbevf_remove - Device Removal Routine 4098 * @pdev: PCI device information struct 4099 * 4100 * ixgbevf_remove is called by the PCI subsystem to alert the driver 4101 * that it should release a PCI device. The could be caused by a 4102 * Hot-Plug event, or because the driver is going to be removed from 4103 * memory. 4104 **/ 4105 static void ixgbevf_remove(struct pci_dev *pdev) 4106 { 4107 struct net_device *netdev = pci_get_drvdata(pdev); 4108 struct ixgbevf_adapter *adapter; 4109 bool disable_dev; 4110 4111 if (!netdev) 4112 return; 4113 4114 adapter = netdev_priv(netdev); 4115 4116 set_bit(__IXGBEVF_REMOVING, &adapter->state); 4117 cancel_work_sync(&adapter->service_task); 4118 4119 if (netdev->reg_state == NETREG_REGISTERED) 4120 unregister_netdev(netdev); 4121 4122 ixgbevf_clear_interrupt_scheme(adapter); 4123 ixgbevf_reset_interrupt_capability(adapter); 4124 4125 iounmap(adapter->io_addr); 4126 pci_release_regions(pdev); 4127 4128 hw_dbg(&adapter->hw, "Remove complete\n"); 4129 4130 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4131 free_netdev(netdev); 4132 4133 if (disable_dev) 4134 pci_disable_device(pdev); 4135 } 4136 4137 /** 4138 * ixgbevf_io_error_detected - called when PCI error is detected 4139 * @pdev: Pointer to PCI device 4140 * @state: The current pci connection state 4141 * 4142 * This function is called after a PCI bus error affecting 4143 * this device has been detected. 4144 **/ 4145 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, 4146 pci_channel_state_t state) 4147 { 4148 struct net_device *netdev = pci_get_drvdata(pdev); 4149 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4150 4151 if (!test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 4152 return PCI_ERS_RESULT_DISCONNECT; 4153 4154 rtnl_lock(); 4155 netif_device_detach(netdev); 4156 4157 if (state == pci_channel_io_perm_failure) { 4158 rtnl_unlock(); 4159 return PCI_ERS_RESULT_DISCONNECT; 4160 } 4161 4162 if (netif_running(netdev)) 4163 ixgbevf_down(adapter); 4164 4165 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 4166 pci_disable_device(pdev); 4167 rtnl_unlock(); 4168 4169 /* Request a slot slot reset. */ 4170 return PCI_ERS_RESULT_NEED_RESET; 4171 } 4172 4173 /** 4174 * ixgbevf_io_slot_reset - called after the pci bus has been reset. 4175 * @pdev: Pointer to PCI device 4176 * 4177 * Restart the card from scratch, as if from a cold-boot. Implementation 4178 * resembles the first-half of the ixgbevf_resume routine. 4179 **/ 4180 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) 4181 { 4182 struct net_device *netdev = pci_get_drvdata(pdev); 4183 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4184 4185 if (pci_enable_device_mem(pdev)) { 4186 dev_err(&pdev->dev, 4187 "Cannot re-enable PCI device after reset.\n"); 4188 return PCI_ERS_RESULT_DISCONNECT; 4189 } 4190 4191 smp_mb__before_atomic(); 4192 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 4193 pci_set_master(pdev); 4194 4195 ixgbevf_reset(adapter); 4196 4197 return PCI_ERS_RESULT_RECOVERED; 4198 } 4199 4200 /** 4201 * ixgbevf_io_resume - called when traffic can start flowing again. 4202 * @pdev: Pointer to PCI device 4203 * 4204 * This callback is called when the error recovery driver tells us that 4205 * its OK to resume normal operation. Implementation resembles the 4206 * second-half of the ixgbevf_resume routine. 4207 **/ 4208 static void ixgbevf_io_resume(struct pci_dev *pdev) 4209 { 4210 struct net_device *netdev = pci_get_drvdata(pdev); 4211 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4212 4213 if (netif_running(netdev)) 4214 ixgbevf_up(adapter); 4215 4216 netif_device_attach(netdev); 4217 } 4218 4219 /* PCI Error Recovery (ERS) */ 4220 static const struct pci_error_handlers ixgbevf_err_handler = { 4221 .error_detected = ixgbevf_io_error_detected, 4222 .slot_reset = ixgbevf_io_slot_reset, 4223 .resume = ixgbevf_io_resume, 4224 }; 4225 4226 static struct pci_driver ixgbevf_driver = { 4227 .name = ixgbevf_driver_name, 4228 .id_table = ixgbevf_pci_tbl, 4229 .probe = ixgbevf_probe, 4230 .remove = ixgbevf_remove, 4231 #ifdef CONFIG_PM 4232 /* Power Management Hooks */ 4233 .suspend = ixgbevf_suspend, 4234 .resume = ixgbevf_resume, 4235 #endif 4236 .shutdown = ixgbevf_shutdown, 4237 .err_handler = &ixgbevf_err_handler 4238 }; 4239 4240 /** 4241 * ixgbevf_init_module - Driver Registration Routine 4242 * 4243 * ixgbevf_init_module is the first routine called when the driver is 4244 * loaded. All it does is register with the PCI subsystem. 4245 **/ 4246 static int __init ixgbevf_init_module(void) 4247 { 4248 int ret; 4249 4250 pr_info("%s - version %s\n", ixgbevf_driver_string, 4251 ixgbevf_driver_version); 4252 4253 pr_info("%s\n", ixgbevf_copyright); 4254 4255 ret = pci_register_driver(&ixgbevf_driver); 4256 return ret; 4257 } 4258 4259 module_init(ixgbevf_init_module); 4260 4261 /** 4262 * ixgbevf_exit_module - Driver Exit Cleanup Routine 4263 * 4264 * ixgbevf_exit_module is called just before the driver is removed 4265 * from memory. 4266 **/ 4267 static void __exit ixgbevf_exit_module(void) 4268 { 4269 pci_unregister_driver(&ixgbevf_driver); 4270 } 4271 4272 #ifdef DEBUG 4273 /** 4274 * ixgbevf_get_hw_dev_name - return device name string 4275 * used by hardware layer to print debugging information 4276 **/ 4277 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 4278 { 4279 struct ixgbevf_adapter *adapter = hw->back; 4280 4281 return adapter->netdev->name; 4282 } 4283 4284 #endif 4285 module_exit(ixgbevf_exit_module); 4286 4287 /* ixgbevf_main.c */ 4288