1 /******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2015 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, see <http://www.gnu.org/licenses/>. 17 18 The full GNU General Public License is included in this distribution in 19 the file called "COPYING". 20 21 Contact Information: 22 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 25 *******************************************************************************/ 26 27 /****************************************************************************** 28 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 29 ******************************************************************************/ 30 31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 32 33 #include <linux/types.h> 34 #include <linux/bitops.h> 35 #include <linux/module.h> 36 #include <linux/pci.h> 37 #include <linux/netdevice.h> 38 #include <linux/vmalloc.h> 39 #include <linux/string.h> 40 #include <linux/in.h> 41 #include <linux/ip.h> 42 #include <linux/tcp.h> 43 #include <linux/sctp.h> 44 #include <linux/ipv6.h> 45 #include <linux/slab.h> 46 #include <net/checksum.h> 47 #include <net/ip6_checksum.h> 48 #include <linux/ethtool.h> 49 #include <linux/if.h> 50 #include <linux/if_vlan.h> 51 #include <linux/prefetch.h> 52 53 #include "ixgbevf.h" 54 55 const char ixgbevf_driver_name[] = "ixgbevf"; 56 static const char ixgbevf_driver_string[] = 57 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 58 59 #define DRV_VERSION "2.12.1-k" 60 const char ixgbevf_driver_version[] = DRV_VERSION; 61 static char ixgbevf_copyright[] = 62 "Copyright (c) 2009 - 2012 Intel Corporation."; 63 64 static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 65 [board_82599_vf] = &ixgbevf_82599_vf_info, 66 [board_X540_vf] = &ixgbevf_X540_vf_info, 67 [board_X550_vf] = &ixgbevf_X550_vf_info, 68 [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info, 69 }; 70 71 /* ixgbevf_pci_tbl - PCI Device ID Table 72 * 73 * Wildcard entries (PCI_ANY_ID) should come last 74 * Last entry must be all 0s 75 * 76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 77 * Class, Class Mask, private data (not used) } 78 */ 79 static const struct pci_device_id ixgbevf_pci_tbl[] = { 80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf }, 81 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf }, 82 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf }, 83 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf }, 84 /* required last entry */ 85 {0, } 86 }; 87 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 88 89 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 90 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); 91 MODULE_LICENSE("GPL"); 92 MODULE_VERSION(DRV_VERSION); 93 94 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 95 static int debug = -1; 96 module_param(debug, int, 0); 97 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 98 99 static void ixgbevf_service_event_schedule(struct ixgbevf_adapter *adapter) 100 { 101 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 102 !test_bit(__IXGBEVF_REMOVING, &adapter->state) && 103 !test_and_set_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)) 104 schedule_work(&adapter->service_task); 105 } 106 107 static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter) 108 { 109 BUG_ON(!test_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)); 110 111 /* flush memory to make sure state is correct before next watchdog */ 112 smp_mb__before_atomic(); 113 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 114 } 115 116 /* forward decls */ 117 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); 118 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); 119 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); 120 121 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) 122 { 123 struct ixgbevf_adapter *adapter = hw->back; 124 125 if (!hw->hw_addr) 126 return; 127 hw->hw_addr = NULL; 128 dev_err(&adapter->pdev->dev, "Adapter removed\n"); 129 if (test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 130 ixgbevf_service_event_schedule(adapter); 131 } 132 133 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg) 134 { 135 u32 value; 136 137 /* The following check not only optimizes a bit by not 138 * performing a read on the status register when the 139 * register just read was a status register read that 140 * returned IXGBE_FAILED_READ_REG. It also blocks any 141 * potential recursion. 142 */ 143 if (reg == IXGBE_VFSTATUS) { 144 ixgbevf_remove_adapter(hw); 145 return; 146 } 147 value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS); 148 if (value == IXGBE_FAILED_READ_REG) 149 ixgbevf_remove_adapter(hw); 150 } 151 152 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg) 153 { 154 u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr); 155 u32 value; 156 157 if (IXGBE_REMOVED(reg_addr)) 158 return IXGBE_FAILED_READ_REG; 159 value = readl(reg_addr + reg); 160 if (unlikely(value == IXGBE_FAILED_READ_REG)) 161 ixgbevf_check_remove(hw, reg); 162 return value; 163 } 164 165 /** 166 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 167 * @adapter: pointer to adapter struct 168 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 169 * @queue: queue to map the corresponding interrupt to 170 * @msix_vector: the vector to map to the corresponding queue 171 **/ 172 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 173 u8 queue, u8 msix_vector) 174 { 175 u32 ivar, index; 176 struct ixgbe_hw *hw = &adapter->hw; 177 178 if (direction == -1) { 179 /* other causes */ 180 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 181 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 182 ivar &= ~0xFF; 183 ivar |= msix_vector; 184 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 185 } else { 186 /* Tx or Rx causes */ 187 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 188 index = ((16 * (queue & 1)) + (8 * direction)); 189 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 190 ivar &= ~(0xFF << index); 191 ivar |= (msix_vector << index); 192 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 193 } 194 } 195 196 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring, 197 struct ixgbevf_tx_buffer *tx_buffer) 198 { 199 if (tx_buffer->skb) { 200 dev_kfree_skb_any(tx_buffer->skb); 201 if (dma_unmap_len(tx_buffer, len)) 202 dma_unmap_single(tx_ring->dev, 203 dma_unmap_addr(tx_buffer, dma), 204 dma_unmap_len(tx_buffer, len), 205 DMA_TO_DEVICE); 206 } else if (dma_unmap_len(tx_buffer, len)) { 207 dma_unmap_page(tx_ring->dev, 208 dma_unmap_addr(tx_buffer, dma), 209 dma_unmap_len(tx_buffer, len), 210 DMA_TO_DEVICE); 211 } 212 tx_buffer->next_to_watch = NULL; 213 tx_buffer->skb = NULL; 214 dma_unmap_len_set(tx_buffer, len, 0); 215 /* tx_buffer must be completely set up in the transmit path */ 216 } 217 218 static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring) 219 { 220 return ring->stats.packets; 221 } 222 223 static u32 ixgbevf_get_tx_pending(struct ixgbevf_ring *ring) 224 { 225 struct ixgbevf_adapter *adapter = netdev_priv(ring->netdev); 226 struct ixgbe_hw *hw = &adapter->hw; 227 228 u32 head = IXGBE_READ_REG(hw, IXGBE_VFTDH(ring->reg_idx)); 229 u32 tail = IXGBE_READ_REG(hw, IXGBE_VFTDT(ring->reg_idx)); 230 231 if (head != tail) 232 return (head < tail) ? 233 tail - head : (tail + ring->count - head); 234 235 return 0; 236 } 237 238 static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring) 239 { 240 u32 tx_done = ixgbevf_get_tx_completed(tx_ring); 241 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 242 u32 tx_pending = ixgbevf_get_tx_pending(tx_ring); 243 244 clear_check_for_tx_hang(tx_ring); 245 246 /* Check for a hung queue, but be thorough. This verifies 247 * that a transmit has been completed since the previous 248 * check AND there is at least one packet pending. The 249 * ARMED bit is set to indicate a potential hang. 250 */ 251 if ((tx_done_old == tx_done) && tx_pending) { 252 /* make sure it is true for two checks in a row */ 253 return test_and_set_bit(__IXGBEVF_HANG_CHECK_ARMED, 254 &tx_ring->state); 255 } 256 /* reset the countdown */ 257 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &tx_ring->state); 258 259 /* update completed stats and continue */ 260 tx_ring->tx_stats.tx_done_old = tx_done; 261 262 return false; 263 } 264 265 static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter) 266 { 267 /* Do the reset outside of interrupt context */ 268 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 269 adapter->flags |= IXGBEVF_FLAG_RESET_REQUESTED; 270 ixgbevf_service_event_schedule(adapter); 271 } 272 } 273 274 /** 275 * ixgbevf_tx_timeout - Respond to a Tx Hang 276 * @netdev: network interface device structure 277 **/ 278 static void ixgbevf_tx_timeout(struct net_device *netdev) 279 { 280 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 281 282 ixgbevf_tx_timeout_reset(adapter); 283 } 284 285 /** 286 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 287 * @q_vector: board private structure 288 * @tx_ring: tx ring to clean 289 **/ 290 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, 291 struct ixgbevf_ring *tx_ring) 292 { 293 struct ixgbevf_adapter *adapter = q_vector->adapter; 294 struct ixgbevf_tx_buffer *tx_buffer; 295 union ixgbe_adv_tx_desc *tx_desc; 296 unsigned int total_bytes = 0, total_packets = 0; 297 unsigned int budget = tx_ring->count / 2; 298 unsigned int i = tx_ring->next_to_clean; 299 300 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 301 return true; 302 303 tx_buffer = &tx_ring->tx_buffer_info[i]; 304 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 305 i -= tx_ring->count; 306 307 do { 308 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 309 310 /* if next_to_watch is not set then there is no work pending */ 311 if (!eop_desc) 312 break; 313 314 /* prevent any other reads prior to eop_desc */ 315 read_barrier_depends(); 316 317 /* if DD is not set pending work has not been completed */ 318 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 319 break; 320 321 /* clear next_to_watch to prevent false hangs */ 322 tx_buffer->next_to_watch = NULL; 323 324 /* update the statistics for this packet */ 325 total_bytes += tx_buffer->bytecount; 326 total_packets += tx_buffer->gso_segs; 327 328 /* free the skb */ 329 dev_kfree_skb_any(tx_buffer->skb); 330 331 /* unmap skb header data */ 332 dma_unmap_single(tx_ring->dev, 333 dma_unmap_addr(tx_buffer, dma), 334 dma_unmap_len(tx_buffer, len), 335 DMA_TO_DEVICE); 336 337 /* clear tx_buffer data */ 338 tx_buffer->skb = NULL; 339 dma_unmap_len_set(tx_buffer, len, 0); 340 341 /* unmap remaining buffers */ 342 while (tx_desc != eop_desc) { 343 tx_buffer++; 344 tx_desc++; 345 i++; 346 if (unlikely(!i)) { 347 i -= tx_ring->count; 348 tx_buffer = tx_ring->tx_buffer_info; 349 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 350 } 351 352 /* unmap any remaining paged data */ 353 if (dma_unmap_len(tx_buffer, len)) { 354 dma_unmap_page(tx_ring->dev, 355 dma_unmap_addr(tx_buffer, dma), 356 dma_unmap_len(tx_buffer, len), 357 DMA_TO_DEVICE); 358 dma_unmap_len_set(tx_buffer, len, 0); 359 } 360 } 361 362 /* move us one more past the eop_desc for start of next pkt */ 363 tx_buffer++; 364 tx_desc++; 365 i++; 366 if (unlikely(!i)) { 367 i -= tx_ring->count; 368 tx_buffer = tx_ring->tx_buffer_info; 369 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 370 } 371 372 /* issue prefetch for next Tx descriptor */ 373 prefetch(tx_desc); 374 375 /* update budget accounting */ 376 budget--; 377 } while (likely(budget)); 378 379 i += tx_ring->count; 380 tx_ring->next_to_clean = i; 381 u64_stats_update_begin(&tx_ring->syncp); 382 tx_ring->stats.bytes += total_bytes; 383 tx_ring->stats.packets += total_packets; 384 u64_stats_update_end(&tx_ring->syncp); 385 q_vector->tx.total_bytes += total_bytes; 386 q_vector->tx.total_packets += total_packets; 387 388 if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) { 389 struct ixgbe_hw *hw = &adapter->hw; 390 union ixgbe_adv_tx_desc *eop_desc; 391 392 eop_desc = tx_ring->tx_buffer_info[i].next_to_watch; 393 394 pr_err("Detected Tx Unit Hang\n" 395 " Tx Queue <%d>\n" 396 " TDH, TDT <%x>, <%x>\n" 397 " next_to_use <%x>\n" 398 " next_to_clean <%x>\n" 399 "tx_buffer_info[next_to_clean]\n" 400 " next_to_watch <%p>\n" 401 " eop_desc->wb.status <%x>\n" 402 " time_stamp <%lx>\n" 403 " jiffies <%lx>\n", 404 tx_ring->queue_index, 405 IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)), 406 IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)), 407 tx_ring->next_to_use, i, 408 eop_desc, (eop_desc ? eop_desc->wb.status : 0), 409 tx_ring->tx_buffer_info[i].time_stamp, jiffies); 410 411 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 412 413 /* schedule immediate reset if we believe we hung */ 414 ixgbevf_tx_timeout_reset(adapter); 415 416 return true; 417 } 418 419 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 420 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 421 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 422 /* Make sure that anybody stopping the queue after this 423 * sees the new next_to_clean. 424 */ 425 smp_mb(); 426 427 if (__netif_subqueue_stopped(tx_ring->netdev, 428 tx_ring->queue_index) && 429 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 430 netif_wake_subqueue(tx_ring->netdev, 431 tx_ring->queue_index); 432 ++tx_ring->tx_stats.restart_queue; 433 } 434 } 435 436 return !!budget; 437 } 438 439 /** 440 * ixgbevf_rx_skb - Helper function to determine proper Rx method 441 * @q_vector: structure containing interrupt and ring information 442 * @skb: packet to send up 443 **/ 444 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, 445 struct sk_buff *skb) 446 { 447 #ifdef CONFIG_NET_RX_BUSY_POLL 448 skb_mark_napi_id(skb, &q_vector->napi); 449 450 if (ixgbevf_qv_busy_polling(q_vector)) { 451 netif_receive_skb(skb); 452 /* exit early if we busy polled */ 453 return; 454 } 455 #endif /* CONFIG_NET_RX_BUSY_POLL */ 456 457 napi_gro_receive(&q_vector->napi, skb); 458 } 459 460 #define IXGBE_RSS_L4_TYPES_MASK \ 461 ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ 462 (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ 463 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ 464 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) 465 466 static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring, 467 union ixgbe_adv_rx_desc *rx_desc, 468 struct sk_buff *skb) 469 { 470 u16 rss_type; 471 472 if (!(ring->netdev->features & NETIF_F_RXHASH)) 473 return; 474 475 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & 476 IXGBE_RXDADV_RSSTYPE_MASK; 477 478 if (!rss_type) 479 return; 480 481 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 482 (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? 483 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); 484 } 485 486 /** 487 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 488 * @ring: structure containig ring specific data 489 * @rx_desc: current Rx descriptor being processed 490 * @skb: skb currently being received and modified 491 **/ 492 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, 493 union ixgbe_adv_rx_desc *rx_desc, 494 struct sk_buff *skb) 495 { 496 skb_checksum_none_assert(skb); 497 498 /* Rx csum disabled */ 499 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 500 return; 501 502 /* if IP and error */ 503 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 504 ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 505 ring->rx_stats.csum_err++; 506 return; 507 } 508 509 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 510 return; 511 512 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 513 ring->rx_stats.csum_err++; 514 return; 515 } 516 517 /* It must be a TCP or UDP packet with a valid checksum */ 518 skb->ip_summed = CHECKSUM_UNNECESSARY; 519 } 520 521 /** 522 * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor 523 * @rx_ring: rx descriptor ring packet is being transacted on 524 * @rx_desc: pointer to the EOP Rx descriptor 525 * @skb: pointer to current skb being populated 526 * 527 * This function checks the ring, descriptor, and packet information in 528 * order to populate the checksum, VLAN, protocol, and other fields within 529 * the skb. 530 **/ 531 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, 532 union ixgbe_adv_rx_desc *rx_desc, 533 struct sk_buff *skb) 534 { 535 ixgbevf_rx_hash(rx_ring, rx_desc, skb); 536 ixgbevf_rx_checksum(rx_ring, rx_desc, skb); 537 538 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 539 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 540 unsigned long *active_vlans = netdev_priv(rx_ring->netdev); 541 542 if (test_bit(vid & VLAN_VID_MASK, active_vlans)) 543 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 544 } 545 546 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 547 } 548 549 /** 550 * ixgbevf_is_non_eop - process handling of non-EOP buffers 551 * @rx_ring: Rx ring being processed 552 * @rx_desc: Rx descriptor for current buffer 553 * @skb: current socket buffer containing buffer in progress 554 * 555 * This function updates next to clean. If the buffer is an EOP buffer 556 * this function exits returning false, otherwise it will place the 557 * sk_buff in the next buffer to be chained and return true indicating 558 * that this is in fact a non-EOP buffer. 559 **/ 560 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, 561 union ixgbe_adv_rx_desc *rx_desc) 562 { 563 u32 ntc = rx_ring->next_to_clean + 1; 564 565 /* fetch, update, and store next to clean */ 566 ntc = (ntc < rx_ring->count) ? ntc : 0; 567 rx_ring->next_to_clean = ntc; 568 569 prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); 570 571 if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 572 return false; 573 574 return true; 575 } 576 577 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, 578 struct ixgbevf_rx_buffer *bi) 579 { 580 struct page *page = bi->page; 581 dma_addr_t dma = bi->dma; 582 583 /* since we are recycling buffers we should seldom need to alloc */ 584 if (likely(page)) 585 return true; 586 587 /* alloc new page for storage */ 588 page = dev_alloc_page(); 589 if (unlikely(!page)) { 590 rx_ring->rx_stats.alloc_rx_page_failed++; 591 return false; 592 } 593 594 /* map page for use */ 595 dma = dma_map_page(rx_ring->dev, page, 0, 596 PAGE_SIZE, DMA_FROM_DEVICE); 597 598 /* if mapping failed free memory back to system since 599 * there isn't much point in holding memory we can't use 600 */ 601 if (dma_mapping_error(rx_ring->dev, dma)) { 602 __free_page(page); 603 604 rx_ring->rx_stats.alloc_rx_buff_failed++; 605 return false; 606 } 607 608 bi->dma = dma; 609 bi->page = page; 610 bi->page_offset = 0; 611 612 return true; 613 } 614 615 /** 616 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 617 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on 618 * @cleaned_count: number of buffers to replace 619 **/ 620 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, 621 u16 cleaned_count) 622 { 623 union ixgbe_adv_rx_desc *rx_desc; 624 struct ixgbevf_rx_buffer *bi; 625 unsigned int i = rx_ring->next_to_use; 626 627 /* nothing to do or no valid netdev defined */ 628 if (!cleaned_count || !rx_ring->netdev) 629 return; 630 631 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 632 bi = &rx_ring->rx_buffer_info[i]; 633 i -= rx_ring->count; 634 635 do { 636 if (!ixgbevf_alloc_mapped_page(rx_ring, bi)) 637 break; 638 639 /* Refresh the desc even if pkt_addr didn't change 640 * because each write-back erases this info. 641 */ 642 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 643 644 rx_desc++; 645 bi++; 646 i++; 647 if (unlikely(!i)) { 648 rx_desc = IXGBEVF_RX_DESC(rx_ring, 0); 649 bi = rx_ring->rx_buffer_info; 650 i -= rx_ring->count; 651 } 652 653 /* clear the hdr_addr for the next_to_use descriptor */ 654 rx_desc->read.hdr_addr = 0; 655 656 cleaned_count--; 657 } while (cleaned_count); 658 659 i += rx_ring->count; 660 661 if (rx_ring->next_to_use != i) { 662 /* record the next descriptor to use */ 663 rx_ring->next_to_use = i; 664 665 /* update next to alloc since we have filled the ring */ 666 rx_ring->next_to_alloc = i; 667 668 /* Force memory writes to complete before letting h/w 669 * know there are new descriptors to fetch. (Only 670 * applicable for weak-ordered memory model archs, 671 * such as IA-64). 672 */ 673 wmb(); 674 ixgbevf_write_tail(rx_ring, i); 675 } 676 } 677 678 /** 679 * ixgbevf_cleanup_headers - Correct corrupted or empty headers 680 * @rx_ring: rx descriptor ring packet is being transacted on 681 * @rx_desc: pointer to the EOP Rx descriptor 682 * @skb: pointer to current skb being fixed 683 * 684 * Check for corrupted packet headers caused by senders on the local L2 685 * embedded NIC switch not setting up their Tx Descriptors right. These 686 * should be very rare. 687 * 688 * Also address the case where we are pulling data in on pages only 689 * and as such no data is present in the skb header. 690 * 691 * In addition if skb is not at least 60 bytes we need to pad it so that 692 * it is large enough to qualify as a valid Ethernet frame. 693 * 694 * Returns true if an error was encountered and skb was freed. 695 **/ 696 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, 697 union ixgbe_adv_rx_desc *rx_desc, 698 struct sk_buff *skb) 699 { 700 /* verify that the packet does not have any known errors */ 701 if (unlikely(ixgbevf_test_staterr(rx_desc, 702 IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { 703 struct net_device *netdev = rx_ring->netdev; 704 705 if (!(netdev->features & NETIF_F_RXALL)) { 706 dev_kfree_skb_any(skb); 707 return true; 708 } 709 } 710 711 /* if eth_skb_pad returns an error the skb was freed */ 712 if (eth_skb_pad(skb)) 713 return true; 714 715 return false; 716 } 717 718 /** 719 * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring 720 * @rx_ring: rx descriptor ring to store buffers on 721 * @old_buff: donor buffer to have page reused 722 * 723 * Synchronizes page for reuse by the adapter 724 **/ 725 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, 726 struct ixgbevf_rx_buffer *old_buff) 727 { 728 struct ixgbevf_rx_buffer *new_buff; 729 u16 nta = rx_ring->next_to_alloc; 730 731 new_buff = &rx_ring->rx_buffer_info[nta]; 732 733 /* update, and store next to alloc */ 734 nta++; 735 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 736 737 /* transfer page from old buffer to new buffer */ 738 new_buff->page = old_buff->page; 739 new_buff->dma = old_buff->dma; 740 new_buff->page_offset = old_buff->page_offset; 741 742 /* sync the buffer for use by the device */ 743 dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma, 744 new_buff->page_offset, 745 IXGBEVF_RX_BUFSZ, 746 DMA_FROM_DEVICE); 747 } 748 749 static inline bool ixgbevf_page_is_reserved(struct page *page) 750 { 751 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 752 } 753 754 /** 755 * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff 756 * @rx_ring: rx descriptor ring to transact packets on 757 * @rx_buffer: buffer containing page to add 758 * @rx_desc: descriptor containing length of buffer written by hardware 759 * @skb: sk_buff to place the data into 760 * 761 * This function will add the data contained in rx_buffer->page to the skb. 762 * This is done either through a direct copy if the data in the buffer is 763 * less than the skb header size, otherwise it will just attach the page as 764 * a frag to the skb. 765 * 766 * The function will then update the page offset if necessary and return 767 * true if the buffer can be reused by the adapter. 768 **/ 769 static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, 770 struct ixgbevf_rx_buffer *rx_buffer, 771 union ixgbe_adv_rx_desc *rx_desc, 772 struct sk_buff *skb) 773 { 774 struct page *page = rx_buffer->page; 775 unsigned char *va = page_address(page) + rx_buffer->page_offset; 776 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length); 777 #if (PAGE_SIZE < 8192) 778 unsigned int truesize = IXGBEVF_RX_BUFSZ; 779 #else 780 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES); 781 #endif 782 unsigned int pull_len; 783 784 if (unlikely(skb_is_nonlinear(skb))) 785 goto add_tail_frag; 786 787 if (likely(size <= IXGBEVF_RX_HDR_SIZE)) { 788 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 789 790 /* page is not reserved, we can reuse buffer as is */ 791 if (likely(!ixgbevf_page_is_reserved(page))) 792 return true; 793 794 /* this page cannot be reused so discard it */ 795 put_page(page); 796 return false; 797 } 798 799 /* we need the header to contain the greater of either ETH_HLEN or 800 * 60 bytes if the skb->len is less than 60 for skb_pad. 801 */ 802 pull_len = eth_get_headlen(va, IXGBEVF_RX_HDR_SIZE); 803 804 /* align pull length to size of long to optimize memcpy performance */ 805 memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long))); 806 807 /* update all of the pointers */ 808 va += pull_len; 809 size -= pull_len; 810 811 add_tail_frag: 812 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 813 (unsigned long)va & ~PAGE_MASK, size, truesize); 814 815 /* avoid re-using remote pages */ 816 if (unlikely(ixgbevf_page_is_reserved(page))) 817 return false; 818 819 #if (PAGE_SIZE < 8192) 820 /* if we are only owner of page we can reuse it */ 821 if (unlikely(page_count(page) != 1)) 822 return false; 823 824 /* flip page offset to other buffer */ 825 rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ; 826 827 #else 828 /* move offset up to the next cache line */ 829 rx_buffer->page_offset += truesize; 830 831 if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ)) 832 return false; 833 834 #endif 835 /* Even if we own the page, we are not allowed to use atomic_set() 836 * This would break get_page_unless_zero() users. 837 */ 838 atomic_inc(&page->_count); 839 840 return true; 841 } 842 843 static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring, 844 union ixgbe_adv_rx_desc *rx_desc, 845 struct sk_buff *skb) 846 { 847 struct ixgbevf_rx_buffer *rx_buffer; 848 struct page *page; 849 850 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 851 page = rx_buffer->page; 852 prefetchw(page); 853 854 if (likely(!skb)) { 855 void *page_addr = page_address(page) + 856 rx_buffer->page_offset; 857 858 /* prefetch first cache line of first page */ 859 prefetch(page_addr); 860 #if L1_CACHE_BYTES < 128 861 prefetch(page_addr + L1_CACHE_BYTES); 862 #endif 863 864 /* allocate a skb to store the frags */ 865 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 866 IXGBEVF_RX_HDR_SIZE); 867 if (unlikely(!skb)) { 868 rx_ring->rx_stats.alloc_rx_buff_failed++; 869 return NULL; 870 } 871 872 /* we will be copying header into skb->data in 873 * pskb_may_pull so it is in our interest to prefetch 874 * it now to avoid a possible cache miss 875 */ 876 prefetchw(skb->data); 877 } 878 879 /* we are reusing so sync this buffer for CPU use */ 880 dma_sync_single_range_for_cpu(rx_ring->dev, 881 rx_buffer->dma, 882 rx_buffer->page_offset, 883 IXGBEVF_RX_BUFSZ, 884 DMA_FROM_DEVICE); 885 886 /* pull page into skb */ 887 if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) { 888 /* hand second half of page back to the ring */ 889 ixgbevf_reuse_rx_page(rx_ring, rx_buffer); 890 } else { 891 /* we are not reusing the buffer so unmap it */ 892 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 893 PAGE_SIZE, DMA_FROM_DEVICE); 894 } 895 896 /* clear contents of buffer_info */ 897 rx_buffer->dma = 0; 898 rx_buffer->page = NULL; 899 900 return skb; 901 } 902 903 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 904 u32 qmask) 905 { 906 struct ixgbe_hw *hw = &adapter->hw; 907 908 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); 909 } 910 911 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 912 struct ixgbevf_ring *rx_ring, 913 int budget) 914 { 915 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 916 u16 cleaned_count = ixgbevf_desc_unused(rx_ring); 917 struct sk_buff *skb = rx_ring->skb; 918 919 while (likely(total_rx_packets < budget)) { 920 union ixgbe_adv_rx_desc *rx_desc; 921 922 /* return some buffers to hardware, one at a time is too slow */ 923 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 924 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count); 925 cleaned_count = 0; 926 } 927 928 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); 929 930 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD)) 931 break; 932 933 /* This memory barrier is needed to keep us from reading 934 * any other fields out of the rx_desc until we know the 935 * RXD_STAT_DD bit is set 936 */ 937 rmb(); 938 939 /* retrieve a buffer from the ring */ 940 skb = ixgbevf_fetch_rx_buffer(rx_ring, rx_desc, skb); 941 942 /* exit if we failed to retrieve a buffer */ 943 if (!skb) 944 break; 945 946 cleaned_count++; 947 948 /* fetch next buffer in frame if non-eop */ 949 if (ixgbevf_is_non_eop(rx_ring, rx_desc)) 950 continue; 951 952 /* verify the packet layout is correct */ 953 if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) { 954 skb = NULL; 955 continue; 956 } 957 958 /* probably a little skewed due to removing CRC */ 959 total_rx_bytes += skb->len; 960 961 /* Workaround hardware that can't do proper VEPA multicast 962 * source pruning. 963 */ 964 if ((skb->pkt_type == PACKET_BROADCAST || 965 skb->pkt_type == PACKET_MULTICAST) && 966 ether_addr_equal(rx_ring->netdev->dev_addr, 967 eth_hdr(skb)->h_source)) { 968 dev_kfree_skb_irq(skb); 969 continue; 970 } 971 972 /* populate checksum, VLAN, and protocol */ 973 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); 974 975 ixgbevf_rx_skb(q_vector, skb); 976 977 /* reset skb pointer */ 978 skb = NULL; 979 980 /* update budget accounting */ 981 total_rx_packets++; 982 } 983 984 /* place incomplete frames back on ring for completion */ 985 rx_ring->skb = skb; 986 987 u64_stats_update_begin(&rx_ring->syncp); 988 rx_ring->stats.packets += total_rx_packets; 989 rx_ring->stats.bytes += total_rx_bytes; 990 u64_stats_update_end(&rx_ring->syncp); 991 q_vector->rx.total_packets += total_rx_packets; 992 q_vector->rx.total_bytes += total_rx_bytes; 993 994 return total_rx_packets; 995 } 996 997 /** 998 * ixgbevf_poll - NAPI polling calback 999 * @napi: napi struct with our devices info in it 1000 * @budget: amount of work driver is allowed to do this pass, in packets 1001 * 1002 * This function will clean more than one or more rings associated with a 1003 * q_vector. 1004 **/ 1005 static int ixgbevf_poll(struct napi_struct *napi, int budget) 1006 { 1007 struct ixgbevf_q_vector *q_vector = 1008 container_of(napi, struct ixgbevf_q_vector, napi); 1009 struct ixgbevf_adapter *adapter = q_vector->adapter; 1010 struct ixgbevf_ring *ring; 1011 int per_ring_budget, work_done = 0; 1012 bool clean_complete = true; 1013 1014 ixgbevf_for_each_ring(ring, q_vector->tx) 1015 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring); 1016 1017 #ifdef CONFIG_NET_RX_BUSY_POLL 1018 if (!ixgbevf_qv_lock_napi(q_vector)) 1019 return budget; 1020 #endif 1021 1022 /* attempt to distribute budget to each queue fairly, but don't allow 1023 * the budget to go below 1 because we'll exit polling 1024 */ 1025 if (q_vector->rx.count > 1) 1026 per_ring_budget = max(budget/q_vector->rx.count, 1); 1027 else 1028 per_ring_budget = budget; 1029 1030 ixgbevf_for_each_ring(ring, q_vector->rx) { 1031 int cleaned = ixgbevf_clean_rx_irq(q_vector, ring, 1032 per_ring_budget); 1033 work_done += cleaned; 1034 clean_complete &= (cleaned < per_ring_budget); 1035 } 1036 1037 #ifdef CONFIG_NET_RX_BUSY_POLL 1038 ixgbevf_qv_unlock_napi(q_vector); 1039 #endif 1040 1041 /* If all work not completed, return budget and keep polling */ 1042 if (!clean_complete) 1043 return budget; 1044 /* all work done, exit the polling mode */ 1045 napi_complete_done(napi, work_done); 1046 if (adapter->rx_itr_setting & 1) 1047 ixgbevf_set_itr(q_vector); 1048 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 1049 !test_bit(__IXGBEVF_REMOVING, &adapter->state)) 1050 ixgbevf_irq_enable_queues(adapter, 1051 1 << q_vector->v_idx); 1052 1053 return 0; 1054 } 1055 1056 /** 1057 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 1058 * @q_vector: structure containing interrupt and ring information 1059 **/ 1060 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) 1061 { 1062 struct ixgbevf_adapter *adapter = q_vector->adapter; 1063 struct ixgbe_hw *hw = &adapter->hw; 1064 int v_idx = q_vector->v_idx; 1065 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 1066 1067 /* set the WDIS bit to not clear the timer bits and cause an 1068 * immediate assertion of the interrupt 1069 */ 1070 itr_reg |= IXGBE_EITR_CNT_WDIS; 1071 1072 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 1073 } 1074 1075 #ifdef CONFIG_NET_RX_BUSY_POLL 1076 /* must be called with local_bh_disable()d */ 1077 static int ixgbevf_busy_poll_recv(struct napi_struct *napi) 1078 { 1079 struct ixgbevf_q_vector *q_vector = 1080 container_of(napi, struct ixgbevf_q_vector, napi); 1081 struct ixgbevf_adapter *adapter = q_vector->adapter; 1082 struct ixgbevf_ring *ring; 1083 int found = 0; 1084 1085 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 1086 return LL_FLUSH_FAILED; 1087 1088 if (!ixgbevf_qv_lock_poll(q_vector)) 1089 return LL_FLUSH_BUSY; 1090 1091 ixgbevf_for_each_ring(ring, q_vector->rx) { 1092 found = ixgbevf_clean_rx_irq(q_vector, ring, 4); 1093 #ifdef BP_EXTENDED_STATS 1094 if (found) 1095 ring->stats.cleaned += found; 1096 else 1097 ring->stats.misses++; 1098 #endif 1099 if (found) 1100 break; 1101 } 1102 1103 ixgbevf_qv_unlock_poll(q_vector); 1104 1105 return found; 1106 } 1107 #endif /* CONFIG_NET_RX_BUSY_POLL */ 1108 1109 /** 1110 * ixgbevf_configure_msix - Configure MSI-X hardware 1111 * @adapter: board private structure 1112 * 1113 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 1114 * interrupts. 1115 **/ 1116 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 1117 { 1118 struct ixgbevf_q_vector *q_vector; 1119 int q_vectors, v_idx; 1120 1121 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1122 adapter->eims_enable_mask = 0; 1123 1124 /* Populate the IVAR table and set the ITR values to the 1125 * corresponding register. 1126 */ 1127 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 1128 struct ixgbevf_ring *ring; 1129 1130 q_vector = adapter->q_vector[v_idx]; 1131 1132 ixgbevf_for_each_ring(ring, q_vector->rx) 1133 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx); 1134 1135 ixgbevf_for_each_ring(ring, q_vector->tx) 1136 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx); 1137 1138 if (q_vector->tx.ring && !q_vector->rx.ring) { 1139 /* Tx only vector */ 1140 if (adapter->tx_itr_setting == 1) 1141 q_vector->itr = IXGBE_10K_ITR; 1142 else 1143 q_vector->itr = adapter->tx_itr_setting; 1144 } else { 1145 /* Rx or Rx/Tx vector */ 1146 if (adapter->rx_itr_setting == 1) 1147 q_vector->itr = IXGBE_20K_ITR; 1148 else 1149 q_vector->itr = adapter->rx_itr_setting; 1150 } 1151 1152 /* add q_vector eims value to global eims_enable_mask */ 1153 adapter->eims_enable_mask |= 1 << v_idx; 1154 1155 ixgbevf_write_eitr(q_vector); 1156 } 1157 1158 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 1159 /* setup eims_other and add value to global eims_enable_mask */ 1160 adapter->eims_other = 1 << v_idx; 1161 adapter->eims_enable_mask |= adapter->eims_other; 1162 } 1163 1164 enum latency_range { 1165 lowest_latency = 0, 1166 low_latency = 1, 1167 bulk_latency = 2, 1168 latency_invalid = 255 1169 }; 1170 1171 /** 1172 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 1173 * @q_vector: structure containing interrupt and ring information 1174 * @ring_container: structure containing ring performance data 1175 * 1176 * Stores a new ITR value based on packets and byte 1177 * counts during the last interrupt. The advantage of per interrupt 1178 * computation is faster updates and more accurate ITR for the current 1179 * traffic pattern. Constants in this function were computed 1180 * based on theoretical maximum wire speed and thresholds were set based 1181 * on testing data as well as attempting to minimize response time 1182 * while increasing bulk throughput. 1183 **/ 1184 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector, 1185 struct ixgbevf_ring_container *ring_container) 1186 { 1187 int bytes = ring_container->total_bytes; 1188 int packets = ring_container->total_packets; 1189 u32 timepassed_us; 1190 u64 bytes_perint; 1191 u8 itr_setting = ring_container->itr; 1192 1193 if (packets == 0) 1194 return; 1195 1196 /* simple throttle rate management 1197 * 0-20MB/s lowest (100000 ints/s) 1198 * 20-100MB/s low (20000 ints/s) 1199 * 100-1249MB/s bulk (8000 ints/s) 1200 */ 1201 /* what was last interrupt timeslice? */ 1202 timepassed_us = q_vector->itr >> 2; 1203 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 1204 1205 switch (itr_setting) { 1206 case lowest_latency: 1207 if (bytes_perint > 10) 1208 itr_setting = low_latency; 1209 break; 1210 case low_latency: 1211 if (bytes_perint > 20) 1212 itr_setting = bulk_latency; 1213 else if (bytes_perint <= 10) 1214 itr_setting = lowest_latency; 1215 break; 1216 case bulk_latency: 1217 if (bytes_perint <= 20) 1218 itr_setting = low_latency; 1219 break; 1220 } 1221 1222 /* clear work counters since we have the values we need */ 1223 ring_container->total_bytes = 0; 1224 ring_container->total_packets = 0; 1225 1226 /* write updated itr to ring container */ 1227 ring_container->itr = itr_setting; 1228 } 1229 1230 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector) 1231 { 1232 u32 new_itr = q_vector->itr; 1233 u8 current_itr; 1234 1235 ixgbevf_update_itr(q_vector, &q_vector->tx); 1236 ixgbevf_update_itr(q_vector, &q_vector->rx); 1237 1238 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 1239 1240 switch (current_itr) { 1241 /* counts and packets in update_itr are dependent on these numbers */ 1242 case lowest_latency: 1243 new_itr = IXGBE_100K_ITR; 1244 break; 1245 case low_latency: 1246 new_itr = IXGBE_20K_ITR; 1247 break; 1248 case bulk_latency: 1249 default: 1250 new_itr = IXGBE_8K_ITR; 1251 break; 1252 } 1253 1254 if (new_itr != q_vector->itr) { 1255 /* do an exponential smoothing */ 1256 new_itr = (10 * new_itr * q_vector->itr) / 1257 ((9 * new_itr) + q_vector->itr); 1258 1259 /* save the algorithm value here */ 1260 q_vector->itr = new_itr; 1261 1262 ixgbevf_write_eitr(q_vector); 1263 } 1264 } 1265 1266 static irqreturn_t ixgbevf_msix_other(int irq, void *data) 1267 { 1268 struct ixgbevf_adapter *adapter = data; 1269 struct ixgbe_hw *hw = &adapter->hw; 1270 1271 hw->mac.get_link_status = 1; 1272 1273 ixgbevf_service_event_schedule(adapter); 1274 1275 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); 1276 1277 return IRQ_HANDLED; 1278 } 1279 1280 /** 1281 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) 1282 * @irq: unused 1283 * @data: pointer to our q_vector struct for this interrupt vector 1284 **/ 1285 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data) 1286 { 1287 struct ixgbevf_q_vector *q_vector = data; 1288 1289 /* EIAM disabled interrupts (on this vector) for us */ 1290 if (q_vector->rx.ring || q_vector->tx.ring) 1291 napi_schedule(&q_vector->napi); 1292 1293 return IRQ_HANDLED; 1294 } 1295 1296 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, 1297 int r_idx) 1298 { 1299 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1300 1301 a->rx_ring[r_idx]->next = q_vector->rx.ring; 1302 q_vector->rx.ring = a->rx_ring[r_idx]; 1303 q_vector->rx.count++; 1304 } 1305 1306 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, 1307 int t_idx) 1308 { 1309 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1310 1311 a->tx_ring[t_idx]->next = q_vector->tx.ring; 1312 q_vector->tx.ring = a->tx_ring[t_idx]; 1313 q_vector->tx.count++; 1314 } 1315 1316 /** 1317 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors 1318 * @adapter: board private structure to initialize 1319 * 1320 * This function maps descriptor rings to the queue-specific vectors 1321 * we were allotted through the MSI-X enabling code. Ideally, we'd have 1322 * one vector per ring/queue, but on a constrained vector budget, we 1323 * group the rings as "efficiently" as possible. You would add new 1324 * mapping configurations in here. 1325 **/ 1326 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter) 1327 { 1328 int q_vectors; 1329 int v_start = 0; 1330 int rxr_idx = 0, txr_idx = 0; 1331 int rxr_remaining = adapter->num_rx_queues; 1332 int txr_remaining = adapter->num_tx_queues; 1333 int i, j; 1334 int rqpv, tqpv; 1335 int err = 0; 1336 1337 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1338 1339 /* The ideal configuration... 1340 * We have enough vectors to map one per queue. 1341 */ 1342 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) { 1343 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) 1344 map_vector_to_rxq(adapter, v_start, rxr_idx); 1345 1346 for (; txr_idx < txr_remaining; v_start++, txr_idx++) 1347 map_vector_to_txq(adapter, v_start, txr_idx); 1348 goto out; 1349 } 1350 1351 /* If we don't have enough vectors for a 1-to-1 1352 * mapping, we'll have to group them so there are 1353 * multiple queues per vector. 1354 */ 1355 /* Re-adjusting *qpv takes care of the remainder. */ 1356 for (i = v_start; i < q_vectors; i++) { 1357 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i); 1358 for (j = 0; j < rqpv; j++) { 1359 map_vector_to_rxq(adapter, i, rxr_idx); 1360 rxr_idx++; 1361 rxr_remaining--; 1362 } 1363 } 1364 for (i = v_start; i < q_vectors; i++) { 1365 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i); 1366 for (j = 0; j < tqpv; j++) { 1367 map_vector_to_txq(adapter, i, txr_idx); 1368 txr_idx++; 1369 txr_remaining--; 1370 } 1371 } 1372 1373 out: 1374 return err; 1375 } 1376 1377 /** 1378 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 1379 * @adapter: board private structure 1380 * 1381 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 1382 * interrupts from the kernel. 1383 **/ 1384 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 1385 { 1386 struct net_device *netdev = adapter->netdev; 1387 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1388 int vector, err; 1389 int ri = 0, ti = 0; 1390 1391 for (vector = 0; vector < q_vectors; vector++) { 1392 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector]; 1393 struct msix_entry *entry = &adapter->msix_entries[vector]; 1394 1395 if (q_vector->tx.ring && q_vector->rx.ring) { 1396 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1397 "%s-%s-%d", netdev->name, "TxRx", ri++); 1398 ti++; 1399 } else if (q_vector->rx.ring) { 1400 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1401 "%s-%s-%d", netdev->name, "rx", ri++); 1402 } else if (q_vector->tx.ring) { 1403 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1404 "%s-%s-%d", netdev->name, "tx", ti++); 1405 } else { 1406 /* skip this unused q_vector */ 1407 continue; 1408 } 1409 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0, 1410 q_vector->name, q_vector); 1411 if (err) { 1412 hw_dbg(&adapter->hw, 1413 "request_irq failed for MSIX interrupt Error: %d\n", 1414 err); 1415 goto free_queue_irqs; 1416 } 1417 } 1418 1419 err = request_irq(adapter->msix_entries[vector].vector, 1420 &ixgbevf_msix_other, 0, netdev->name, adapter); 1421 if (err) { 1422 hw_dbg(&adapter->hw, "request_irq for msix_other failed: %d\n", 1423 err); 1424 goto free_queue_irqs; 1425 } 1426 1427 return 0; 1428 1429 free_queue_irqs: 1430 while (vector) { 1431 vector--; 1432 free_irq(adapter->msix_entries[vector].vector, 1433 adapter->q_vector[vector]); 1434 } 1435 /* This failure is non-recoverable - it indicates the system is 1436 * out of MSIX vector resources and the VF driver cannot run 1437 * without them. Set the number of msix vectors to zero 1438 * indicating that not enough can be allocated. The error 1439 * will be returned to the user indicating device open failed. 1440 * Any further attempts to force the driver to open will also 1441 * fail. The only way to recover is to unload the driver and 1442 * reload it again. If the system has recovered some MSIX 1443 * vectors then it may succeed. 1444 */ 1445 adapter->num_msix_vectors = 0; 1446 return err; 1447 } 1448 1449 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter) 1450 { 1451 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1452 1453 for (i = 0; i < q_vectors; i++) { 1454 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i]; 1455 1456 q_vector->rx.ring = NULL; 1457 q_vector->tx.ring = NULL; 1458 q_vector->rx.count = 0; 1459 q_vector->tx.count = 0; 1460 } 1461 } 1462 1463 /** 1464 * ixgbevf_request_irq - initialize interrupts 1465 * @adapter: board private structure 1466 * 1467 * Attempts to configure interrupts using the best available 1468 * capabilities of the hardware and kernel. 1469 **/ 1470 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 1471 { 1472 int err = 0; 1473 1474 err = ixgbevf_request_msix_irqs(adapter); 1475 1476 if (err) 1477 hw_dbg(&adapter->hw, "request_irq failed, Error %d\n", err); 1478 1479 return err; 1480 } 1481 1482 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 1483 { 1484 int i, q_vectors; 1485 1486 q_vectors = adapter->num_msix_vectors; 1487 i = q_vectors - 1; 1488 1489 free_irq(adapter->msix_entries[i].vector, adapter); 1490 i--; 1491 1492 for (; i >= 0; i--) { 1493 /* free only the irqs that were actually requested */ 1494 if (!adapter->q_vector[i]->rx.ring && 1495 !adapter->q_vector[i]->tx.ring) 1496 continue; 1497 1498 free_irq(adapter->msix_entries[i].vector, 1499 adapter->q_vector[i]); 1500 } 1501 1502 ixgbevf_reset_q_vectors(adapter); 1503 } 1504 1505 /** 1506 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1507 * @adapter: board private structure 1508 **/ 1509 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1510 { 1511 struct ixgbe_hw *hw = &adapter->hw; 1512 int i; 1513 1514 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0); 1515 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1516 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0); 1517 1518 IXGBE_WRITE_FLUSH(hw); 1519 1520 for (i = 0; i < adapter->num_msix_vectors; i++) 1521 synchronize_irq(adapter->msix_entries[i].vector); 1522 } 1523 1524 /** 1525 * ixgbevf_irq_enable - Enable default interrupt generation settings 1526 * @adapter: board private structure 1527 **/ 1528 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) 1529 { 1530 struct ixgbe_hw *hw = &adapter->hw; 1531 1532 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask); 1533 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask); 1534 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); 1535 } 1536 1537 /** 1538 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset 1539 * @adapter: board private structure 1540 * @ring: structure containing ring specific data 1541 * 1542 * Configure the Tx descriptor ring after a reset. 1543 **/ 1544 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 1545 struct ixgbevf_ring *ring) 1546 { 1547 struct ixgbe_hw *hw = &adapter->hw; 1548 u64 tdba = ring->dma; 1549 int wait_loop = 10; 1550 u32 txdctl = IXGBE_TXDCTL_ENABLE; 1551 u8 reg_idx = ring->reg_idx; 1552 1553 /* disable queue to avoid issues while updating state */ 1554 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 1555 IXGBE_WRITE_FLUSH(hw); 1556 1557 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 1558 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32); 1559 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx), 1560 ring->count * sizeof(union ixgbe_adv_tx_desc)); 1561 1562 /* disable head writeback */ 1563 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0); 1564 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0); 1565 1566 /* enable relaxed ordering */ 1567 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx), 1568 (IXGBE_DCA_TXCTRL_DESC_RRO_EN | 1569 IXGBE_DCA_TXCTRL_DATA_RRO_EN)); 1570 1571 /* reset head and tail pointers */ 1572 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0); 1573 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0); 1574 ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx); 1575 1576 /* reset ntu and ntc to place SW in sync with hardwdare */ 1577 ring->next_to_clean = 0; 1578 ring->next_to_use = 0; 1579 1580 /* In order to avoid issues WTHRESH + PTHRESH should always be equal 1581 * to or less than the number of on chip descriptors, which is 1582 * currently 40. 1583 */ 1584 txdctl |= (8 << 16); /* WTHRESH = 8 */ 1585 1586 /* Setting PTHRESH to 32 both improves performance */ 1587 txdctl |= (1 << 8) | /* HTHRESH = 1 */ 1588 32; /* PTHRESH = 32 */ 1589 1590 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); 1591 1592 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl); 1593 1594 /* poll to verify queue is enabled */ 1595 do { 1596 usleep_range(1000, 2000); 1597 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx)); 1598 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 1599 if (!wait_loop) 1600 pr_err("Could not enable Tx Queue %d\n", reg_idx); 1601 } 1602 1603 /** 1604 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1605 * @adapter: board private structure 1606 * 1607 * Configure the Tx unit of the MAC after a reset. 1608 **/ 1609 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1610 { 1611 u32 i; 1612 1613 /* Setup the HW Tx Head and Tail descriptor pointers */ 1614 for (i = 0; i < adapter->num_tx_queues; i++) 1615 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]); 1616 } 1617 1618 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1619 1620 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) 1621 { 1622 struct ixgbe_hw *hw = &adapter->hw; 1623 u32 srrctl; 1624 1625 srrctl = IXGBE_SRRCTL_DROP_EN; 1626 1627 srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; 1628 srrctl |= IXGBEVF_RX_BUFSZ >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1629 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1630 1631 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1632 } 1633 1634 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) 1635 { 1636 struct ixgbe_hw *hw = &adapter->hw; 1637 1638 /* PSRTYPE must be initialized in 82599 */ 1639 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 1640 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR | 1641 IXGBE_PSRTYPE_L2HDR; 1642 1643 if (adapter->num_rx_queues > 1) 1644 psrtype |= 1 << 29; 1645 1646 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1647 } 1648 1649 #define IXGBEVF_MAX_RX_DESC_POLL 10 1650 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, 1651 struct ixgbevf_ring *ring) 1652 { 1653 struct ixgbe_hw *hw = &adapter->hw; 1654 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1655 u32 rxdctl; 1656 u8 reg_idx = ring->reg_idx; 1657 1658 if (IXGBE_REMOVED(hw->hw_addr)) 1659 return; 1660 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1661 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1662 1663 /* write value back with RXDCTL.ENABLE bit cleared */ 1664 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1665 1666 /* the hardware may take up to 100us to really disable the Rx queue */ 1667 do { 1668 udelay(10); 1669 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1670 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 1671 1672 if (!wait_loop) 1673 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", 1674 reg_idx); 1675 } 1676 1677 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1678 struct ixgbevf_ring *ring) 1679 { 1680 struct ixgbe_hw *hw = &adapter->hw; 1681 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1682 u32 rxdctl; 1683 u8 reg_idx = ring->reg_idx; 1684 1685 if (IXGBE_REMOVED(hw->hw_addr)) 1686 return; 1687 do { 1688 usleep_range(1000, 2000); 1689 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1690 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 1691 1692 if (!wait_loop) 1693 pr_err("RXDCTL.ENABLE queue %d not set while polling\n", 1694 reg_idx); 1695 } 1696 1697 static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) 1698 { 1699 struct ixgbe_hw *hw = &adapter->hw; 1700 u32 vfmrqc = 0, vfreta = 0; 1701 u16 rss_i = adapter->num_rx_queues; 1702 u8 i, j; 1703 1704 /* Fill out hash function seeds */ 1705 netdev_rss_key_fill(adapter->rss_key, sizeof(adapter->rss_key)); 1706 for (i = 0; i < IXGBEVF_VFRSSRK_REGS; i++) 1707 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), adapter->rss_key[i]); 1708 1709 for (i = 0, j = 0; i < IXGBEVF_X550_VFRETA_SIZE; i++, j++) { 1710 if (j == rss_i) 1711 j = 0; 1712 1713 adapter->rss_indir_tbl[i] = j; 1714 1715 vfreta |= j << (i & 0x3) * 8; 1716 if ((i & 3) == 3) { 1717 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), vfreta); 1718 vfreta = 0; 1719 } 1720 } 1721 1722 /* Perform hash on these packet types */ 1723 vfmrqc |= IXGBE_VFMRQC_RSS_FIELD_IPV4 | 1724 IXGBE_VFMRQC_RSS_FIELD_IPV4_TCP | 1725 IXGBE_VFMRQC_RSS_FIELD_IPV6 | 1726 IXGBE_VFMRQC_RSS_FIELD_IPV6_TCP; 1727 1728 vfmrqc |= IXGBE_VFMRQC_RSSEN; 1729 1730 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); 1731 } 1732 1733 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, 1734 struct ixgbevf_ring *ring) 1735 { 1736 struct ixgbe_hw *hw = &adapter->hw; 1737 u64 rdba = ring->dma; 1738 u32 rxdctl; 1739 u8 reg_idx = ring->reg_idx; 1740 1741 /* disable queue to avoid issues while updating state */ 1742 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1743 ixgbevf_disable_rx_queue(adapter, ring); 1744 1745 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 1746 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32); 1747 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx), 1748 ring->count * sizeof(union ixgbe_adv_rx_desc)); 1749 1750 /* enable relaxed ordering */ 1751 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1752 IXGBE_DCA_RXCTRL_DESC_RRO_EN); 1753 1754 /* reset head and tail pointers */ 1755 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0); 1756 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); 1757 ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); 1758 1759 /* reset ntu and ntc to place SW in sync with hardwdare */ 1760 ring->next_to_clean = 0; 1761 ring->next_to_use = 0; 1762 ring->next_to_alloc = 0; 1763 1764 ixgbevf_configure_srrctl(adapter, reg_idx); 1765 1766 /* allow any size packet since we can handle overflow */ 1767 rxdctl &= ~IXGBE_RXDCTL_RLPML_EN; 1768 1769 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1770 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1771 1772 ixgbevf_rx_desc_queue_enable(adapter, ring); 1773 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); 1774 } 1775 1776 /** 1777 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1778 * @adapter: board private structure 1779 * 1780 * Configure the Rx unit of the MAC after a reset. 1781 **/ 1782 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1783 { 1784 int i; 1785 struct ixgbe_hw *hw = &adapter->hw; 1786 struct net_device *netdev = adapter->netdev; 1787 1788 ixgbevf_setup_psrtype(adapter); 1789 if (hw->mac.type >= ixgbe_mac_X550_vf) 1790 ixgbevf_setup_vfmrqc(adapter); 1791 1792 /* notify the PF of our intent to use this size of frame */ 1793 ixgbevf_rlpml_set_vf(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN); 1794 1795 /* Setup the HW Rx Head and Tail Descriptor Pointers and 1796 * the Base and Length of the Rx Descriptor Ring 1797 */ 1798 for (i = 0; i < adapter->num_rx_queues; i++) 1799 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]); 1800 } 1801 1802 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, 1803 __be16 proto, u16 vid) 1804 { 1805 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1806 struct ixgbe_hw *hw = &adapter->hw; 1807 int err; 1808 1809 spin_lock_bh(&adapter->mbx_lock); 1810 1811 /* add VID to filter table */ 1812 err = hw->mac.ops.set_vfta(hw, vid, 0, true); 1813 1814 spin_unlock_bh(&adapter->mbx_lock); 1815 1816 /* translate error return types so error makes sense */ 1817 if (err == IXGBE_ERR_MBX) 1818 return -EIO; 1819 1820 if (err == IXGBE_ERR_INVALID_ARGUMENT) 1821 return -EACCES; 1822 1823 set_bit(vid, adapter->active_vlans); 1824 1825 return err; 1826 } 1827 1828 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, 1829 __be16 proto, u16 vid) 1830 { 1831 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1832 struct ixgbe_hw *hw = &adapter->hw; 1833 int err = -EOPNOTSUPP; 1834 1835 spin_lock_bh(&adapter->mbx_lock); 1836 1837 /* remove VID from filter table */ 1838 err = hw->mac.ops.set_vfta(hw, vid, 0, false); 1839 1840 spin_unlock_bh(&adapter->mbx_lock); 1841 1842 clear_bit(vid, adapter->active_vlans); 1843 1844 return err; 1845 } 1846 1847 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 1848 { 1849 u16 vid; 1850 1851 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1852 ixgbevf_vlan_rx_add_vid(adapter->netdev, 1853 htons(ETH_P_8021Q), vid); 1854 } 1855 1856 static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 1857 { 1858 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1859 struct ixgbe_hw *hw = &adapter->hw; 1860 int count = 0; 1861 1862 if ((netdev_uc_count(netdev)) > 10) { 1863 pr_err("Too many unicast filters - No Space\n"); 1864 return -ENOSPC; 1865 } 1866 1867 if (!netdev_uc_empty(netdev)) { 1868 struct netdev_hw_addr *ha; 1869 1870 netdev_for_each_uc_addr(ha, netdev) { 1871 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 1872 udelay(200); 1873 } 1874 } else { 1875 /* If the list is empty then send message to PF driver to 1876 * clear all MAC VLANs on this VF. 1877 */ 1878 hw->mac.ops.set_uc_addr(hw, 0, NULL); 1879 } 1880 1881 return count; 1882 } 1883 1884 /** 1885 * ixgbevf_set_rx_mode - Multicast and unicast set 1886 * @netdev: network interface device structure 1887 * 1888 * The set_rx_method entry point is called whenever the multicast address 1889 * list, unicast address list or the network interface flags are updated. 1890 * This routine is responsible for configuring the hardware for proper 1891 * multicast mode and configuring requested unicast filters. 1892 **/ 1893 static void ixgbevf_set_rx_mode(struct net_device *netdev) 1894 { 1895 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1896 struct ixgbe_hw *hw = &adapter->hw; 1897 unsigned int flags = netdev->flags; 1898 int xcast_mode; 1899 1900 xcast_mode = (flags & IFF_ALLMULTI) ? IXGBEVF_XCAST_MODE_ALLMULTI : 1901 (flags & (IFF_BROADCAST | IFF_MULTICAST)) ? 1902 IXGBEVF_XCAST_MODE_MULTI : IXGBEVF_XCAST_MODE_NONE; 1903 1904 spin_lock_bh(&adapter->mbx_lock); 1905 1906 hw->mac.ops.update_xcast_mode(hw, netdev, xcast_mode); 1907 1908 /* reprogram multicast list */ 1909 hw->mac.ops.update_mc_addr_list(hw, netdev); 1910 1911 ixgbevf_write_uc_addr_list(netdev); 1912 1913 spin_unlock_bh(&adapter->mbx_lock); 1914 } 1915 1916 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 1917 { 1918 int q_idx; 1919 struct ixgbevf_q_vector *q_vector; 1920 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1921 1922 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1923 q_vector = adapter->q_vector[q_idx]; 1924 #ifdef CONFIG_NET_RX_BUSY_POLL 1925 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]); 1926 #endif 1927 napi_enable(&q_vector->napi); 1928 } 1929 } 1930 1931 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 1932 { 1933 int q_idx; 1934 struct ixgbevf_q_vector *q_vector; 1935 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1936 1937 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1938 q_vector = adapter->q_vector[q_idx]; 1939 napi_disable(&q_vector->napi); 1940 #ifdef CONFIG_NET_RX_BUSY_POLL 1941 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) { 1942 pr_info("QV %d locked\n", q_idx); 1943 usleep_range(1000, 20000); 1944 } 1945 #endif /* CONFIG_NET_RX_BUSY_POLL */ 1946 } 1947 } 1948 1949 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter) 1950 { 1951 struct ixgbe_hw *hw = &adapter->hw; 1952 unsigned int def_q = 0; 1953 unsigned int num_tcs = 0; 1954 unsigned int num_rx_queues = adapter->num_rx_queues; 1955 unsigned int num_tx_queues = adapter->num_tx_queues; 1956 int err; 1957 1958 spin_lock_bh(&adapter->mbx_lock); 1959 1960 /* fetch queue configuration from the PF */ 1961 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 1962 1963 spin_unlock_bh(&adapter->mbx_lock); 1964 1965 if (err) 1966 return err; 1967 1968 if (num_tcs > 1) { 1969 /* we need only one Tx queue */ 1970 num_tx_queues = 1; 1971 1972 /* update default Tx ring register index */ 1973 adapter->tx_ring[0]->reg_idx = def_q; 1974 1975 /* we need as many queues as traffic classes */ 1976 num_rx_queues = num_tcs; 1977 } 1978 1979 /* if we have a bad config abort request queue reset */ 1980 if ((adapter->num_rx_queues != num_rx_queues) || 1981 (adapter->num_tx_queues != num_tx_queues)) { 1982 /* force mailbox timeout to prevent further messages */ 1983 hw->mbx.timeout = 0; 1984 1985 /* wait for watchdog to come around and bail us out */ 1986 adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED; 1987 } 1988 1989 return 0; 1990 } 1991 1992 static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 1993 { 1994 ixgbevf_configure_dcb(adapter); 1995 1996 ixgbevf_set_rx_mode(adapter->netdev); 1997 1998 ixgbevf_restore_vlan(adapter); 1999 2000 ixgbevf_configure_tx(adapter); 2001 ixgbevf_configure_rx(adapter); 2002 } 2003 2004 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 2005 { 2006 /* Only save pre-reset stats if there are some */ 2007 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 2008 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 2009 adapter->stats.base_vfgprc; 2010 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 2011 adapter->stats.base_vfgptc; 2012 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 2013 adapter->stats.base_vfgorc; 2014 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 2015 adapter->stats.base_vfgotc; 2016 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 2017 adapter->stats.base_vfmprc; 2018 } 2019 } 2020 2021 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 2022 { 2023 struct ixgbe_hw *hw = &adapter->hw; 2024 2025 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2026 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2027 adapter->stats.last_vfgorc |= 2028 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2029 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2030 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2031 adapter->stats.last_vfgotc |= 2032 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2033 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2034 2035 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 2036 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 2037 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 2038 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 2039 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 2040 } 2041 2042 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) 2043 { 2044 struct ixgbe_hw *hw = &adapter->hw; 2045 int api[] = { ixgbe_mbox_api_12, 2046 ixgbe_mbox_api_11, 2047 ixgbe_mbox_api_10, 2048 ixgbe_mbox_api_unknown }; 2049 int err = 0, idx = 0; 2050 2051 spin_lock_bh(&adapter->mbx_lock); 2052 2053 while (api[idx] != ixgbe_mbox_api_unknown) { 2054 err = ixgbevf_negotiate_api_version(hw, api[idx]); 2055 if (!err) 2056 break; 2057 idx++; 2058 } 2059 2060 spin_unlock_bh(&adapter->mbx_lock); 2061 } 2062 2063 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 2064 { 2065 struct net_device *netdev = adapter->netdev; 2066 struct ixgbe_hw *hw = &adapter->hw; 2067 2068 ixgbevf_configure_msix(adapter); 2069 2070 spin_lock_bh(&adapter->mbx_lock); 2071 2072 if (is_valid_ether_addr(hw->mac.addr)) 2073 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 2074 else 2075 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 2076 2077 spin_unlock_bh(&adapter->mbx_lock); 2078 2079 smp_mb__before_atomic(); 2080 clear_bit(__IXGBEVF_DOWN, &adapter->state); 2081 ixgbevf_napi_enable_all(adapter); 2082 2083 /* clear any pending interrupts, may auto mask */ 2084 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2085 ixgbevf_irq_enable(adapter); 2086 2087 /* enable transmits */ 2088 netif_tx_start_all_queues(netdev); 2089 2090 ixgbevf_save_reset_stats(adapter); 2091 ixgbevf_init_last_counter_stats(adapter); 2092 2093 hw->mac.get_link_status = 1; 2094 mod_timer(&adapter->service_timer, jiffies); 2095 } 2096 2097 void ixgbevf_up(struct ixgbevf_adapter *adapter) 2098 { 2099 ixgbevf_configure(adapter); 2100 2101 ixgbevf_up_complete(adapter); 2102 } 2103 2104 /** 2105 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 2106 * @rx_ring: ring to free buffers from 2107 **/ 2108 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) 2109 { 2110 struct device *dev = rx_ring->dev; 2111 unsigned long size; 2112 unsigned int i; 2113 2114 /* Free Rx ring sk_buff */ 2115 if (rx_ring->skb) { 2116 dev_kfree_skb(rx_ring->skb); 2117 rx_ring->skb = NULL; 2118 } 2119 2120 /* ring already cleared, nothing to do */ 2121 if (!rx_ring->rx_buffer_info) 2122 return; 2123 2124 /* Free all the Rx ring pages */ 2125 for (i = 0; i < rx_ring->count; i++) { 2126 struct ixgbevf_rx_buffer *rx_buffer; 2127 2128 rx_buffer = &rx_ring->rx_buffer_info[i]; 2129 if (rx_buffer->dma) 2130 dma_unmap_page(dev, rx_buffer->dma, 2131 PAGE_SIZE, DMA_FROM_DEVICE); 2132 rx_buffer->dma = 0; 2133 if (rx_buffer->page) 2134 __free_page(rx_buffer->page); 2135 rx_buffer->page = NULL; 2136 } 2137 2138 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 2139 memset(rx_ring->rx_buffer_info, 0, size); 2140 2141 /* Zero out the descriptor ring */ 2142 memset(rx_ring->desc, 0, rx_ring->size); 2143 } 2144 2145 /** 2146 * ixgbevf_clean_tx_ring - Free Tx Buffers 2147 * @tx_ring: ring to be cleaned 2148 **/ 2149 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) 2150 { 2151 struct ixgbevf_tx_buffer *tx_buffer_info; 2152 unsigned long size; 2153 unsigned int i; 2154 2155 if (!tx_ring->tx_buffer_info) 2156 return; 2157 2158 /* Free all the Tx ring sk_buffs */ 2159 for (i = 0; i < tx_ring->count; i++) { 2160 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2161 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 2162 } 2163 2164 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2165 memset(tx_ring->tx_buffer_info, 0, size); 2166 2167 memset(tx_ring->desc, 0, tx_ring->size); 2168 } 2169 2170 /** 2171 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 2172 * @adapter: board private structure 2173 **/ 2174 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 2175 { 2176 int i; 2177 2178 for (i = 0; i < adapter->num_rx_queues; i++) 2179 ixgbevf_clean_rx_ring(adapter->rx_ring[i]); 2180 } 2181 2182 /** 2183 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 2184 * @adapter: board private structure 2185 **/ 2186 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 2187 { 2188 int i; 2189 2190 for (i = 0; i < adapter->num_tx_queues; i++) 2191 ixgbevf_clean_tx_ring(adapter->tx_ring[i]); 2192 } 2193 2194 void ixgbevf_down(struct ixgbevf_adapter *adapter) 2195 { 2196 struct net_device *netdev = adapter->netdev; 2197 struct ixgbe_hw *hw = &adapter->hw; 2198 int i; 2199 2200 /* signal that we are down to the interrupt handler */ 2201 if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state)) 2202 return; /* do nothing if already down */ 2203 2204 /* disable all enabled Rx queues */ 2205 for (i = 0; i < adapter->num_rx_queues; i++) 2206 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]); 2207 2208 usleep_range(10000, 20000); 2209 2210 netif_tx_stop_all_queues(netdev); 2211 2212 /* call carrier off first to avoid false dev_watchdog timeouts */ 2213 netif_carrier_off(netdev); 2214 netif_tx_disable(netdev); 2215 2216 ixgbevf_irq_disable(adapter); 2217 2218 ixgbevf_napi_disable_all(adapter); 2219 2220 del_timer_sync(&adapter->service_timer); 2221 2222 /* disable transmits in the hardware now that interrupts are off */ 2223 for (i = 0; i < adapter->num_tx_queues; i++) { 2224 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 2225 2226 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), 2227 IXGBE_TXDCTL_SWFLSH); 2228 } 2229 2230 if (!pci_channel_offline(adapter->pdev)) 2231 ixgbevf_reset(adapter); 2232 2233 ixgbevf_clean_all_tx_rings(adapter); 2234 ixgbevf_clean_all_rx_rings(adapter); 2235 } 2236 2237 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 2238 { 2239 WARN_ON(in_interrupt()); 2240 2241 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 2242 msleep(1); 2243 2244 ixgbevf_down(adapter); 2245 ixgbevf_up(adapter); 2246 2247 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 2248 } 2249 2250 void ixgbevf_reset(struct ixgbevf_adapter *adapter) 2251 { 2252 struct ixgbe_hw *hw = &adapter->hw; 2253 struct net_device *netdev = adapter->netdev; 2254 2255 if (hw->mac.ops.reset_hw(hw)) { 2256 hw_dbg(hw, "PF still resetting\n"); 2257 } else { 2258 hw->mac.ops.init_hw(hw); 2259 ixgbevf_negotiate_api(adapter); 2260 } 2261 2262 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 2263 memcpy(netdev->dev_addr, adapter->hw.mac.addr, 2264 netdev->addr_len); 2265 memcpy(netdev->perm_addr, adapter->hw.mac.addr, 2266 netdev->addr_len); 2267 } 2268 2269 adapter->last_reset = jiffies; 2270 } 2271 2272 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 2273 int vectors) 2274 { 2275 int vector_threshold; 2276 2277 /* We'll want at least 2 (vector_threshold): 2278 * 1) TxQ[0] + RxQ[0] handler 2279 * 2) Other (Link Status Change, etc.) 2280 */ 2281 vector_threshold = MIN_MSIX_COUNT; 2282 2283 /* The more we get, the more we will assign to Tx/Rx Cleanup 2284 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 2285 * Right now, we simply care about how many we'll get; we'll 2286 * set them up later while requesting irq's. 2287 */ 2288 vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 2289 vector_threshold, vectors); 2290 2291 if (vectors < 0) { 2292 dev_err(&adapter->pdev->dev, 2293 "Unable to allocate MSI-X interrupts\n"); 2294 kfree(adapter->msix_entries); 2295 adapter->msix_entries = NULL; 2296 return vectors; 2297 } 2298 2299 /* Adjust for only the vectors we'll use, which is minimum 2300 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 2301 * vectors we were allocated. 2302 */ 2303 adapter->num_msix_vectors = vectors; 2304 2305 return 0; 2306 } 2307 2308 /** 2309 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent 2310 * @adapter: board private structure to initialize 2311 * 2312 * This is the top level queue allocation routine. The order here is very 2313 * important, starting with the "most" number of features turned on at once, 2314 * and ending with the smallest set of features. This way large combinations 2315 * can be allocated if they're turned on, and smaller combinations are the 2316 * fallthrough conditions. 2317 * 2318 **/ 2319 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 2320 { 2321 struct ixgbe_hw *hw = &adapter->hw; 2322 unsigned int def_q = 0; 2323 unsigned int num_tcs = 0; 2324 int err; 2325 2326 /* Start with base case */ 2327 adapter->num_rx_queues = 1; 2328 adapter->num_tx_queues = 1; 2329 2330 spin_lock_bh(&adapter->mbx_lock); 2331 2332 /* fetch queue configuration from the PF */ 2333 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2334 2335 spin_unlock_bh(&adapter->mbx_lock); 2336 2337 if (err) 2338 return; 2339 2340 /* we need as many queues as traffic classes */ 2341 if (num_tcs > 1) { 2342 adapter->num_rx_queues = num_tcs; 2343 } else { 2344 u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES); 2345 2346 switch (hw->api_version) { 2347 case ixgbe_mbox_api_11: 2348 case ixgbe_mbox_api_12: 2349 adapter->num_rx_queues = rss; 2350 adapter->num_tx_queues = rss; 2351 default: 2352 break; 2353 } 2354 } 2355 } 2356 2357 /** 2358 * ixgbevf_alloc_queues - Allocate memory for all rings 2359 * @adapter: board private structure to initialize 2360 * 2361 * We allocate one ring per queue at run-time since we don't know the 2362 * number of queues at compile-time. The polling_netdev array is 2363 * intended for Multiqueue, but should work fine with a single queue. 2364 **/ 2365 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) 2366 { 2367 struct ixgbevf_ring *ring; 2368 int rx = 0, tx = 0; 2369 2370 for (; tx < adapter->num_tx_queues; tx++) { 2371 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 2372 if (!ring) 2373 goto err_allocation; 2374 2375 ring->dev = &adapter->pdev->dev; 2376 ring->netdev = adapter->netdev; 2377 ring->count = adapter->tx_ring_count; 2378 ring->queue_index = tx; 2379 ring->reg_idx = tx; 2380 2381 adapter->tx_ring[tx] = ring; 2382 } 2383 2384 for (; rx < adapter->num_rx_queues; rx++) { 2385 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 2386 if (!ring) 2387 goto err_allocation; 2388 2389 ring->dev = &adapter->pdev->dev; 2390 ring->netdev = adapter->netdev; 2391 2392 ring->count = adapter->rx_ring_count; 2393 ring->queue_index = rx; 2394 ring->reg_idx = rx; 2395 2396 adapter->rx_ring[rx] = ring; 2397 } 2398 2399 return 0; 2400 2401 err_allocation: 2402 while (tx) { 2403 kfree(adapter->tx_ring[--tx]); 2404 adapter->tx_ring[tx] = NULL; 2405 } 2406 2407 while (rx) { 2408 kfree(adapter->rx_ring[--rx]); 2409 adapter->rx_ring[rx] = NULL; 2410 } 2411 return -ENOMEM; 2412 } 2413 2414 /** 2415 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 2416 * @adapter: board private structure to initialize 2417 * 2418 * Attempt to configure the interrupts using the best available 2419 * capabilities of the hardware and the kernel. 2420 **/ 2421 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 2422 { 2423 struct net_device *netdev = adapter->netdev; 2424 int err = 0; 2425 int vector, v_budget; 2426 2427 /* It's easy to be greedy for MSI-X vectors, but it really 2428 * doesn't do us much good if we have a lot more vectors 2429 * than CPU's. So let's be conservative and only ask for 2430 * (roughly) the same number of vectors as there are CPU's. 2431 * The default is to use pairs of vectors. 2432 */ 2433 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); 2434 v_budget = min_t(int, v_budget, num_online_cpus()); 2435 v_budget += NON_Q_VECTORS; 2436 2437 /* A failure in MSI-X entry allocation isn't fatal, but it does 2438 * mean we disable MSI-X capabilities of the adapter. 2439 */ 2440 adapter->msix_entries = kcalloc(v_budget, 2441 sizeof(struct msix_entry), GFP_KERNEL); 2442 if (!adapter->msix_entries) { 2443 err = -ENOMEM; 2444 goto out; 2445 } 2446 2447 for (vector = 0; vector < v_budget; vector++) 2448 adapter->msix_entries[vector].entry = vector; 2449 2450 err = ixgbevf_acquire_msix_vectors(adapter, v_budget); 2451 if (err) 2452 goto out; 2453 2454 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 2455 if (err) 2456 goto out; 2457 2458 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 2459 2460 out: 2461 return err; 2462 } 2463 2464 /** 2465 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 2466 * @adapter: board private structure to initialize 2467 * 2468 * We allocate one q_vector per queue interrupt. If allocation fails we 2469 * return -ENOMEM. 2470 **/ 2471 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 2472 { 2473 int q_idx, num_q_vectors; 2474 struct ixgbevf_q_vector *q_vector; 2475 2476 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2477 2478 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2479 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL); 2480 if (!q_vector) 2481 goto err_out; 2482 q_vector->adapter = adapter; 2483 q_vector->v_idx = q_idx; 2484 netif_napi_add(adapter->netdev, &q_vector->napi, 2485 ixgbevf_poll, 64); 2486 #ifdef CONFIG_NET_RX_BUSY_POLL 2487 napi_hash_add(&q_vector->napi); 2488 #endif 2489 adapter->q_vector[q_idx] = q_vector; 2490 } 2491 2492 return 0; 2493 2494 err_out: 2495 while (q_idx) { 2496 q_idx--; 2497 q_vector = adapter->q_vector[q_idx]; 2498 #ifdef CONFIG_NET_RX_BUSY_POLL 2499 napi_hash_del(&q_vector->napi); 2500 #endif 2501 netif_napi_del(&q_vector->napi); 2502 kfree(q_vector); 2503 adapter->q_vector[q_idx] = NULL; 2504 } 2505 return -ENOMEM; 2506 } 2507 2508 /** 2509 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 2510 * @adapter: board private structure to initialize 2511 * 2512 * This function frees the memory allocated to the q_vectors. In addition if 2513 * NAPI is enabled it will delete any references to the NAPI struct prior 2514 * to freeing the q_vector. 2515 **/ 2516 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 2517 { 2518 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2519 2520 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2521 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; 2522 2523 adapter->q_vector[q_idx] = NULL; 2524 #ifdef CONFIG_NET_RX_BUSY_POLL 2525 napi_hash_del(&q_vector->napi); 2526 #endif 2527 netif_napi_del(&q_vector->napi); 2528 kfree(q_vector); 2529 } 2530 } 2531 2532 /** 2533 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 2534 * @adapter: board private structure 2535 * 2536 **/ 2537 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 2538 { 2539 pci_disable_msix(adapter->pdev); 2540 kfree(adapter->msix_entries); 2541 adapter->msix_entries = NULL; 2542 } 2543 2544 /** 2545 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 2546 * @adapter: board private structure to initialize 2547 * 2548 **/ 2549 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 2550 { 2551 int err; 2552 2553 /* Number of supported queues */ 2554 ixgbevf_set_num_queues(adapter); 2555 2556 err = ixgbevf_set_interrupt_capability(adapter); 2557 if (err) { 2558 hw_dbg(&adapter->hw, 2559 "Unable to setup interrupt capabilities\n"); 2560 goto err_set_interrupt; 2561 } 2562 2563 err = ixgbevf_alloc_q_vectors(adapter); 2564 if (err) { 2565 hw_dbg(&adapter->hw, "Unable to allocate memory for queue vectors\n"); 2566 goto err_alloc_q_vectors; 2567 } 2568 2569 err = ixgbevf_alloc_queues(adapter); 2570 if (err) { 2571 pr_err("Unable to allocate memory for queues\n"); 2572 goto err_alloc_queues; 2573 } 2574 2575 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u\n", 2576 (adapter->num_rx_queues > 1) ? "Enabled" : 2577 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues); 2578 2579 set_bit(__IXGBEVF_DOWN, &adapter->state); 2580 2581 return 0; 2582 err_alloc_queues: 2583 ixgbevf_free_q_vectors(adapter); 2584 err_alloc_q_vectors: 2585 ixgbevf_reset_interrupt_capability(adapter); 2586 err_set_interrupt: 2587 return err; 2588 } 2589 2590 /** 2591 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings 2592 * @adapter: board private structure to clear interrupt scheme on 2593 * 2594 * We go through and clear interrupt specific resources and reset the structure 2595 * to pre-load conditions 2596 **/ 2597 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) 2598 { 2599 int i; 2600 2601 for (i = 0; i < adapter->num_tx_queues; i++) { 2602 kfree(adapter->tx_ring[i]); 2603 adapter->tx_ring[i] = NULL; 2604 } 2605 for (i = 0; i < adapter->num_rx_queues; i++) { 2606 kfree(adapter->rx_ring[i]); 2607 adapter->rx_ring[i] = NULL; 2608 } 2609 2610 adapter->num_tx_queues = 0; 2611 adapter->num_rx_queues = 0; 2612 2613 ixgbevf_free_q_vectors(adapter); 2614 ixgbevf_reset_interrupt_capability(adapter); 2615 } 2616 2617 /** 2618 * ixgbevf_sw_init - Initialize general software structures 2619 * @adapter: board private structure to initialize 2620 * 2621 * ixgbevf_sw_init initializes the Adapter private data structure. 2622 * Fields are initialized based on PCI device information and 2623 * OS network device settings (MTU size). 2624 **/ 2625 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 2626 { 2627 struct ixgbe_hw *hw = &adapter->hw; 2628 struct pci_dev *pdev = adapter->pdev; 2629 struct net_device *netdev = adapter->netdev; 2630 int err; 2631 2632 /* PCI config space info */ 2633 hw->vendor_id = pdev->vendor; 2634 hw->device_id = pdev->device; 2635 hw->revision_id = pdev->revision; 2636 hw->subsystem_vendor_id = pdev->subsystem_vendor; 2637 hw->subsystem_device_id = pdev->subsystem_device; 2638 2639 hw->mbx.ops.init_params(hw); 2640 2641 /* assume legacy case in which PF would only give VF 2 queues */ 2642 hw->mac.max_tx_queues = 2; 2643 hw->mac.max_rx_queues = 2; 2644 2645 /* lock to protect mailbox accesses */ 2646 spin_lock_init(&adapter->mbx_lock); 2647 2648 err = hw->mac.ops.reset_hw(hw); 2649 if (err) { 2650 dev_info(&pdev->dev, 2651 "PF still in reset state. Is the PF interface up?\n"); 2652 } else { 2653 err = hw->mac.ops.init_hw(hw); 2654 if (err) { 2655 pr_err("init_shared_code failed: %d\n", err); 2656 goto out; 2657 } 2658 ixgbevf_negotiate_api(adapter); 2659 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr); 2660 if (err) 2661 dev_info(&pdev->dev, "Error reading MAC address\n"); 2662 else if (is_zero_ether_addr(adapter->hw.mac.addr)) 2663 dev_info(&pdev->dev, 2664 "MAC address not assigned by administrator.\n"); 2665 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 2666 } 2667 2668 if (!is_valid_ether_addr(netdev->dev_addr)) { 2669 dev_info(&pdev->dev, "Assigning random MAC address\n"); 2670 eth_hw_addr_random(netdev); 2671 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len); 2672 } 2673 2674 /* Enable dynamic interrupt throttling rates */ 2675 adapter->rx_itr_setting = 1; 2676 adapter->tx_itr_setting = 1; 2677 2678 /* set default ring sizes */ 2679 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 2680 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 2681 2682 set_bit(__IXGBEVF_DOWN, &adapter->state); 2683 return 0; 2684 2685 out: 2686 return err; 2687 } 2688 2689 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 2690 { \ 2691 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 2692 if (current_counter < last_counter) \ 2693 counter += 0x100000000LL; \ 2694 last_counter = current_counter; \ 2695 counter &= 0xFFFFFFFF00000000LL; \ 2696 counter |= current_counter; \ 2697 } 2698 2699 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 2700 { \ 2701 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 2702 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 2703 u64 current_counter = (current_counter_msb << 32) | \ 2704 current_counter_lsb; \ 2705 if (current_counter < last_counter) \ 2706 counter += 0x1000000000LL; \ 2707 last_counter = current_counter; \ 2708 counter &= 0xFFFFFFF000000000LL; \ 2709 counter |= current_counter; \ 2710 } 2711 /** 2712 * ixgbevf_update_stats - Update the board statistics counters. 2713 * @adapter: board private structure 2714 **/ 2715 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 2716 { 2717 struct ixgbe_hw *hw = &adapter->hw; 2718 int i; 2719 2720 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2721 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2722 return; 2723 2724 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 2725 adapter->stats.vfgprc); 2726 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 2727 adapter->stats.vfgptc); 2728 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2729 adapter->stats.last_vfgorc, 2730 adapter->stats.vfgorc); 2731 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2732 adapter->stats.last_vfgotc, 2733 adapter->stats.vfgotc); 2734 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 2735 adapter->stats.vfmprc); 2736 2737 for (i = 0; i < adapter->num_rx_queues; i++) { 2738 adapter->hw_csum_rx_error += 2739 adapter->rx_ring[i]->hw_csum_rx_error; 2740 adapter->rx_ring[i]->hw_csum_rx_error = 0; 2741 } 2742 } 2743 2744 /** 2745 * ixgbevf_service_timer - Timer Call-back 2746 * @data: pointer to adapter cast into an unsigned long 2747 **/ 2748 static void ixgbevf_service_timer(unsigned long data) 2749 { 2750 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data; 2751 2752 /* Reset the timer */ 2753 mod_timer(&adapter->service_timer, (HZ * 2) + jiffies); 2754 2755 ixgbevf_service_event_schedule(adapter); 2756 } 2757 2758 static void ixgbevf_reset_subtask(struct ixgbevf_adapter *adapter) 2759 { 2760 if (!(adapter->flags & IXGBEVF_FLAG_RESET_REQUESTED)) 2761 return; 2762 2763 adapter->flags &= ~IXGBEVF_FLAG_RESET_REQUESTED; 2764 2765 /* If we're already down or resetting, just bail */ 2766 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2767 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2768 return; 2769 2770 adapter->tx_timeout_count++; 2771 2772 ixgbevf_reinit_locked(adapter); 2773 } 2774 2775 /** 2776 * ixgbevf_check_hang_subtask - check for hung queues and dropped interrupts 2777 * @adapter: pointer to the device adapter structure 2778 * 2779 * This function serves two purposes. First it strobes the interrupt lines 2780 * in order to make certain interrupts are occurring. Secondly it sets the 2781 * bits needed to check for TX hangs. As a result we should immediately 2782 * determine if a hang has occurred. 2783 **/ 2784 static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter) 2785 { 2786 struct ixgbe_hw *hw = &adapter->hw; 2787 u32 eics = 0; 2788 int i; 2789 2790 /* If we're down or resetting, just bail */ 2791 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2792 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2793 return; 2794 2795 /* Force detection of hung controller */ 2796 if (netif_carrier_ok(adapter->netdev)) { 2797 for (i = 0; i < adapter->num_tx_queues; i++) 2798 set_check_for_tx_hang(adapter->tx_ring[i]); 2799 } 2800 2801 /* get one bit for every active Tx/Rx interrupt vector */ 2802 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 2803 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 2804 2805 if (qv->rx.ring || qv->tx.ring) 2806 eics |= 1 << i; 2807 } 2808 2809 /* Cause software interrupt to ensure rings are cleaned */ 2810 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics); 2811 } 2812 2813 /** 2814 * ixgbevf_watchdog_update_link - update the link status 2815 * @adapter: pointer to the device adapter structure 2816 **/ 2817 static void ixgbevf_watchdog_update_link(struct ixgbevf_adapter *adapter) 2818 { 2819 struct ixgbe_hw *hw = &adapter->hw; 2820 u32 link_speed = adapter->link_speed; 2821 bool link_up = adapter->link_up; 2822 s32 err; 2823 2824 spin_lock_bh(&adapter->mbx_lock); 2825 2826 err = hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 2827 2828 spin_unlock_bh(&adapter->mbx_lock); 2829 2830 /* if check for link returns error we will need to reset */ 2831 if (err && time_after(jiffies, adapter->last_reset + (10 * HZ))) { 2832 adapter->flags |= IXGBEVF_FLAG_RESET_REQUESTED; 2833 link_up = false; 2834 } 2835 2836 adapter->link_up = link_up; 2837 adapter->link_speed = link_speed; 2838 } 2839 2840 /** 2841 * ixgbevf_watchdog_link_is_up - update netif_carrier status and 2842 * print link up message 2843 * @adapter: pointer to the device adapter structure 2844 **/ 2845 static void ixgbevf_watchdog_link_is_up(struct ixgbevf_adapter *adapter) 2846 { 2847 struct net_device *netdev = adapter->netdev; 2848 2849 /* only continue if link was previously down */ 2850 if (netif_carrier_ok(netdev)) 2851 return; 2852 2853 dev_info(&adapter->pdev->dev, "NIC Link is Up %s\n", 2854 (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 2855 "10 Gbps" : 2856 (adapter->link_speed == IXGBE_LINK_SPEED_1GB_FULL) ? 2857 "1 Gbps" : 2858 (adapter->link_speed == IXGBE_LINK_SPEED_100_FULL) ? 2859 "100 Mbps" : 2860 "unknown speed"); 2861 2862 netif_carrier_on(netdev); 2863 } 2864 2865 /** 2866 * ixgbevf_watchdog_link_is_down - update netif_carrier status and 2867 * print link down message 2868 * @adapter: pointer to the adapter structure 2869 **/ 2870 static void ixgbevf_watchdog_link_is_down(struct ixgbevf_adapter *adapter) 2871 { 2872 struct net_device *netdev = adapter->netdev; 2873 2874 adapter->link_speed = 0; 2875 2876 /* only continue if link was up previously */ 2877 if (!netif_carrier_ok(netdev)) 2878 return; 2879 2880 dev_info(&adapter->pdev->dev, "NIC Link is Down\n"); 2881 2882 netif_carrier_off(netdev); 2883 } 2884 2885 /** 2886 * ixgbevf_watchdog_subtask - worker thread to bring link up 2887 * @work: pointer to work_struct containing our data 2888 **/ 2889 static void ixgbevf_watchdog_subtask(struct ixgbevf_adapter *adapter) 2890 { 2891 /* if interface is down do nothing */ 2892 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2893 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2894 return; 2895 2896 ixgbevf_watchdog_update_link(adapter); 2897 2898 if (adapter->link_up) 2899 ixgbevf_watchdog_link_is_up(adapter); 2900 else 2901 ixgbevf_watchdog_link_is_down(adapter); 2902 2903 ixgbevf_update_stats(adapter); 2904 } 2905 2906 /** 2907 * ixgbevf_service_task - manages and runs subtasks 2908 * @work: pointer to work_struct containing our data 2909 **/ 2910 static void ixgbevf_service_task(struct work_struct *work) 2911 { 2912 struct ixgbevf_adapter *adapter = container_of(work, 2913 struct ixgbevf_adapter, 2914 service_task); 2915 struct ixgbe_hw *hw = &adapter->hw; 2916 2917 if (IXGBE_REMOVED(hw->hw_addr)) { 2918 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 2919 rtnl_lock(); 2920 ixgbevf_down(adapter); 2921 rtnl_unlock(); 2922 } 2923 return; 2924 } 2925 2926 ixgbevf_queue_reset_subtask(adapter); 2927 ixgbevf_reset_subtask(adapter); 2928 ixgbevf_watchdog_subtask(adapter); 2929 ixgbevf_check_hang_subtask(adapter); 2930 2931 ixgbevf_service_event_complete(adapter); 2932 } 2933 2934 /** 2935 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 2936 * @tx_ring: Tx descriptor ring for a specific queue 2937 * 2938 * Free all transmit software resources 2939 **/ 2940 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) 2941 { 2942 ixgbevf_clean_tx_ring(tx_ring); 2943 2944 vfree(tx_ring->tx_buffer_info); 2945 tx_ring->tx_buffer_info = NULL; 2946 2947 /* if not set, then don't free */ 2948 if (!tx_ring->desc) 2949 return; 2950 2951 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, 2952 tx_ring->dma); 2953 2954 tx_ring->desc = NULL; 2955 } 2956 2957 /** 2958 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 2959 * @adapter: board private structure 2960 * 2961 * Free all transmit software resources 2962 **/ 2963 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 2964 { 2965 int i; 2966 2967 for (i = 0; i < adapter->num_tx_queues; i++) 2968 if (adapter->tx_ring[i]->desc) 2969 ixgbevf_free_tx_resources(adapter->tx_ring[i]); 2970 } 2971 2972 /** 2973 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 2974 * @tx_ring: Tx descriptor ring (for a specific queue) to setup 2975 * 2976 * Return 0 on success, negative on failure 2977 **/ 2978 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) 2979 { 2980 int size; 2981 2982 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2983 tx_ring->tx_buffer_info = vzalloc(size); 2984 if (!tx_ring->tx_buffer_info) 2985 goto err; 2986 2987 /* round up to nearest 4K */ 2988 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 2989 tx_ring->size = ALIGN(tx_ring->size, 4096); 2990 2991 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, 2992 &tx_ring->dma, GFP_KERNEL); 2993 if (!tx_ring->desc) 2994 goto err; 2995 2996 return 0; 2997 2998 err: 2999 vfree(tx_ring->tx_buffer_info); 3000 tx_ring->tx_buffer_info = NULL; 3001 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit descriptor ring\n"); 3002 return -ENOMEM; 3003 } 3004 3005 /** 3006 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 3007 * @adapter: board private structure 3008 * 3009 * If this function returns with an error, then it's possible one or 3010 * more of the rings is populated (while the rest are not). It is the 3011 * callers duty to clean those orphaned rings. 3012 * 3013 * Return 0 on success, negative on failure 3014 **/ 3015 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 3016 { 3017 int i, err = 0; 3018 3019 for (i = 0; i < adapter->num_tx_queues; i++) { 3020 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]); 3021 if (!err) 3022 continue; 3023 hw_dbg(&adapter->hw, "Allocation for Tx Queue %u failed\n", i); 3024 break; 3025 } 3026 3027 return err; 3028 } 3029 3030 /** 3031 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 3032 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 3033 * 3034 * Returns 0 on success, negative on failure 3035 **/ 3036 int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring) 3037 { 3038 int size; 3039 3040 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 3041 rx_ring->rx_buffer_info = vzalloc(size); 3042 if (!rx_ring->rx_buffer_info) 3043 goto err; 3044 3045 /* Round up to nearest 4K */ 3046 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 3047 rx_ring->size = ALIGN(rx_ring->size, 4096); 3048 3049 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, 3050 &rx_ring->dma, GFP_KERNEL); 3051 3052 if (!rx_ring->desc) 3053 goto err; 3054 3055 return 0; 3056 err: 3057 vfree(rx_ring->rx_buffer_info); 3058 rx_ring->rx_buffer_info = NULL; 3059 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); 3060 return -ENOMEM; 3061 } 3062 3063 /** 3064 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 3065 * @adapter: board private structure 3066 * 3067 * If this function returns with an error, then it's possible one or 3068 * more of the rings is populated (while the rest are not). It is the 3069 * callers duty to clean those orphaned rings. 3070 * 3071 * Return 0 on success, negative on failure 3072 **/ 3073 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 3074 { 3075 int i, err = 0; 3076 3077 for (i = 0; i < adapter->num_rx_queues; i++) { 3078 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]); 3079 if (!err) 3080 continue; 3081 hw_dbg(&adapter->hw, "Allocation for Rx Queue %u failed\n", i); 3082 break; 3083 } 3084 return err; 3085 } 3086 3087 /** 3088 * ixgbevf_free_rx_resources - Free Rx Resources 3089 * @rx_ring: ring to clean the resources from 3090 * 3091 * Free all receive software resources 3092 **/ 3093 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) 3094 { 3095 ixgbevf_clean_rx_ring(rx_ring); 3096 3097 vfree(rx_ring->rx_buffer_info); 3098 rx_ring->rx_buffer_info = NULL; 3099 3100 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, 3101 rx_ring->dma); 3102 3103 rx_ring->desc = NULL; 3104 } 3105 3106 /** 3107 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 3108 * @adapter: board private structure 3109 * 3110 * Free all receive software resources 3111 **/ 3112 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 3113 { 3114 int i; 3115 3116 for (i = 0; i < adapter->num_rx_queues; i++) 3117 if (adapter->rx_ring[i]->desc) 3118 ixgbevf_free_rx_resources(adapter->rx_ring[i]); 3119 } 3120 3121 /** 3122 * ixgbevf_open - Called when a network interface is made active 3123 * @netdev: network interface device structure 3124 * 3125 * Returns 0 on success, negative value on failure 3126 * 3127 * The open entry point is called when a network interface is made 3128 * active by the system (IFF_UP). At this point all resources needed 3129 * for transmit and receive operations are allocated, the interrupt 3130 * handler is registered with the OS, the watchdog timer is started, 3131 * and the stack is notified that the interface is ready. 3132 **/ 3133 static int ixgbevf_open(struct net_device *netdev) 3134 { 3135 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3136 struct ixgbe_hw *hw = &adapter->hw; 3137 int err; 3138 3139 /* A previous failure to open the device because of a lack of 3140 * available MSIX vector resources may have reset the number 3141 * of msix vectors variable to zero. The only way to recover 3142 * is to unload/reload the driver and hope that the system has 3143 * been able to recover some MSIX vector resources. 3144 */ 3145 if (!adapter->num_msix_vectors) 3146 return -ENOMEM; 3147 3148 if (hw->adapter_stopped) { 3149 ixgbevf_reset(adapter); 3150 /* if adapter is still stopped then PF isn't up and 3151 * the VF can't start. 3152 */ 3153 if (hw->adapter_stopped) { 3154 err = IXGBE_ERR_MBX; 3155 pr_err("Unable to start - perhaps the PF Driver isn't up yet\n"); 3156 goto err_setup_reset; 3157 } 3158 } 3159 3160 /* disallow open during test */ 3161 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 3162 return -EBUSY; 3163 3164 netif_carrier_off(netdev); 3165 3166 /* allocate transmit descriptors */ 3167 err = ixgbevf_setup_all_tx_resources(adapter); 3168 if (err) 3169 goto err_setup_tx; 3170 3171 /* allocate receive descriptors */ 3172 err = ixgbevf_setup_all_rx_resources(adapter); 3173 if (err) 3174 goto err_setup_rx; 3175 3176 ixgbevf_configure(adapter); 3177 3178 /* Map the Tx/Rx rings to the vectors we were allotted. 3179 * if request_irq will be called in this function map_rings 3180 * must be called *before* up_complete 3181 */ 3182 ixgbevf_map_rings_to_vectors(adapter); 3183 3184 err = ixgbevf_request_irq(adapter); 3185 if (err) 3186 goto err_req_irq; 3187 3188 ixgbevf_up_complete(adapter); 3189 3190 return 0; 3191 3192 err_req_irq: 3193 ixgbevf_down(adapter); 3194 err_setup_rx: 3195 ixgbevf_free_all_rx_resources(adapter); 3196 err_setup_tx: 3197 ixgbevf_free_all_tx_resources(adapter); 3198 ixgbevf_reset(adapter); 3199 3200 err_setup_reset: 3201 3202 return err; 3203 } 3204 3205 /** 3206 * ixgbevf_close - Disables a network interface 3207 * @netdev: network interface device structure 3208 * 3209 * Returns 0, this is not allowed to fail 3210 * 3211 * The close entry point is called when an interface is de-activated 3212 * by the OS. The hardware is still under the drivers control, but 3213 * needs to be disabled. A global MAC reset is issued to stop the 3214 * hardware, and all transmit and receive resources are freed. 3215 **/ 3216 static int ixgbevf_close(struct net_device *netdev) 3217 { 3218 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3219 3220 ixgbevf_down(adapter); 3221 ixgbevf_free_irq(adapter); 3222 3223 ixgbevf_free_all_tx_resources(adapter); 3224 ixgbevf_free_all_rx_resources(adapter); 3225 3226 return 0; 3227 } 3228 3229 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter) 3230 { 3231 struct net_device *dev = adapter->netdev; 3232 3233 if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED)) 3234 return; 3235 3236 adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED; 3237 3238 /* if interface is down do nothing */ 3239 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3240 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3241 return; 3242 3243 /* Hardware has to reinitialize queues and interrupts to 3244 * match packet buffer alignment. Unfortunately, the 3245 * hardware is not flexible enough to do this dynamically. 3246 */ 3247 if (netif_running(dev)) 3248 ixgbevf_close(dev); 3249 3250 ixgbevf_clear_interrupt_scheme(adapter); 3251 ixgbevf_init_interrupt_scheme(adapter); 3252 3253 if (netif_running(dev)) 3254 ixgbevf_open(dev); 3255 } 3256 3257 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, 3258 u32 vlan_macip_lens, u32 type_tucmd, 3259 u32 mss_l4len_idx) 3260 { 3261 struct ixgbe_adv_tx_context_desc *context_desc; 3262 u16 i = tx_ring->next_to_use; 3263 3264 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i); 3265 3266 i++; 3267 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 3268 3269 /* set bits to identify this as an advanced context descriptor */ 3270 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3271 3272 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 3273 context_desc->seqnum_seed = 0; 3274 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 3275 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 3276 } 3277 3278 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, 3279 struct ixgbevf_tx_buffer *first, 3280 u8 *hdr_len) 3281 { 3282 struct sk_buff *skb = first->skb; 3283 u32 vlan_macip_lens, type_tucmd; 3284 u32 mss_l4len_idx, l4len; 3285 int err; 3286 3287 if (skb->ip_summed != CHECKSUM_PARTIAL) 3288 return 0; 3289 3290 if (!skb_is_gso(skb)) 3291 return 0; 3292 3293 err = skb_cow_head(skb, 0); 3294 if (err < 0) 3295 return err; 3296 3297 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 3298 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3299 3300 if (first->protocol == htons(ETH_P_IP)) { 3301 struct iphdr *iph = ip_hdr(skb); 3302 3303 iph->tot_len = 0; 3304 iph->check = 0; 3305 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 3306 iph->daddr, 0, 3307 IPPROTO_TCP, 3308 0); 3309 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3310 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3311 IXGBE_TX_FLAGS_CSUM | 3312 IXGBE_TX_FLAGS_IPV4; 3313 } else if (skb_is_gso_v6(skb)) { 3314 ipv6_hdr(skb)->payload_len = 0; 3315 tcp_hdr(skb)->check = 3316 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 3317 &ipv6_hdr(skb)->daddr, 3318 0, IPPROTO_TCP, 0); 3319 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3320 IXGBE_TX_FLAGS_CSUM; 3321 } 3322 3323 /* compute header lengths */ 3324 l4len = tcp_hdrlen(skb); 3325 *hdr_len += l4len; 3326 *hdr_len = skb_transport_offset(skb) + l4len; 3327 3328 /* update GSO size and bytecount with header size */ 3329 first->gso_segs = skb_shinfo(skb)->gso_segs; 3330 first->bytecount += (first->gso_segs - 1) * *hdr_len; 3331 3332 /* mss_l4len_id: use 1 as index for TSO */ 3333 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; 3334 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 3335 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT; 3336 3337 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 3338 vlan_macip_lens = skb_network_header_len(skb); 3339 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3340 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3341 3342 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3343 type_tucmd, mss_l4len_idx); 3344 3345 return 1; 3346 } 3347 3348 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, 3349 struct ixgbevf_tx_buffer *first) 3350 { 3351 struct sk_buff *skb = first->skb; 3352 u32 vlan_macip_lens = 0; 3353 u32 mss_l4len_idx = 0; 3354 u32 type_tucmd = 0; 3355 3356 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3357 u8 l4_hdr = 0; 3358 3359 switch (first->protocol) { 3360 case htons(ETH_P_IP): 3361 vlan_macip_lens |= skb_network_header_len(skb); 3362 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3363 l4_hdr = ip_hdr(skb)->protocol; 3364 break; 3365 case htons(ETH_P_IPV6): 3366 vlan_macip_lens |= skb_network_header_len(skb); 3367 l4_hdr = ipv6_hdr(skb)->nexthdr; 3368 break; 3369 default: 3370 if (unlikely(net_ratelimit())) { 3371 dev_warn(tx_ring->dev, 3372 "partial checksum but proto=%x!\n", 3373 first->protocol); 3374 } 3375 break; 3376 } 3377 3378 switch (l4_hdr) { 3379 case IPPROTO_TCP: 3380 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3381 mss_l4len_idx = tcp_hdrlen(skb) << 3382 IXGBE_ADVTXD_L4LEN_SHIFT; 3383 break; 3384 case IPPROTO_SCTP: 3385 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 3386 mss_l4len_idx = sizeof(struct sctphdr) << 3387 IXGBE_ADVTXD_L4LEN_SHIFT; 3388 break; 3389 case IPPROTO_UDP: 3390 mss_l4len_idx = sizeof(struct udphdr) << 3391 IXGBE_ADVTXD_L4LEN_SHIFT; 3392 break; 3393 default: 3394 if (unlikely(net_ratelimit())) { 3395 dev_warn(tx_ring->dev, 3396 "partial checksum but l4 proto=%x!\n", 3397 l4_hdr); 3398 } 3399 break; 3400 } 3401 3402 /* update TX checksum flag */ 3403 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 3404 } 3405 3406 /* vlan_macip_lens: MACLEN, VLAN tag */ 3407 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3408 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3409 3410 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3411 type_tucmd, mss_l4len_idx); 3412 } 3413 3414 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags) 3415 { 3416 /* set type for advanced descriptor with frame checksum insertion */ 3417 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | 3418 IXGBE_ADVTXD_DCMD_IFCS | 3419 IXGBE_ADVTXD_DCMD_DEXT); 3420 3421 /* set HW VLAN bit if VLAN is present */ 3422 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 3423 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); 3424 3425 /* set segmentation enable bits for TSO/FSO */ 3426 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3427 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); 3428 3429 return cmd_type; 3430 } 3431 3432 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 3433 u32 tx_flags, unsigned int paylen) 3434 { 3435 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT); 3436 3437 /* enable L4 checksum for TSO and TX checksum offload */ 3438 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 3439 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM); 3440 3441 /* enble IPv4 checksum for TSO */ 3442 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 3443 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM); 3444 3445 /* use index 1 context for TSO/FSO/FCOE */ 3446 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3447 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT); 3448 3449 /* Check Context must be set if Tx switch is enabled, which it 3450 * always is for case where virtual functions are running 3451 */ 3452 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC); 3453 3454 tx_desc->read.olinfo_status = olinfo_status; 3455 } 3456 3457 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, 3458 struct ixgbevf_tx_buffer *first, 3459 const u8 hdr_len) 3460 { 3461 dma_addr_t dma; 3462 struct sk_buff *skb = first->skb; 3463 struct ixgbevf_tx_buffer *tx_buffer; 3464 union ixgbe_adv_tx_desc *tx_desc; 3465 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 3466 unsigned int data_len = skb->data_len; 3467 unsigned int size = skb_headlen(skb); 3468 unsigned int paylen = skb->len - hdr_len; 3469 u32 tx_flags = first->tx_flags; 3470 __le32 cmd_type; 3471 u16 i = tx_ring->next_to_use; 3472 3473 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 3474 3475 ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen); 3476 cmd_type = ixgbevf_tx_cmd_type(tx_flags); 3477 3478 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 3479 if (dma_mapping_error(tx_ring->dev, dma)) 3480 goto dma_error; 3481 3482 /* record length, and DMA address */ 3483 dma_unmap_len_set(first, len, size); 3484 dma_unmap_addr_set(first, dma, dma); 3485 3486 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3487 3488 for (;;) { 3489 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 3490 tx_desc->read.cmd_type_len = 3491 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); 3492 3493 i++; 3494 tx_desc++; 3495 if (i == tx_ring->count) { 3496 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3497 i = 0; 3498 } 3499 3500 dma += IXGBE_MAX_DATA_PER_TXD; 3501 size -= IXGBE_MAX_DATA_PER_TXD; 3502 3503 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3504 tx_desc->read.olinfo_status = 0; 3505 } 3506 3507 if (likely(!data_len)) 3508 break; 3509 3510 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); 3511 3512 i++; 3513 tx_desc++; 3514 if (i == tx_ring->count) { 3515 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3516 i = 0; 3517 } 3518 3519 size = skb_frag_size(frag); 3520 data_len -= size; 3521 3522 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 3523 DMA_TO_DEVICE); 3524 if (dma_mapping_error(tx_ring->dev, dma)) 3525 goto dma_error; 3526 3527 tx_buffer = &tx_ring->tx_buffer_info[i]; 3528 dma_unmap_len_set(tx_buffer, len, size); 3529 dma_unmap_addr_set(tx_buffer, dma, dma); 3530 3531 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3532 tx_desc->read.olinfo_status = 0; 3533 3534 frag++; 3535 } 3536 3537 /* write last descriptor with RS and EOP bits */ 3538 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); 3539 tx_desc->read.cmd_type_len = cmd_type; 3540 3541 /* set the timestamp */ 3542 first->time_stamp = jiffies; 3543 3544 /* Force memory writes to complete before letting h/w know there 3545 * are new descriptors to fetch. (Only applicable for weak-ordered 3546 * memory model archs, such as IA-64). 3547 * 3548 * We also need this memory barrier (wmb) to make certain all of the 3549 * status bits have been updated before next_to_watch is written. 3550 */ 3551 wmb(); 3552 3553 /* set next_to_watch value indicating a packet is present */ 3554 first->next_to_watch = tx_desc; 3555 3556 i++; 3557 if (i == tx_ring->count) 3558 i = 0; 3559 3560 tx_ring->next_to_use = i; 3561 3562 /* notify HW of packet */ 3563 ixgbevf_write_tail(tx_ring, i); 3564 3565 return; 3566 dma_error: 3567 dev_err(tx_ring->dev, "TX DMA map failed\n"); 3568 3569 /* clear dma mappings for failed tx_buffer_info map */ 3570 for (;;) { 3571 tx_buffer = &tx_ring->tx_buffer_info[i]; 3572 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer); 3573 if (tx_buffer == first) 3574 break; 3575 if (i == 0) 3576 i = tx_ring->count; 3577 i--; 3578 } 3579 3580 tx_ring->next_to_use = i; 3581 } 3582 3583 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 3584 { 3585 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 3586 /* Herbert's original patch had: 3587 * smp_mb__after_netif_stop_queue(); 3588 * but since that doesn't exist yet, just open code it. 3589 */ 3590 smp_mb(); 3591 3592 /* We need to check again in a case another CPU has just 3593 * made room available. 3594 */ 3595 if (likely(ixgbevf_desc_unused(tx_ring) < size)) 3596 return -EBUSY; 3597 3598 /* A reprieve! - use start_queue because it doesn't call schedule */ 3599 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 3600 ++tx_ring->tx_stats.restart_queue; 3601 3602 return 0; 3603 } 3604 3605 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 3606 { 3607 if (likely(ixgbevf_desc_unused(tx_ring) >= size)) 3608 return 0; 3609 return __ixgbevf_maybe_stop_tx(tx_ring, size); 3610 } 3611 3612 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 3613 { 3614 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3615 struct ixgbevf_tx_buffer *first; 3616 struct ixgbevf_ring *tx_ring; 3617 int tso; 3618 u32 tx_flags = 0; 3619 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 3620 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3621 unsigned short f; 3622 #endif 3623 u8 hdr_len = 0; 3624 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL); 3625 3626 if (!dst_mac || is_link_local_ether_addr(dst_mac)) { 3627 dev_kfree_skb_any(skb); 3628 return NETDEV_TX_OK; 3629 } 3630 3631 tx_ring = adapter->tx_ring[skb->queue_mapping]; 3632 3633 /* need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 3634 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 3635 * + 2 desc gap to keep tail from touching head, 3636 * + 1 desc for context descriptor, 3637 * otherwise try next time 3638 */ 3639 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3640 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 3641 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 3642 #else 3643 count += skb_shinfo(skb)->nr_frags; 3644 #endif 3645 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { 3646 tx_ring->tx_stats.tx_busy++; 3647 return NETDEV_TX_BUSY; 3648 } 3649 3650 /* record the location of the first descriptor for this packet */ 3651 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 3652 first->skb = skb; 3653 first->bytecount = skb->len; 3654 first->gso_segs = 1; 3655 3656 if (skb_vlan_tag_present(skb)) { 3657 tx_flags |= skb_vlan_tag_get(skb); 3658 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 3659 tx_flags |= IXGBE_TX_FLAGS_VLAN; 3660 } 3661 3662 /* record initial flags and protocol */ 3663 first->tx_flags = tx_flags; 3664 first->protocol = vlan_get_protocol(skb); 3665 3666 tso = ixgbevf_tso(tx_ring, first, &hdr_len); 3667 if (tso < 0) 3668 goto out_drop; 3669 else if (!tso) 3670 ixgbevf_tx_csum(tx_ring, first); 3671 3672 ixgbevf_tx_map(tx_ring, first, hdr_len); 3673 3674 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); 3675 3676 return NETDEV_TX_OK; 3677 3678 out_drop: 3679 dev_kfree_skb_any(first->skb); 3680 first->skb = NULL; 3681 3682 return NETDEV_TX_OK; 3683 } 3684 3685 /** 3686 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 3687 * @netdev: network interface device structure 3688 * @p: pointer to an address structure 3689 * 3690 * Returns 0 on success, negative on failure 3691 **/ 3692 static int ixgbevf_set_mac(struct net_device *netdev, void *p) 3693 { 3694 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3695 struct ixgbe_hw *hw = &adapter->hw; 3696 struct sockaddr *addr = p; 3697 3698 if (!is_valid_ether_addr(addr->sa_data)) 3699 return -EADDRNOTAVAIL; 3700 3701 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 3702 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 3703 3704 spin_lock_bh(&adapter->mbx_lock); 3705 3706 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 3707 3708 spin_unlock_bh(&adapter->mbx_lock); 3709 3710 return 0; 3711 } 3712 3713 /** 3714 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 3715 * @netdev: network interface device structure 3716 * @new_mtu: new value for maximum frame size 3717 * 3718 * Returns 0 on success, negative on failure 3719 **/ 3720 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 3721 { 3722 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3723 struct ixgbe_hw *hw = &adapter->hw; 3724 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 3725 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; 3726 3727 switch (adapter->hw.api_version) { 3728 case ixgbe_mbox_api_11: 3729 case ixgbe_mbox_api_12: 3730 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3731 break; 3732 default: 3733 if (adapter->hw.mac.type != ixgbe_mac_82599_vf) 3734 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3735 break; 3736 } 3737 3738 /* MTU < 68 is an error and causes problems on some kernels */ 3739 if ((new_mtu < 68) || (max_frame > max_possible_frame)) 3740 return -EINVAL; 3741 3742 hw_dbg(hw, "changing MTU from %d to %d\n", 3743 netdev->mtu, new_mtu); 3744 /* must set new MTU before calling down or up */ 3745 netdev->mtu = new_mtu; 3746 3747 /* notify the PF of our intent to use this size of frame */ 3748 ixgbevf_rlpml_set_vf(hw, max_frame); 3749 3750 return 0; 3751 } 3752 3753 #ifdef CONFIG_NET_POLL_CONTROLLER 3754 /* Polling 'interrupt' - used by things like netconsole to send skbs 3755 * without having to re-enable interrupts. It's not called while 3756 * the interrupt routine is executing. 3757 */ 3758 static void ixgbevf_netpoll(struct net_device *netdev) 3759 { 3760 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3761 int i; 3762 3763 /* if interface is down do nothing */ 3764 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 3765 return; 3766 for (i = 0; i < adapter->num_rx_queues; i++) 3767 ixgbevf_msix_clean_rings(0, adapter->q_vector[i]); 3768 } 3769 #endif /* CONFIG_NET_POLL_CONTROLLER */ 3770 3771 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) 3772 { 3773 struct net_device *netdev = pci_get_drvdata(pdev); 3774 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3775 #ifdef CONFIG_PM 3776 int retval = 0; 3777 #endif 3778 3779 netif_device_detach(netdev); 3780 3781 if (netif_running(netdev)) { 3782 rtnl_lock(); 3783 ixgbevf_down(adapter); 3784 ixgbevf_free_irq(adapter); 3785 ixgbevf_free_all_tx_resources(adapter); 3786 ixgbevf_free_all_rx_resources(adapter); 3787 rtnl_unlock(); 3788 } 3789 3790 ixgbevf_clear_interrupt_scheme(adapter); 3791 3792 #ifdef CONFIG_PM 3793 retval = pci_save_state(pdev); 3794 if (retval) 3795 return retval; 3796 3797 #endif 3798 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 3799 pci_disable_device(pdev); 3800 3801 return 0; 3802 } 3803 3804 #ifdef CONFIG_PM 3805 static int ixgbevf_resume(struct pci_dev *pdev) 3806 { 3807 struct net_device *netdev = pci_get_drvdata(pdev); 3808 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3809 u32 err; 3810 3811 pci_restore_state(pdev); 3812 /* pci_restore_state clears dev->state_saved so call 3813 * pci_save_state to restore it. 3814 */ 3815 pci_save_state(pdev); 3816 3817 err = pci_enable_device_mem(pdev); 3818 if (err) { 3819 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 3820 return err; 3821 } 3822 smp_mb__before_atomic(); 3823 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 3824 pci_set_master(pdev); 3825 3826 ixgbevf_reset(adapter); 3827 3828 rtnl_lock(); 3829 err = ixgbevf_init_interrupt_scheme(adapter); 3830 rtnl_unlock(); 3831 if (err) { 3832 dev_err(&pdev->dev, "Cannot initialize interrupts\n"); 3833 return err; 3834 } 3835 3836 if (netif_running(netdev)) { 3837 err = ixgbevf_open(netdev); 3838 if (err) 3839 return err; 3840 } 3841 3842 netif_device_attach(netdev); 3843 3844 return err; 3845 } 3846 3847 #endif /* CONFIG_PM */ 3848 static void ixgbevf_shutdown(struct pci_dev *pdev) 3849 { 3850 ixgbevf_suspend(pdev, PMSG_SUSPEND); 3851 } 3852 3853 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, 3854 struct rtnl_link_stats64 *stats) 3855 { 3856 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3857 unsigned int start; 3858 u64 bytes, packets; 3859 const struct ixgbevf_ring *ring; 3860 int i; 3861 3862 ixgbevf_update_stats(adapter); 3863 3864 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3865 3866 for (i = 0; i < adapter->num_rx_queues; i++) { 3867 ring = adapter->rx_ring[i]; 3868 do { 3869 start = u64_stats_fetch_begin_irq(&ring->syncp); 3870 bytes = ring->stats.bytes; 3871 packets = ring->stats.packets; 3872 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 3873 stats->rx_bytes += bytes; 3874 stats->rx_packets += packets; 3875 } 3876 3877 for (i = 0; i < adapter->num_tx_queues; i++) { 3878 ring = adapter->tx_ring[i]; 3879 do { 3880 start = u64_stats_fetch_begin_irq(&ring->syncp); 3881 bytes = ring->stats.bytes; 3882 packets = ring->stats.packets; 3883 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 3884 stats->tx_bytes += bytes; 3885 stats->tx_packets += packets; 3886 } 3887 3888 return stats; 3889 } 3890 3891 static const struct net_device_ops ixgbevf_netdev_ops = { 3892 .ndo_open = ixgbevf_open, 3893 .ndo_stop = ixgbevf_close, 3894 .ndo_start_xmit = ixgbevf_xmit_frame, 3895 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 3896 .ndo_get_stats64 = ixgbevf_get_stats, 3897 .ndo_validate_addr = eth_validate_addr, 3898 .ndo_set_mac_address = ixgbevf_set_mac, 3899 .ndo_change_mtu = ixgbevf_change_mtu, 3900 .ndo_tx_timeout = ixgbevf_tx_timeout, 3901 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 3902 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 3903 #ifdef CONFIG_NET_RX_BUSY_POLL 3904 .ndo_busy_poll = ixgbevf_busy_poll_recv, 3905 #endif 3906 #ifdef CONFIG_NET_POLL_CONTROLLER 3907 .ndo_poll_controller = ixgbevf_netpoll, 3908 #endif 3909 .ndo_features_check = passthru_features_check, 3910 }; 3911 3912 static void ixgbevf_assign_netdev_ops(struct net_device *dev) 3913 { 3914 dev->netdev_ops = &ixgbevf_netdev_ops; 3915 ixgbevf_set_ethtool_ops(dev); 3916 dev->watchdog_timeo = 5 * HZ; 3917 } 3918 3919 /** 3920 * ixgbevf_probe - Device Initialization Routine 3921 * @pdev: PCI device information struct 3922 * @ent: entry in ixgbevf_pci_tbl 3923 * 3924 * Returns 0 on success, negative on failure 3925 * 3926 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 3927 * The OS initialization, configuring of the adapter private structure, 3928 * and a hardware reset occur. 3929 **/ 3930 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3931 { 3932 struct net_device *netdev; 3933 struct ixgbevf_adapter *adapter = NULL; 3934 struct ixgbe_hw *hw = NULL; 3935 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 3936 int err, pci_using_dac; 3937 bool disable_dev = false; 3938 3939 err = pci_enable_device(pdev); 3940 if (err) 3941 return err; 3942 3943 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 3944 pci_using_dac = 1; 3945 } else { 3946 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 3947 if (err) { 3948 dev_err(&pdev->dev, "No usable DMA configuration, aborting\n"); 3949 goto err_dma; 3950 } 3951 pci_using_dac = 0; 3952 } 3953 3954 err = pci_request_regions(pdev, ixgbevf_driver_name); 3955 if (err) { 3956 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 3957 goto err_pci_reg; 3958 } 3959 3960 pci_set_master(pdev); 3961 3962 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 3963 MAX_TX_QUEUES); 3964 if (!netdev) { 3965 err = -ENOMEM; 3966 goto err_alloc_etherdev; 3967 } 3968 3969 SET_NETDEV_DEV(netdev, &pdev->dev); 3970 3971 adapter = netdev_priv(netdev); 3972 3973 adapter->netdev = netdev; 3974 adapter->pdev = pdev; 3975 hw = &adapter->hw; 3976 hw->back = adapter; 3977 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 3978 3979 /* call save state here in standalone driver because it relies on 3980 * adapter struct to exist, and needs to call netdev_priv 3981 */ 3982 pci_save_state(pdev); 3983 3984 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3985 pci_resource_len(pdev, 0)); 3986 adapter->io_addr = hw->hw_addr; 3987 if (!hw->hw_addr) { 3988 err = -EIO; 3989 goto err_ioremap; 3990 } 3991 3992 ixgbevf_assign_netdev_ops(netdev); 3993 3994 /* Setup HW API */ 3995 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 3996 hw->mac.type = ii->mac; 3997 3998 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 3999 sizeof(struct ixgbe_mbx_operations)); 4000 4001 /* setup the private structure */ 4002 err = ixgbevf_sw_init(adapter); 4003 if (err) 4004 goto err_sw_init; 4005 4006 /* The HW MAC address was set and/or determined in sw_init */ 4007 if (!is_valid_ether_addr(netdev->dev_addr)) { 4008 pr_err("invalid MAC address\n"); 4009 err = -EIO; 4010 goto err_sw_init; 4011 } 4012 4013 netdev->hw_features = NETIF_F_SG | 4014 NETIF_F_IP_CSUM | 4015 NETIF_F_IPV6_CSUM | 4016 NETIF_F_TSO | 4017 NETIF_F_TSO6 | 4018 NETIF_F_RXCSUM; 4019 4020 netdev->features = netdev->hw_features | 4021 NETIF_F_HW_VLAN_CTAG_TX | 4022 NETIF_F_HW_VLAN_CTAG_RX | 4023 NETIF_F_HW_VLAN_CTAG_FILTER; 4024 4025 netdev->vlan_features |= NETIF_F_TSO | 4026 NETIF_F_TSO6 | 4027 NETIF_F_IP_CSUM | 4028 NETIF_F_IPV6_CSUM | 4029 NETIF_F_SG; 4030 4031 if (pci_using_dac) 4032 netdev->features |= NETIF_F_HIGHDMA; 4033 4034 netdev->priv_flags |= IFF_UNICAST_FLT; 4035 4036 if (IXGBE_REMOVED(hw->hw_addr)) { 4037 err = -EIO; 4038 goto err_sw_init; 4039 } 4040 4041 setup_timer(&adapter->service_timer, &ixgbevf_service_timer, 4042 (unsigned long)adapter); 4043 4044 INIT_WORK(&adapter->service_task, ixgbevf_service_task); 4045 set_bit(__IXGBEVF_SERVICE_INITED, &adapter->state); 4046 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 4047 4048 err = ixgbevf_init_interrupt_scheme(adapter); 4049 if (err) 4050 goto err_sw_init; 4051 4052 strcpy(netdev->name, "eth%d"); 4053 4054 err = register_netdev(netdev); 4055 if (err) 4056 goto err_register; 4057 4058 pci_set_drvdata(pdev, netdev); 4059 netif_carrier_off(netdev); 4060 4061 ixgbevf_init_last_counter_stats(adapter); 4062 4063 /* print the VF info */ 4064 dev_info(&pdev->dev, "%pM\n", netdev->dev_addr); 4065 dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type); 4066 4067 switch (hw->mac.type) { 4068 case ixgbe_mac_X550_vf: 4069 dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n"); 4070 break; 4071 case ixgbe_mac_X540_vf: 4072 dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n"); 4073 break; 4074 case ixgbe_mac_82599_vf: 4075 default: 4076 dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n"); 4077 break; 4078 } 4079 4080 return 0; 4081 4082 err_register: 4083 ixgbevf_clear_interrupt_scheme(adapter); 4084 err_sw_init: 4085 ixgbevf_reset_interrupt_capability(adapter); 4086 iounmap(adapter->io_addr); 4087 err_ioremap: 4088 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4089 free_netdev(netdev); 4090 err_alloc_etherdev: 4091 pci_release_regions(pdev); 4092 err_pci_reg: 4093 err_dma: 4094 if (!adapter || disable_dev) 4095 pci_disable_device(pdev); 4096 return err; 4097 } 4098 4099 /** 4100 * ixgbevf_remove - Device Removal Routine 4101 * @pdev: PCI device information struct 4102 * 4103 * ixgbevf_remove is called by the PCI subsystem to alert the driver 4104 * that it should release a PCI device. The could be caused by a 4105 * Hot-Plug event, or because the driver is going to be removed from 4106 * memory. 4107 **/ 4108 static void ixgbevf_remove(struct pci_dev *pdev) 4109 { 4110 struct net_device *netdev = pci_get_drvdata(pdev); 4111 struct ixgbevf_adapter *adapter; 4112 bool disable_dev; 4113 4114 if (!netdev) 4115 return; 4116 4117 adapter = netdev_priv(netdev); 4118 4119 set_bit(__IXGBEVF_REMOVING, &adapter->state); 4120 cancel_work_sync(&adapter->service_task); 4121 4122 if (netdev->reg_state == NETREG_REGISTERED) 4123 unregister_netdev(netdev); 4124 4125 ixgbevf_clear_interrupt_scheme(adapter); 4126 ixgbevf_reset_interrupt_capability(adapter); 4127 4128 iounmap(adapter->io_addr); 4129 pci_release_regions(pdev); 4130 4131 hw_dbg(&adapter->hw, "Remove complete\n"); 4132 4133 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4134 free_netdev(netdev); 4135 4136 if (disable_dev) 4137 pci_disable_device(pdev); 4138 } 4139 4140 /** 4141 * ixgbevf_io_error_detected - called when PCI error is detected 4142 * @pdev: Pointer to PCI device 4143 * @state: The current pci connection state 4144 * 4145 * This function is called after a PCI bus error affecting 4146 * this device has been detected. 4147 **/ 4148 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, 4149 pci_channel_state_t state) 4150 { 4151 struct net_device *netdev = pci_get_drvdata(pdev); 4152 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4153 4154 if (!test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 4155 return PCI_ERS_RESULT_DISCONNECT; 4156 4157 rtnl_lock(); 4158 netif_device_detach(netdev); 4159 4160 if (state == pci_channel_io_perm_failure) { 4161 rtnl_unlock(); 4162 return PCI_ERS_RESULT_DISCONNECT; 4163 } 4164 4165 if (netif_running(netdev)) 4166 ixgbevf_down(adapter); 4167 4168 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 4169 pci_disable_device(pdev); 4170 rtnl_unlock(); 4171 4172 /* Request a slot slot reset. */ 4173 return PCI_ERS_RESULT_NEED_RESET; 4174 } 4175 4176 /** 4177 * ixgbevf_io_slot_reset - called after the pci bus has been reset. 4178 * @pdev: Pointer to PCI device 4179 * 4180 * Restart the card from scratch, as if from a cold-boot. Implementation 4181 * resembles the first-half of the ixgbevf_resume routine. 4182 **/ 4183 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) 4184 { 4185 struct net_device *netdev = pci_get_drvdata(pdev); 4186 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4187 4188 if (pci_enable_device_mem(pdev)) { 4189 dev_err(&pdev->dev, 4190 "Cannot re-enable PCI device after reset.\n"); 4191 return PCI_ERS_RESULT_DISCONNECT; 4192 } 4193 4194 smp_mb__before_atomic(); 4195 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 4196 pci_set_master(pdev); 4197 4198 ixgbevf_reset(adapter); 4199 4200 return PCI_ERS_RESULT_RECOVERED; 4201 } 4202 4203 /** 4204 * ixgbevf_io_resume - called when traffic can start flowing again. 4205 * @pdev: Pointer to PCI device 4206 * 4207 * This callback is called when the error recovery driver tells us that 4208 * its OK to resume normal operation. Implementation resembles the 4209 * second-half of the ixgbevf_resume routine. 4210 **/ 4211 static void ixgbevf_io_resume(struct pci_dev *pdev) 4212 { 4213 struct net_device *netdev = pci_get_drvdata(pdev); 4214 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4215 4216 if (netif_running(netdev)) 4217 ixgbevf_up(adapter); 4218 4219 netif_device_attach(netdev); 4220 } 4221 4222 /* PCI Error Recovery (ERS) */ 4223 static const struct pci_error_handlers ixgbevf_err_handler = { 4224 .error_detected = ixgbevf_io_error_detected, 4225 .slot_reset = ixgbevf_io_slot_reset, 4226 .resume = ixgbevf_io_resume, 4227 }; 4228 4229 static struct pci_driver ixgbevf_driver = { 4230 .name = ixgbevf_driver_name, 4231 .id_table = ixgbevf_pci_tbl, 4232 .probe = ixgbevf_probe, 4233 .remove = ixgbevf_remove, 4234 #ifdef CONFIG_PM 4235 /* Power Management Hooks */ 4236 .suspend = ixgbevf_suspend, 4237 .resume = ixgbevf_resume, 4238 #endif 4239 .shutdown = ixgbevf_shutdown, 4240 .err_handler = &ixgbevf_err_handler 4241 }; 4242 4243 /** 4244 * ixgbevf_init_module - Driver Registration Routine 4245 * 4246 * ixgbevf_init_module is the first routine called when the driver is 4247 * loaded. All it does is register with the PCI subsystem. 4248 **/ 4249 static int __init ixgbevf_init_module(void) 4250 { 4251 int ret; 4252 4253 pr_info("%s - version %s\n", ixgbevf_driver_string, 4254 ixgbevf_driver_version); 4255 4256 pr_info("%s\n", ixgbevf_copyright); 4257 4258 ret = pci_register_driver(&ixgbevf_driver); 4259 return ret; 4260 } 4261 4262 module_init(ixgbevf_init_module); 4263 4264 /** 4265 * ixgbevf_exit_module - Driver Exit Cleanup Routine 4266 * 4267 * ixgbevf_exit_module is called just before the driver is removed 4268 * from memory. 4269 **/ 4270 static void __exit ixgbevf_exit_module(void) 4271 { 4272 pci_unregister_driver(&ixgbevf_driver); 4273 } 4274 4275 #ifdef DEBUG 4276 /** 4277 * ixgbevf_get_hw_dev_name - return device name string 4278 * used by hardware layer to print debugging information 4279 **/ 4280 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 4281 { 4282 struct ixgbevf_adapter *adapter = hw->back; 4283 4284 return adapter->netdev->name; 4285 } 4286 4287 #endif 4288 module_exit(ixgbevf_exit_module); 4289 4290 /* ixgbevf_main.c */ 4291