1 /******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2014 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 26 *******************************************************************************/ 27 28 29 /****************************************************************************** 30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 31 ******************************************************************************/ 32 33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35 #include <linux/types.h> 36 #include <linux/bitops.h> 37 #include <linux/module.h> 38 #include <linux/pci.h> 39 #include <linux/netdevice.h> 40 #include <linux/vmalloc.h> 41 #include <linux/string.h> 42 #include <linux/in.h> 43 #include <linux/ip.h> 44 #include <linux/tcp.h> 45 #include <linux/sctp.h> 46 #include <linux/ipv6.h> 47 #include <linux/slab.h> 48 #include <net/checksum.h> 49 #include <net/ip6_checksum.h> 50 #include <linux/ethtool.h> 51 #include <linux/if.h> 52 #include <linux/if_vlan.h> 53 #include <linux/prefetch.h> 54 55 #include "ixgbevf.h" 56 57 const char ixgbevf_driver_name[] = "ixgbevf"; 58 static const char ixgbevf_driver_string[] = 59 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 60 61 #define DRV_VERSION "2.12.1-k" 62 const char ixgbevf_driver_version[] = DRV_VERSION; 63 static char ixgbevf_copyright[] = 64 "Copyright (c) 2009 - 2012 Intel Corporation."; 65 66 static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 67 [board_82599_vf] = &ixgbevf_82599_vf_info, 68 [board_X540_vf] = &ixgbevf_X540_vf_info, 69 [board_X550_vf] = &ixgbevf_X550_vf_info, 70 [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info, 71 }; 72 73 /* ixgbevf_pci_tbl - PCI Device ID Table 74 * 75 * Wildcard entries (PCI_ANY_ID) should come last 76 * Last entry must be all 0s 77 * 78 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 79 * Class, Class Mask, private data (not used) } 80 */ 81 static const struct pci_device_id ixgbevf_pci_tbl[] = { 82 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf }, 83 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf }, 84 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf }, 85 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf }, 86 /* required last entry */ 87 {0, } 88 }; 89 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 90 91 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 92 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); 93 MODULE_LICENSE("GPL"); 94 MODULE_VERSION(DRV_VERSION); 95 96 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 97 static int debug = -1; 98 module_param(debug, int, 0); 99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 100 101 /* forward decls */ 102 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); 103 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); 104 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); 105 106 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) 107 { 108 struct ixgbevf_adapter *adapter = hw->back; 109 110 if (!hw->hw_addr) 111 return; 112 hw->hw_addr = NULL; 113 dev_err(&adapter->pdev->dev, "Adapter removed\n"); 114 if (test_bit(__IXGBEVF_WORK_INIT, &adapter->state)) 115 schedule_work(&adapter->watchdog_task); 116 } 117 118 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg) 119 { 120 u32 value; 121 122 /* The following check not only optimizes a bit by not 123 * performing a read on the status register when the 124 * register just read was a status register read that 125 * returned IXGBE_FAILED_READ_REG. It also blocks any 126 * potential recursion. 127 */ 128 if (reg == IXGBE_VFSTATUS) { 129 ixgbevf_remove_adapter(hw); 130 return; 131 } 132 value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS); 133 if (value == IXGBE_FAILED_READ_REG) 134 ixgbevf_remove_adapter(hw); 135 } 136 137 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg) 138 { 139 u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr); 140 u32 value; 141 142 if (IXGBE_REMOVED(reg_addr)) 143 return IXGBE_FAILED_READ_REG; 144 value = readl(reg_addr + reg); 145 if (unlikely(value == IXGBE_FAILED_READ_REG)) 146 ixgbevf_check_remove(hw, reg); 147 return value; 148 } 149 150 /** 151 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 152 * @adapter: pointer to adapter struct 153 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 154 * @queue: queue to map the corresponding interrupt to 155 * @msix_vector: the vector to map to the corresponding queue 156 */ 157 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 158 u8 queue, u8 msix_vector) 159 { 160 u32 ivar, index; 161 struct ixgbe_hw *hw = &adapter->hw; 162 if (direction == -1) { 163 /* other causes */ 164 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 165 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 166 ivar &= ~0xFF; 167 ivar |= msix_vector; 168 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 169 } else { 170 /* tx or rx causes */ 171 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 172 index = ((16 * (queue & 1)) + (8 * direction)); 173 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 174 ivar &= ~(0xFF << index); 175 ivar |= (msix_vector << index); 176 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 177 } 178 } 179 180 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring, 181 struct ixgbevf_tx_buffer *tx_buffer) 182 { 183 if (tx_buffer->skb) { 184 dev_kfree_skb_any(tx_buffer->skb); 185 if (dma_unmap_len(tx_buffer, len)) 186 dma_unmap_single(tx_ring->dev, 187 dma_unmap_addr(tx_buffer, dma), 188 dma_unmap_len(tx_buffer, len), 189 DMA_TO_DEVICE); 190 } else if (dma_unmap_len(tx_buffer, len)) { 191 dma_unmap_page(tx_ring->dev, 192 dma_unmap_addr(tx_buffer, dma), 193 dma_unmap_len(tx_buffer, len), 194 DMA_TO_DEVICE); 195 } 196 tx_buffer->next_to_watch = NULL; 197 tx_buffer->skb = NULL; 198 dma_unmap_len_set(tx_buffer, len, 0); 199 /* tx_buffer must be completely set up in the transmit path */ 200 } 201 202 #define IXGBE_MAX_TXD_PWR 14 203 #define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR) 204 205 /* Tx Descriptors needed, worst case */ 206 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD) 207 #define DESC_NEEDED (MAX_SKB_FRAGS + 4) 208 209 static void ixgbevf_tx_timeout(struct net_device *netdev); 210 211 /** 212 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 213 * @q_vector: board private structure 214 * @tx_ring: tx ring to clean 215 **/ 216 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, 217 struct ixgbevf_ring *tx_ring) 218 { 219 struct ixgbevf_adapter *adapter = q_vector->adapter; 220 struct ixgbevf_tx_buffer *tx_buffer; 221 union ixgbe_adv_tx_desc *tx_desc; 222 unsigned int total_bytes = 0, total_packets = 0; 223 unsigned int budget = tx_ring->count / 2; 224 unsigned int i = tx_ring->next_to_clean; 225 226 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 227 return true; 228 229 tx_buffer = &tx_ring->tx_buffer_info[i]; 230 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 231 i -= tx_ring->count; 232 233 do { 234 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 235 236 /* if next_to_watch is not set then there is no work pending */ 237 if (!eop_desc) 238 break; 239 240 /* prevent any other reads prior to eop_desc */ 241 read_barrier_depends(); 242 243 /* if DD is not set pending work has not been completed */ 244 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 245 break; 246 247 /* clear next_to_watch to prevent false hangs */ 248 tx_buffer->next_to_watch = NULL; 249 250 /* update the statistics for this packet */ 251 total_bytes += tx_buffer->bytecount; 252 total_packets += tx_buffer->gso_segs; 253 254 /* free the skb */ 255 dev_kfree_skb_any(tx_buffer->skb); 256 257 /* unmap skb header data */ 258 dma_unmap_single(tx_ring->dev, 259 dma_unmap_addr(tx_buffer, dma), 260 dma_unmap_len(tx_buffer, len), 261 DMA_TO_DEVICE); 262 263 /* clear tx_buffer data */ 264 tx_buffer->skb = NULL; 265 dma_unmap_len_set(tx_buffer, len, 0); 266 267 /* unmap remaining buffers */ 268 while (tx_desc != eop_desc) { 269 tx_buffer++; 270 tx_desc++; 271 i++; 272 if (unlikely(!i)) { 273 i -= tx_ring->count; 274 tx_buffer = tx_ring->tx_buffer_info; 275 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 276 } 277 278 /* unmap any remaining paged data */ 279 if (dma_unmap_len(tx_buffer, len)) { 280 dma_unmap_page(tx_ring->dev, 281 dma_unmap_addr(tx_buffer, dma), 282 dma_unmap_len(tx_buffer, len), 283 DMA_TO_DEVICE); 284 dma_unmap_len_set(tx_buffer, len, 0); 285 } 286 } 287 288 /* move us one more past the eop_desc for start of next pkt */ 289 tx_buffer++; 290 tx_desc++; 291 i++; 292 if (unlikely(!i)) { 293 i -= tx_ring->count; 294 tx_buffer = tx_ring->tx_buffer_info; 295 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 296 } 297 298 /* issue prefetch for next Tx descriptor */ 299 prefetch(tx_desc); 300 301 /* update budget accounting */ 302 budget--; 303 } while (likely(budget)); 304 305 i += tx_ring->count; 306 tx_ring->next_to_clean = i; 307 u64_stats_update_begin(&tx_ring->syncp); 308 tx_ring->stats.bytes += total_bytes; 309 tx_ring->stats.packets += total_packets; 310 u64_stats_update_end(&tx_ring->syncp); 311 q_vector->tx.total_bytes += total_bytes; 312 q_vector->tx.total_packets += total_packets; 313 314 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 315 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 316 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 317 /* Make sure that anybody stopping the queue after this 318 * sees the new next_to_clean. 319 */ 320 smp_mb(); 321 322 if (__netif_subqueue_stopped(tx_ring->netdev, 323 tx_ring->queue_index) && 324 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 325 netif_wake_subqueue(tx_ring->netdev, 326 tx_ring->queue_index); 327 ++tx_ring->tx_stats.restart_queue; 328 } 329 } 330 331 return !!budget; 332 } 333 334 /** 335 * ixgbevf_rx_skb - Helper function to determine proper Rx method 336 * @q_vector: structure containing interrupt and ring information 337 * @skb: packet to send up 338 **/ 339 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, 340 struct sk_buff *skb) 341 { 342 #ifdef CONFIG_NET_RX_BUSY_POLL 343 skb_mark_napi_id(skb, &q_vector->napi); 344 345 if (ixgbevf_qv_busy_polling(q_vector)) { 346 netif_receive_skb(skb); 347 /* exit early if we busy polled */ 348 return; 349 } 350 #endif /* CONFIG_NET_RX_BUSY_POLL */ 351 352 napi_gro_receive(&q_vector->napi, skb); 353 } 354 355 /* ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 356 * @ring: structure containig ring specific data 357 * @rx_desc: current Rx descriptor being processed 358 * @skb: skb currently being received and modified 359 */ 360 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, 361 union ixgbe_adv_rx_desc *rx_desc, 362 struct sk_buff *skb) 363 { 364 skb_checksum_none_assert(skb); 365 366 /* Rx csum disabled */ 367 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 368 return; 369 370 /* if IP and error */ 371 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 372 ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 373 ring->rx_stats.csum_err++; 374 return; 375 } 376 377 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 378 return; 379 380 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 381 ring->rx_stats.csum_err++; 382 return; 383 } 384 385 /* It must be a TCP or UDP packet with a valid checksum */ 386 skb->ip_summed = CHECKSUM_UNNECESSARY; 387 } 388 389 /* ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor 390 * @rx_ring: rx descriptor ring packet is being transacted on 391 * @rx_desc: pointer to the EOP Rx descriptor 392 * @skb: pointer to current skb being populated 393 * 394 * This function checks the ring, descriptor, and packet information in 395 * order to populate the checksum, VLAN, protocol, and other fields within 396 * the skb. 397 */ 398 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, 399 union ixgbe_adv_rx_desc *rx_desc, 400 struct sk_buff *skb) 401 { 402 ixgbevf_rx_checksum(rx_ring, rx_desc, skb); 403 404 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 405 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 406 unsigned long *active_vlans = netdev_priv(rx_ring->netdev); 407 408 if (test_bit(vid & VLAN_VID_MASK, active_vlans)) 409 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 410 } 411 412 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 413 } 414 415 /** 416 * ixgbevf_is_non_eop - process handling of non-EOP buffers 417 * @rx_ring: Rx ring being processed 418 * @rx_desc: Rx descriptor for current buffer 419 * @skb: current socket buffer containing buffer in progress 420 * 421 * This function updates next to clean. If the buffer is an EOP buffer 422 * this function exits returning false, otherwise it will place the 423 * sk_buff in the next buffer to be chained and return true indicating 424 * that this is in fact a non-EOP buffer. 425 **/ 426 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, 427 union ixgbe_adv_rx_desc *rx_desc) 428 { 429 u32 ntc = rx_ring->next_to_clean + 1; 430 431 /* fetch, update, and store next to clean */ 432 ntc = (ntc < rx_ring->count) ? ntc : 0; 433 rx_ring->next_to_clean = ntc; 434 435 prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); 436 437 if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 438 return false; 439 440 return true; 441 } 442 443 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, 444 struct ixgbevf_rx_buffer *bi) 445 { 446 struct page *page = bi->page; 447 dma_addr_t dma = bi->dma; 448 449 /* since we are recycling buffers we should seldom need to alloc */ 450 if (likely(page)) 451 return true; 452 453 /* alloc new page for storage */ 454 page = dev_alloc_page(); 455 if (unlikely(!page)) { 456 rx_ring->rx_stats.alloc_rx_page_failed++; 457 return false; 458 } 459 460 /* map page for use */ 461 dma = dma_map_page(rx_ring->dev, page, 0, 462 PAGE_SIZE, DMA_FROM_DEVICE); 463 464 /* if mapping failed free memory back to system since 465 * there isn't much point in holding memory we can't use 466 */ 467 if (dma_mapping_error(rx_ring->dev, dma)) { 468 __free_page(page); 469 470 rx_ring->rx_stats.alloc_rx_buff_failed++; 471 return false; 472 } 473 474 bi->dma = dma; 475 bi->page = page; 476 bi->page_offset = 0; 477 478 return true; 479 } 480 481 /** 482 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 483 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on 484 * @cleaned_count: number of buffers to replace 485 **/ 486 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, 487 u16 cleaned_count) 488 { 489 union ixgbe_adv_rx_desc *rx_desc; 490 struct ixgbevf_rx_buffer *bi; 491 unsigned int i = rx_ring->next_to_use; 492 493 /* nothing to do or no valid netdev defined */ 494 if (!cleaned_count || !rx_ring->netdev) 495 return; 496 497 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 498 bi = &rx_ring->rx_buffer_info[i]; 499 i -= rx_ring->count; 500 501 do { 502 if (!ixgbevf_alloc_mapped_page(rx_ring, bi)) 503 break; 504 505 /* Refresh the desc even if pkt_addr didn't change 506 * because each write-back erases this info. 507 */ 508 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 509 510 rx_desc++; 511 bi++; 512 i++; 513 if (unlikely(!i)) { 514 rx_desc = IXGBEVF_RX_DESC(rx_ring, 0); 515 bi = rx_ring->rx_buffer_info; 516 i -= rx_ring->count; 517 } 518 519 /* clear the hdr_addr for the next_to_use descriptor */ 520 rx_desc->read.hdr_addr = 0; 521 522 cleaned_count--; 523 } while (cleaned_count); 524 525 i += rx_ring->count; 526 527 if (rx_ring->next_to_use != i) { 528 /* record the next descriptor to use */ 529 rx_ring->next_to_use = i; 530 531 /* update next to alloc since we have filled the ring */ 532 rx_ring->next_to_alloc = i; 533 534 /* Force memory writes to complete before letting h/w 535 * know there are new descriptors to fetch. (Only 536 * applicable for weak-ordered memory model archs, 537 * such as IA-64). 538 */ 539 wmb(); 540 ixgbevf_write_tail(rx_ring, i); 541 } 542 } 543 544 /* ixgbevf_pull_tail - ixgbevf specific version of skb_pull_tail 545 * @rx_ring: rx descriptor ring packet is being transacted on 546 * @skb: pointer to current skb being adjusted 547 * 548 * This function is an ixgbevf specific version of __pskb_pull_tail. The 549 * main difference between this version and the original function is that 550 * this function can make several assumptions about the state of things 551 * that allow for significant optimizations versus the standard function. 552 * As a result we can do things like drop a frag and maintain an accurate 553 * truesize for the skb. 554 */ 555 static void ixgbevf_pull_tail(struct ixgbevf_ring *rx_ring, 556 struct sk_buff *skb) 557 { 558 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 559 unsigned char *va; 560 unsigned int pull_len; 561 562 /* it is valid to use page_address instead of kmap since we are 563 * working with pages allocated out of the lomem pool per 564 * alloc_page(GFP_ATOMIC) 565 */ 566 va = skb_frag_address(frag); 567 568 /* we need the header to contain the greater of either ETH_HLEN or 569 * 60 bytes if the skb->len is less than 60 for skb_pad. 570 */ 571 pull_len = eth_get_headlen(va, IXGBEVF_RX_HDR_SIZE); 572 573 /* align pull length to size of long to optimize memcpy performance */ 574 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); 575 576 /* update all of the pointers */ 577 skb_frag_size_sub(frag, pull_len); 578 frag->page_offset += pull_len; 579 skb->data_len -= pull_len; 580 skb->tail += pull_len; 581 } 582 583 /* ixgbevf_cleanup_headers - Correct corrupted or empty headers 584 * @rx_ring: rx descriptor ring packet is being transacted on 585 * @rx_desc: pointer to the EOP Rx descriptor 586 * @skb: pointer to current skb being fixed 587 * 588 * Check for corrupted packet headers caused by senders on the local L2 589 * embedded NIC switch not setting up their Tx Descriptors right. These 590 * should be very rare. 591 * 592 * Also address the case where we are pulling data in on pages only 593 * and as such no data is present in the skb header. 594 * 595 * In addition if skb is not at least 60 bytes we need to pad it so that 596 * it is large enough to qualify as a valid Ethernet frame. 597 * 598 * Returns true if an error was encountered and skb was freed. 599 */ 600 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, 601 union ixgbe_adv_rx_desc *rx_desc, 602 struct sk_buff *skb) 603 { 604 /* verify that the packet does not have any known errors */ 605 if (unlikely(ixgbevf_test_staterr(rx_desc, 606 IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { 607 struct net_device *netdev = rx_ring->netdev; 608 609 if (!(netdev->features & NETIF_F_RXALL)) { 610 dev_kfree_skb_any(skb); 611 return true; 612 } 613 } 614 615 /* place header in linear portion of buffer */ 616 if (skb_is_nonlinear(skb)) 617 ixgbevf_pull_tail(rx_ring, skb); 618 619 /* if eth_skb_pad returns an error the skb was freed */ 620 if (eth_skb_pad(skb)) 621 return true; 622 623 return false; 624 } 625 626 /* ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring 627 * @rx_ring: rx descriptor ring to store buffers on 628 * @old_buff: donor buffer to have page reused 629 * 630 * Synchronizes page for reuse by the adapter 631 */ 632 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, 633 struct ixgbevf_rx_buffer *old_buff) 634 { 635 struct ixgbevf_rx_buffer *new_buff; 636 u16 nta = rx_ring->next_to_alloc; 637 638 new_buff = &rx_ring->rx_buffer_info[nta]; 639 640 /* update, and store next to alloc */ 641 nta++; 642 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 643 644 /* transfer page from old buffer to new buffer */ 645 new_buff->page = old_buff->page; 646 new_buff->dma = old_buff->dma; 647 new_buff->page_offset = old_buff->page_offset; 648 649 /* sync the buffer for use by the device */ 650 dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma, 651 new_buff->page_offset, 652 IXGBEVF_RX_BUFSZ, 653 DMA_FROM_DEVICE); 654 } 655 656 static inline bool ixgbevf_page_is_reserved(struct page *page) 657 { 658 return (page_to_nid(page) != numa_mem_id()) || page->pfmemalloc; 659 } 660 661 /* ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff 662 * @rx_ring: rx descriptor ring to transact packets on 663 * @rx_buffer: buffer containing page to add 664 * @rx_desc: descriptor containing length of buffer written by hardware 665 * @skb: sk_buff to place the data into 666 * 667 * This function will add the data contained in rx_buffer->page to the skb. 668 * This is done either through a direct copy if the data in the buffer is 669 * less than the skb header size, otherwise it will just attach the page as 670 * a frag to the skb. 671 * 672 * The function will then update the page offset if necessary and return 673 * true if the buffer can be reused by the adapter. 674 */ 675 static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, 676 struct ixgbevf_rx_buffer *rx_buffer, 677 union ixgbe_adv_rx_desc *rx_desc, 678 struct sk_buff *skb) 679 { 680 struct page *page = rx_buffer->page; 681 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length); 682 #if (PAGE_SIZE < 8192) 683 unsigned int truesize = IXGBEVF_RX_BUFSZ; 684 #else 685 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES); 686 #endif 687 688 if ((size <= IXGBEVF_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) { 689 unsigned char *va = page_address(page) + rx_buffer->page_offset; 690 691 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 692 693 /* page is not reserved, we can reuse buffer as is */ 694 if (likely(!ixgbevf_page_is_reserved(page))) 695 return true; 696 697 /* this page cannot be reused so discard it */ 698 put_page(page); 699 return false; 700 } 701 702 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 703 rx_buffer->page_offset, size, truesize); 704 705 /* avoid re-using remote pages */ 706 if (unlikely(ixgbevf_page_is_reserved(page))) 707 return false; 708 709 #if (PAGE_SIZE < 8192) 710 /* if we are only owner of page we can reuse it */ 711 if (unlikely(page_count(page) != 1)) 712 return false; 713 714 /* flip page offset to other buffer */ 715 rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ; 716 717 #else 718 /* move offset up to the next cache line */ 719 rx_buffer->page_offset += truesize; 720 721 if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ)) 722 return false; 723 724 #endif 725 /* Even if we own the page, we are not allowed to use atomic_set() 726 * This would break get_page_unless_zero() users. 727 */ 728 atomic_inc(&page->_count); 729 730 return true; 731 } 732 733 static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring, 734 union ixgbe_adv_rx_desc *rx_desc, 735 struct sk_buff *skb) 736 { 737 struct ixgbevf_rx_buffer *rx_buffer; 738 struct page *page; 739 740 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 741 page = rx_buffer->page; 742 prefetchw(page); 743 744 if (likely(!skb)) { 745 void *page_addr = page_address(page) + 746 rx_buffer->page_offset; 747 748 /* prefetch first cache line of first page */ 749 prefetch(page_addr); 750 #if L1_CACHE_BYTES < 128 751 prefetch(page_addr + L1_CACHE_BYTES); 752 #endif 753 754 /* allocate a skb to store the frags */ 755 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 756 IXGBEVF_RX_HDR_SIZE); 757 if (unlikely(!skb)) { 758 rx_ring->rx_stats.alloc_rx_buff_failed++; 759 return NULL; 760 } 761 762 /* we will be copying header into skb->data in 763 * pskb_may_pull so it is in our interest to prefetch 764 * it now to avoid a possible cache miss 765 */ 766 prefetchw(skb->data); 767 } 768 769 /* we are reusing so sync this buffer for CPU use */ 770 dma_sync_single_range_for_cpu(rx_ring->dev, 771 rx_buffer->dma, 772 rx_buffer->page_offset, 773 IXGBEVF_RX_BUFSZ, 774 DMA_FROM_DEVICE); 775 776 /* pull page into skb */ 777 if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) { 778 /* hand second half of page back to the ring */ 779 ixgbevf_reuse_rx_page(rx_ring, rx_buffer); 780 } else { 781 /* we are not reusing the buffer so unmap it */ 782 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 783 PAGE_SIZE, DMA_FROM_DEVICE); 784 } 785 786 /* clear contents of buffer_info */ 787 rx_buffer->dma = 0; 788 rx_buffer->page = NULL; 789 790 return skb; 791 } 792 793 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 794 u32 qmask) 795 { 796 struct ixgbe_hw *hw = &adapter->hw; 797 798 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); 799 } 800 801 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 802 struct ixgbevf_ring *rx_ring, 803 int budget) 804 { 805 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 806 u16 cleaned_count = ixgbevf_desc_unused(rx_ring); 807 struct sk_buff *skb = rx_ring->skb; 808 809 while (likely(total_rx_packets < budget)) { 810 union ixgbe_adv_rx_desc *rx_desc; 811 812 /* return some buffers to hardware, one at a time is too slow */ 813 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 814 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count); 815 cleaned_count = 0; 816 } 817 818 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); 819 820 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD)) 821 break; 822 823 /* This memory barrier is needed to keep us from reading 824 * any other fields out of the rx_desc until we know the 825 * RXD_STAT_DD bit is set 826 */ 827 rmb(); 828 829 /* retrieve a buffer from the ring */ 830 skb = ixgbevf_fetch_rx_buffer(rx_ring, rx_desc, skb); 831 832 /* exit if we failed to retrieve a buffer */ 833 if (!skb) 834 break; 835 836 cleaned_count++; 837 838 /* fetch next buffer in frame if non-eop */ 839 if (ixgbevf_is_non_eop(rx_ring, rx_desc)) 840 continue; 841 842 /* verify the packet layout is correct */ 843 if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) { 844 skb = NULL; 845 continue; 846 } 847 848 /* probably a little skewed due to removing CRC */ 849 total_rx_bytes += skb->len; 850 851 /* Workaround hardware that can't do proper VEPA multicast 852 * source pruning. 853 */ 854 if ((skb->pkt_type == PACKET_BROADCAST || 855 skb->pkt_type == PACKET_MULTICAST) && 856 ether_addr_equal(rx_ring->netdev->dev_addr, 857 eth_hdr(skb)->h_source)) { 858 dev_kfree_skb_irq(skb); 859 continue; 860 } 861 862 /* populate checksum, VLAN, and protocol */ 863 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); 864 865 ixgbevf_rx_skb(q_vector, skb); 866 867 /* reset skb pointer */ 868 skb = NULL; 869 870 /* update budget accounting */ 871 total_rx_packets++; 872 } 873 874 /* place incomplete frames back on ring for completion */ 875 rx_ring->skb = skb; 876 877 u64_stats_update_begin(&rx_ring->syncp); 878 rx_ring->stats.packets += total_rx_packets; 879 rx_ring->stats.bytes += total_rx_bytes; 880 u64_stats_update_end(&rx_ring->syncp); 881 q_vector->rx.total_packets += total_rx_packets; 882 q_vector->rx.total_bytes += total_rx_bytes; 883 884 return total_rx_packets; 885 } 886 887 /** 888 * ixgbevf_poll - NAPI polling calback 889 * @napi: napi struct with our devices info in it 890 * @budget: amount of work driver is allowed to do this pass, in packets 891 * 892 * This function will clean more than one or more rings associated with a 893 * q_vector. 894 **/ 895 static int ixgbevf_poll(struct napi_struct *napi, int budget) 896 { 897 struct ixgbevf_q_vector *q_vector = 898 container_of(napi, struct ixgbevf_q_vector, napi); 899 struct ixgbevf_adapter *adapter = q_vector->adapter; 900 struct ixgbevf_ring *ring; 901 int per_ring_budget; 902 bool clean_complete = true; 903 904 ixgbevf_for_each_ring(ring, q_vector->tx) 905 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring); 906 907 #ifdef CONFIG_NET_RX_BUSY_POLL 908 if (!ixgbevf_qv_lock_napi(q_vector)) 909 return budget; 910 #endif 911 912 /* attempt to distribute budget to each queue fairly, but don't allow 913 * the budget to go below 1 because we'll exit polling */ 914 if (q_vector->rx.count > 1) 915 per_ring_budget = max(budget/q_vector->rx.count, 1); 916 else 917 per_ring_budget = budget; 918 919 ixgbevf_for_each_ring(ring, q_vector->rx) 920 clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring, 921 per_ring_budget) 922 < per_ring_budget); 923 924 #ifdef CONFIG_NET_RX_BUSY_POLL 925 ixgbevf_qv_unlock_napi(q_vector); 926 #endif 927 928 /* If all work not completed, return budget and keep polling */ 929 if (!clean_complete) 930 return budget; 931 /* all work done, exit the polling mode */ 932 napi_complete(napi); 933 if (adapter->rx_itr_setting & 1) 934 ixgbevf_set_itr(q_vector); 935 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 936 !test_bit(__IXGBEVF_REMOVING, &adapter->state)) 937 ixgbevf_irq_enable_queues(adapter, 938 1 << q_vector->v_idx); 939 940 return 0; 941 } 942 943 /** 944 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 945 * @q_vector: structure containing interrupt and ring information 946 */ 947 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) 948 { 949 struct ixgbevf_adapter *adapter = q_vector->adapter; 950 struct ixgbe_hw *hw = &adapter->hw; 951 int v_idx = q_vector->v_idx; 952 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 953 954 /* 955 * set the WDIS bit to not clear the timer bits and cause an 956 * immediate assertion of the interrupt 957 */ 958 itr_reg |= IXGBE_EITR_CNT_WDIS; 959 960 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 961 } 962 963 #ifdef CONFIG_NET_RX_BUSY_POLL 964 /* must be called with local_bh_disable()d */ 965 static int ixgbevf_busy_poll_recv(struct napi_struct *napi) 966 { 967 struct ixgbevf_q_vector *q_vector = 968 container_of(napi, struct ixgbevf_q_vector, napi); 969 struct ixgbevf_adapter *adapter = q_vector->adapter; 970 struct ixgbevf_ring *ring; 971 int found = 0; 972 973 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 974 return LL_FLUSH_FAILED; 975 976 if (!ixgbevf_qv_lock_poll(q_vector)) 977 return LL_FLUSH_BUSY; 978 979 ixgbevf_for_each_ring(ring, q_vector->rx) { 980 found = ixgbevf_clean_rx_irq(q_vector, ring, 4); 981 #ifdef BP_EXTENDED_STATS 982 if (found) 983 ring->stats.cleaned += found; 984 else 985 ring->stats.misses++; 986 #endif 987 if (found) 988 break; 989 } 990 991 ixgbevf_qv_unlock_poll(q_vector); 992 993 return found; 994 } 995 #endif /* CONFIG_NET_RX_BUSY_POLL */ 996 997 /** 998 * ixgbevf_configure_msix - Configure MSI-X hardware 999 * @adapter: board private structure 1000 * 1001 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 1002 * interrupts. 1003 **/ 1004 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 1005 { 1006 struct ixgbevf_q_vector *q_vector; 1007 int q_vectors, v_idx; 1008 1009 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1010 adapter->eims_enable_mask = 0; 1011 1012 /* 1013 * Populate the IVAR table and set the ITR values to the 1014 * corresponding register. 1015 */ 1016 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 1017 struct ixgbevf_ring *ring; 1018 q_vector = adapter->q_vector[v_idx]; 1019 1020 ixgbevf_for_each_ring(ring, q_vector->rx) 1021 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx); 1022 1023 ixgbevf_for_each_ring(ring, q_vector->tx) 1024 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx); 1025 1026 if (q_vector->tx.ring && !q_vector->rx.ring) { 1027 /* tx only vector */ 1028 if (adapter->tx_itr_setting == 1) 1029 q_vector->itr = IXGBE_10K_ITR; 1030 else 1031 q_vector->itr = adapter->tx_itr_setting; 1032 } else { 1033 /* rx or rx/tx vector */ 1034 if (adapter->rx_itr_setting == 1) 1035 q_vector->itr = IXGBE_20K_ITR; 1036 else 1037 q_vector->itr = adapter->rx_itr_setting; 1038 } 1039 1040 /* add q_vector eims value to global eims_enable_mask */ 1041 adapter->eims_enable_mask |= 1 << v_idx; 1042 1043 ixgbevf_write_eitr(q_vector); 1044 } 1045 1046 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 1047 /* setup eims_other and add value to global eims_enable_mask */ 1048 adapter->eims_other = 1 << v_idx; 1049 adapter->eims_enable_mask |= adapter->eims_other; 1050 } 1051 1052 enum latency_range { 1053 lowest_latency = 0, 1054 low_latency = 1, 1055 bulk_latency = 2, 1056 latency_invalid = 255 1057 }; 1058 1059 /** 1060 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 1061 * @q_vector: structure containing interrupt and ring information 1062 * @ring_container: structure containing ring performance data 1063 * 1064 * Stores a new ITR value based on packets and byte 1065 * counts during the last interrupt. The advantage of per interrupt 1066 * computation is faster updates and more accurate ITR for the current 1067 * traffic pattern. Constants in this function were computed 1068 * based on theoretical maximum wire speed and thresholds were set based 1069 * on testing data as well as attempting to minimize response time 1070 * while increasing bulk throughput. 1071 **/ 1072 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector, 1073 struct ixgbevf_ring_container *ring_container) 1074 { 1075 int bytes = ring_container->total_bytes; 1076 int packets = ring_container->total_packets; 1077 u32 timepassed_us; 1078 u64 bytes_perint; 1079 u8 itr_setting = ring_container->itr; 1080 1081 if (packets == 0) 1082 return; 1083 1084 /* simple throttlerate management 1085 * 0-20MB/s lowest (100000 ints/s) 1086 * 20-100MB/s low (20000 ints/s) 1087 * 100-1249MB/s bulk (8000 ints/s) 1088 */ 1089 /* what was last interrupt timeslice? */ 1090 timepassed_us = q_vector->itr >> 2; 1091 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 1092 1093 switch (itr_setting) { 1094 case lowest_latency: 1095 if (bytes_perint > 10) 1096 itr_setting = low_latency; 1097 break; 1098 case low_latency: 1099 if (bytes_perint > 20) 1100 itr_setting = bulk_latency; 1101 else if (bytes_perint <= 10) 1102 itr_setting = lowest_latency; 1103 break; 1104 case bulk_latency: 1105 if (bytes_perint <= 20) 1106 itr_setting = low_latency; 1107 break; 1108 } 1109 1110 /* clear work counters since we have the values we need */ 1111 ring_container->total_bytes = 0; 1112 ring_container->total_packets = 0; 1113 1114 /* write updated itr to ring container */ 1115 ring_container->itr = itr_setting; 1116 } 1117 1118 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector) 1119 { 1120 u32 new_itr = q_vector->itr; 1121 u8 current_itr; 1122 1123 ixgbevf_update_itr(q_vector, &q_vector->tx); 1124 ixgbevf_update_itr(q_vector, &q_vector->rx); 1125 1126 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 1127 1128 switch (current_itr) { 1129 /* counts and packets in update_itr are dependent on these numbers */ 1130 case lowest_latency: 1131 new_itr = IXGBE_100K_ITR; 1132 break; 1133 case low_latency: 1134 new_itr = IXGBE_20K_ITR; 1135 break; 1136 case bulk_latency: 1137 default: 1138 new_itr = IXGBE_8K_ITR; 1139 break; 1140 } 1141 1142 if (new_itr != q_vector->itr) { 1143 /* do an exponential smoothing */ 1144 new_itr = (10 * new_itr * q_vector->itr) / 1145 ((9 * new_itr) + q_vector->itr); 1146 1147 /* save the algorithm value here */ 1148 q_vector->itr = new_itr; 1149 1150 ixgbevf_write_eitr(q_vector); 1151 } 1152 } 1153 1154 static irqreturn_t ixgbevf_msix_other(int irq, void *data) 1155 { 1156 struct ixgbevf_adapter *adapter = data; 1157 struct ixgbe_hw *hw = &adapter->hw; 1158 1159 hw->mac.get_link_status = 1; 1160 1161 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 1162 !test_bit(__IXGBEVF_REMOVING, &adapter->state)) 1163 mod_timer(&adapter->watchdog_timer, jiffies); 1164 1165 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); 1166 1167 return IRQ_HANDLED; 1168 } 1169 1170 /** 1171 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) 1172 * @irq: unused 1173 * @data: pointer to our q_vector struct for this interrupt vector 1174 **/ 1175 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data) 1176 { 1177 struct ixgbevf_q_vector *q_vector = data; 1178 1179 /* EIAM disabled interrupts (on this vector) for us */ 1180 if (q_vector->rx.ring || q_vector->tx.ring) 1181 napi_schedule(&q_vector->napi); 1182 1183 return IRQ_HANDLED; 1184 } 1185 1186 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, 1187 int r_idx) 1188 { 1189 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1190 1191 a->rx_ring[r_idx]->next = q_vector->rx.ring; 1192 q_vector->rx.ring = a->rx_ring[r_idx]; 1193 q_vector->rx.count++; 1194 } 1195 1196 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, 1197 int t_idx) 1198 { 1199 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1200 1201 a->tx_ring[t_idx]->next = q_vector->tx.ring; 1202 q_vector->tx.ring = a->tx_ring[t_idx]; 1203 q_vector->tx.count++; 1204 } 1205 1206 /** 1207 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors 1208 * @adapter: board private structure to initialize 1209 * 1210 * This function maps descriptor rings to the queue-specific vectors 1211 * we were allotted through the MSI-X enabling code. Ideally, we'd have 1212 * one vector per ring/queue, but on a constrained vector budget, we 1213 * group the rings as "efficiently" as possible. You would add new 1214 * mapping configurations in here. 1215 **/ 1216 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter) 1217 { 1218 int q_vectors; 1219 int v_start = 0; 1220 int rxr_idx = 0, txr_idx = 0; 1221 int rxr_remaining = adapter->num_rx_queues; 1222 int txr_remaining = adapter->num_tx_queues; 1223 int i, j; 1224 int rqpv, tqpv; 1225 int err = 0; 1226 1227 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1228 1229 /* 1230 * The ideal configuration... 1231 * We have enough vectors to map one per queue. 1232 */ 1233 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) { 1234 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) 1235 map_vector_to_rxq(adapter, v_start, rxr_idx); 1236 1237 for (; txr_idx < txr_remaining; v_start++, txr_idx++) 1238 map_vector_to_txq(adapter, v_start, txr_idx); 1239 goto out; 1240 } 1241 1242 /* 1243 * If we don't have enough vectors for a 1-to-1 1244 * mapping, we'll have to group them so there are 1245 * multiple queues per vector. 1246 */ 1247 /* Re-adjusting *qpv takes care of the remainder. */ 1248 for (i = v_start; i < q_vectors; i++) { 1249 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i); 1250 for (j = 0; j < rqpv; j++) { 1251 map_vector_to_rxq(adapter, i, rxr_idx); 1252 rxr_idx++; 1253 rxr_remaining--; 1254 } 1255 } 1256 for (i = v_start; i < q_vectors; i++) { 1257 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i); 1258 for (j = 0; j < tqpv; j++) { 1259 map_vector_to_txq(adapter, i, txr_idx); 1260 txr_idx++; 1261 txr_remaining--; 1262 } 1263 } 1264 1265 out: 1266 return err; 1267 } 1268 1269 /** 1270 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 1271 * @adapter: board private structure 1272 * 1273 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 1274 * interrupts from the kernel. 1275 **/ 1276 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 1277 { 1278 struct net_device *netdev = adapter->netdev; 1279 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1280 int vector, err; 1281 int ri = 0, ti = 0; 1282 1283 for (vector = 0; vector < q_vectors; vector++) { 1284 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector]; 1285 struct msix_entry *entry = &adapter->msix_entries[vector]; 1286 1287 if (q_vector->tx.ring && q_vector->rx.ring) { 1288 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1289 "%s-%s-%d", netdev->name, "TxRx", ri++); 1290 ti++; 1291 } else if (q_vector->rx.ring) { 1292 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1293 "%s-%s-%d", netdev->name, "rx", ri++); 1294 } else if (q_vector->tx.ring) { 1295 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1296 "%s-%s-%d", netdev->name, "tx", ti++); 1297 } else { 1298 /* skip this unused q_vector */ 1299 continue; 1300 } 1301 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0, 1302 q_vector->name, q_vector); 1303 if (err) { 1304 hw_dbg(&adapter->hw, 1305 "request_irq failed for MSIX interrupt " 1306 "Error: %d\n", err); 1307 goto free_queue_irqs; 1308 } 1309 } 1310 1311 err = request_irq(adapter->msix_entries[vector].vector, 1312 &ixgbevf_msix_other, 0, netdev->name, adapter); 1313 if (err) { 1314 hw_dbg(&adapter->hw, 1315 "request_irq for msix_other failed: %d\n", err); 1316 goto free_queue_irqs; 1317 } 1318 1319 return 0; 1320 1321 free_queue_irqs: 1322 while (vector) { 1323 vector--; 1324 free_irq(adapter->msix_entries[vector].vector, 1325 adapter->q_vector[vector]); 1326 } 1327 /* This failure is non-recoverable - it indicates the system is 1328 * out of MSIX vector resources and the VF driver cannot run 1329 * without them. Set the number of msix vectors to zero 1330 * indicating that not enough can be allocated. The error 1331 * will be returned to the user indicating device open failed. 1332 * Any further attempts to force the driver to open will also 1333 * fail. The only way to recover is to unload the driver and 1334 * reload it again. If the system has recovered some MSIX 1335 * vectors then it may succeed. 1336 */ 1337 adapter->num_msix_vectors = 0; 1338 return err; 1339 } 1340 1341 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter) 1342 { 1343 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1344 1345 for (i = 0; i < q_vectors; i++) { 1346 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i]; 1347 q_vector->rx.ring = NULL; 1348 q_vector->tx.ring = NULL; 1349 q_vector->rx.count = 0; 1350 q_vector->tx.count = 0; 1351 } 1352 } 1353 1354 /** 1355 * ixgbevf_request_irq - initialize interrupts 1356 * @adapter: board private structure 1357 * 1358 * Attempts to configure interrupts using the best available 1359 * capabilities of the hardware and kernel. 1360 **/ 1361 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 1362 { 1363 int err = 0; 1364 1365 err = ixgbevf_request_msix_irqs(adapter); 1366 1367 if (err) 1368 hw_dbg(&adapter->hw, 1369 "request_irq failed, Error %d\n", err); 1370 1371 return err; 1372 } 1373 1374 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 1375 { 1376 int i, q_vectors; 1377 1378 q_vectors = adapter->num_msix_vectors; 1379 i = q_vectors - 1; 1380 1381 free_irq(adapter->msix_entries[i].vector, adapter); 1382 i--; 1383 1384 for (; i >= 0; i--) { 1385 /* free only the irqs that were actually requested */ 1386 if (!adapter->q_vector[i]->rx.ring && 1387 !adapter->q_vector[i]->tx.ring) 1388 continue; 1389 1390 free_irq(adapter->msix_entries[i].vector, 1391 adapter->q_vector[i]); 1392 } 1393 1394 ixgbevf_reset_q_vectors(adapter); 1395 } 1396 1397 /** 1398 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1399 * @adapter: board private structure 1400 **/ 1401 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1402 { 1403 struct ixgbe_hw *hw = &adapter->hw; 1404 int i; 1405 1406 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0); 1407 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1408 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0); 1409 1410 IXGBE_WRITE_FLUSH(hw); 1411 1412 for (i = 0; i < adapter->num_msix_vectors; i++) 1413 synchronize_irq(adapter->msix_entries[i].vector); 1414 } 1415 1416 /** 1417 * ixgbevf_irq_enable - Enable default interrupt generation settings 1418 * @adapter: board private structure 1419 **/ 1420 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) 1421 { 1422 struct ixgbe_hw *hw = &adapter->hw; 1423 1424 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask); 1425 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask); 1426 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); 1427 } 1428 1429 /** 1430 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset 1431 * @adapter: board private structure 1432 * @ring: structure containing ring specific data 1433 * 1434 * Configure the Tx descriptor ring after a reset. 1435 **/ 1436 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 1437 struct ixgbevf_ring *ring) 1438 { 1439 struct ixgbe_hw *hw = &adapter->hw; 1440 u64 tdba = ring->dma; 1441 int wait_loop = 10; 1442 u32 txdctl = IXGBE_TXDCTL_ENABLE; 1443 u8 reg_idx = ring->reg_idx; 1444 1445 /* disable queue to avoid issues while updating state */ 1446 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 1447 IXGBE_WRITE_FLUSH(hw); 1448 1449 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 1450 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32); 1451 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx), 1452 ring->count * sizeof(union ixgbe_adv_tx_desc)); 1453 1454 /* disable head writeback */ 1455 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0); 1456 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0); 1457 1458 /* enable relaxed ordering */ 1459 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx), 1460 (IXGBE_DCA_TXCTRL_DESC_RRO_EN | 1461 IXGBE_DCA_TXCTRL_DATA_RRO_EN)); 1462 1463 /* reset head and tail pointers */ 1464 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0); 1465 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0); 1466 ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx); 1467 1468 /* reset ntu and ntc to place SW in sync with hardwdare */ 1469 ring->next_to_clean = 0; 1470 ring->next_to_use = 0; 1471 1472 /* In order to avoid issues WTHRESH + PTHRESH should always be equal 1473 * to or less than the number of on chip descriptors, which is 1474 * currently 40. 1475 */ 1476 txdctl |= (8 << 16); /* WTHRESH = 8 */ 1477 1478 /* Setting PTHRESH to 32 both improves performance */ 1479 txdctl |= (1 << 8) | /* HTHRESH = 1 */ 1480 32; /* PTHRESH = 32 */ 1481 1482 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl); 1483 1484 /* poll to verify queue is enabled */ 1485 do { 1486 usleep_range(1000, 2000); 1487 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx)); 1488 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 1489 if (!wait_loop) 1490 pr_err("Could not enable Tx Queue %d\n", reg_idx); 1491 } 1492 1493 /** 1494 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1495 * @adapter: board private structure 1496 * 1497 * Configure the Tx unit of the MAC after a reset. 1498 **/ 1499 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1500 { 1501 u32 i; 1502 1503 /* Setup the HW Tx Head and Tail descriptor pointers */ 1504 for (i = 0; i < adapter->num_tx_queues; i++) 1505 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]); 1506 } 1507 1508 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1509 1510 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) 1511 { 1512 struct ixgbe_hw *hw = &adapter->hw; 1513 u32 srrctl; 1514 1515 srrctl = IXGBE_SRRCTL_DROP_EN; 1516 1517 srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; 1518 srrctl |= IXGBEVF_RX_BUFSZ >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1519 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1520 1521 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1522 } 1523 1524 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) 1525 { 1526 struct ixgbe_hw *hw = &adapter->hw; 1527 1528 /* PSRTYPE must be initialized in 82599 */ 1529 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 1530 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR | 1531 IXGBE_PSRTYPE_L2HDR; 1532 1533 if (adapter->num_rx_queues > 1) 1534 psrtype |= 1 << 29; 1535 1536 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1537 } 1538 1539 #define IXGBEVF_MAX_RX_DESC_POLL 10 1540 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, 1541 struct ixgbevf_ring *ring) 1542 { 1543 struct ixgbe_hw *hw = &adapter->hw; 1544 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1545 u32 rxdctl; 1546 u8 reg_idx = ring->reg_idx; 1547 1548 if (IXGBE_REMOVED(hw->hw_addr)) 1549 return; 1550 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1551 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1552 1553 /* write value back with RXDCTL.ENABLE bit cleared */ 1554 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1555 1556 /* the hardware may take up to 100us to really disable the rx queue */ 1557 do { 1558 udelay(10); 1559 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1560 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 1561 1562 if (!wait_loop) 1563 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", 1564 reg_idx); 1565 } 1566 1567 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1568 struct ixgbevf_ring *ring) 1569 { 1570 struct ixgbe_hw *hw = &adapter->hw; 1571 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1572 u32 rxdctl; 1573 u8 reg_idx = ring->reg_idx; 1574 1575 if (IXGBE_REMOVED(hw->hw_addr)) 1576 return; 1577 do { 1578 usleep_range(1000, 2000); 1579 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1580 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 1581 1582 if (!wait_loop) 1583 pr_err("RXDCTL.ENABLE queue %d not set while polling\n", 1584 reg_idx); 1585 } 1586 1587 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, 1588 struct ixgbevf_ring *ring) 1589 { 1590 struct ixgbe_hw *hw = &adapter->hw; 1591 u64 rdba = ring->dma; 1592 u32 rxdctl; 1593 u8 reg_idx = ring->reg_idx; 1594 1595 /* disable queue to avoid issues while updating state */ 1596 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1597 ixgbevf_disable_rx_queue(adapter, ring); 1598 1599 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 1600 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32); 1601 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx), 1602 ring->count * sizeof(union ixgbe_adv_rx_desc)); 1603 1604 /* enable relaxed ordering */ 1605 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1606 IXGBE_DCA_RXCTRL_DESC_RRO_EN); 1607 1608 /* reset head and tail pointers */ 1609 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0); 1610 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); 1611 ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); 1612 1613 /* reset ntu and ntc to place SW in sync with hardwdare */ 1614 ring->next_to_clean = 0; 1615 ring->next_to_use = 0; 1616 ring->next_to_alloc = 0; 1617 1618 ixgbevf_configure_srrctl(adapter, reg_idx); 1619 1620 /* allow any size packet since we can handle overflow */ 1621 rxdctl &= ~IXGBE_RXDCTL_RLPML_EN; 1622 1623 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1624 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1625 1626 ixgbevf_rx_desc_queue_enable(adapter, ring); 1627 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); 1628 } 1629 1630 /** 1631 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1632 * @adapter: board private structure 1633 * 1634 * Configure the Rx unit of the MAC after a reset. 1635 **/ 1636 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1637 { 1638 int i; 1639 struct ixgbe_hw *hw = &adapter->hw; 1640 struct net_device *netdev = adapter->netdev; 1641 1642 ixgbevf_setup_psrtype(adapter); 1643 1644 /* notify the PF of our intent to use this size of frame */ 1645 ixgbevf_rlpml_set_vf(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN); 1646 1647 /* Setup the HW Rx Head and Tail Descriptor Pointers and 1648 * the Base and Length of the Rx Descriptor Ring */ 1649 for (i = 0; i < adapter->num_rx_queues; i++) 1650 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]); 1651 } 1652 1653 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, 1654 __be16 proto, u16 vid) 1655 { 1656 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1657 struct ixgbe_hw *hw = &adapter->hw; 1658 int err; 1659 1660 spin_lock_bh(&adapter->mbx_lock); 1661 1662 /* add VID to filter table */ 1663 err = hw->mac.ops.set_vfta(hw, vid, 0, true); 1664 1665 spin_unlock_bh(&adapter->mbx_lock); 1666 1667 /* translate error return types so error makes sense */ 1668 if (err == IXGBE_ERR_MBX) 1669 return -EIO; 1670 1671 if (err == IXGBE_ERR_INVALID_ARGUMENT) 1672 return -EACCES; 1673 1674 set_bit(vid, adapter->active_vlans); 1675 1676 return err; 1677 } 1678 1679 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, 1680 __be16 proto, u16 vid) 1681 { 1682 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1683 struct ixgbe_hw *hw = &adapter->hw; 1684 int err = -EOPNOTSUPP; 1685 1686 spin_lock_bh(&adapter->mbx_lock); 1687 1688 /* remove VID from filter table */ 1689 err = hw->mac.ops.set_vfta(hw, vid, 0, false); 1690 1691 spin_unlock_bh(&adapter->mbx_lock); 1692 1693 clear_bit(vid, adapter->active_vlans); 1694 1695 return err; 1696 } 1697 1698 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 1699 { 1700 u16 vid; 1701 1702 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1703 ixgbevf_vlan_rx_add_vid(adapter->netdev, 1704 htons(ETH_P_8021Q), vid); 1705 } 1706 1707 static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 1708 { 1709 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1710 struct ixgbe_hw *hw = &adapter->hw; 1711 int count = 0; 1712 1713 if ((netdev_uc_count(netdev)) > 10) { 1714 pr_err("Too many unicast filters - No Space\n"); 1715 return -ENOSPC; 1716 } 1717 1718 if (!netdev_uc_empty(netdev)) { 1719 struct netdev_hw_addr *ha; 1720 netdev_for_each_uc_addr(ha, netdev) { 1721 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 1722 udelay(200); 1723 } 1724 } else { 1725 /* 1726 * If the list is empty then send message to PF driver to 1727 * clear all macvlans on this VF. 1728 */ 1729 hw->mac.ops.set_uc_addr(hw, 0, NULL); 1730 } 1731 1732 return count; 1733 } 1734 1735 /** 1736 * ixgbevf_set_rx_mode - Multicast and unicast set 1737 * @netdev: network interface device structure 1738 * 1739 * The set_rx_method entry point is called whenever the multicast address 1740 * list, unicast address list or the network interface flags are updated. 1741 * This routine is responsible for configuring the hardware for proper 1742 * multicast mode and configuring requested unicast filters. 1743 **/ 1744 static void ixgbevf_set_rx_mode(struct net_device *netdev) 1745 { 1746 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1747 struct ixgbe_hw *hw = &adapter->hw; 1748 1749 spin_lock_bh(&adapter->mbx_lock); 1750 1751 /* reprogram multicast list */ 1752 hw->mac.ops.update_mc_addr_list(hw, netdev); 1753 1754 ixgbevf_write_uc_addr_list(netdev); 1755 1756 spin_unlock_bh(&adapter->mbx_lock); 1757 } 1758 1759 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 1760 { 1761 int q_idx; 1762 struct ixgbevf_q_vector *q_vector; 1763 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1764 1765 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1766 q_vector = adapter->q_vector[q_idx]; 1767 #ifdef CONFIG_NET_RX_BUSY_POLL 1768 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]); 1769 #endif 1770 napi_enable(&q_vector->napi); 1771 } 1772 } 1773 1774 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 1775 { 1776 int q_idx; 1777 struct ixgbevf_q_vector *q_vector; 1778 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1779 1780 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1781 q_vector = adapter->q_vector[q_idx]; 1782 napi_disable(&q_vector->napi); 1783 #ifdef CONFIG_NET_RX_BUSY_POLL 1784 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) { 1785 pr_info("QV %d locked\n", q_idx); 1786 usleep_range(1000, 20000); 1787 } 1788 #endif /* CONFIG_NET_RX_BUSY_POLL */ 1789 } 1790 } 1791 1792 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter) 1793 { 1794 struct ixgbe_hw *hw = &adapter->hw; 1795 unsigned int def_q = 0; 1796 unsigned int num_tcs = 0; 1797 unsigned int num_rx_queues = 1; 1798 int err; 1799 1800 spin_lock_bh(&adapter->mbx_lock); 1801 1802 /* fetch queue configuration from the PF */ 1803 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 1804 1805 spin_unlock_bh(&adapter->mbx_lock); 1806 1807 if (err) 1808 return err; 1809 1810 if (num_tcs > 1) { 1811 /* update default Tx ring register index */ 1812 adapter->tx_ring[0]->reg_idx = def_q; 1813 1814 /* we need as many queues as traffic classes */ 1815 num_rx_queues = num_tcs; 1816 } 1817 1818 /* if we have a bad config abort request queue reset */ 1819 if (adapter->num_rx_queues != num_rx_queues) { 1820 /* force mailbox timeout to prevent further messages */ 1821 hw->mbx.timeout = 0; 1822 1823 /* wait for watchdog to come around and bail us out */ 1824 adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED; 1825 } 1826 1827 return 0; 1828 } 1829 1830 static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 1831 { 1832 ixgbevf_configure_dcb(adapter); 1833 1834 ixgbevf_set_rx_mode(adapter->netdev); 1835 1836 ixgbevf_restore_vlan(adapter); 1837 1838 ixgbevf_configure_tx(adapter); 1839 ixgbevf_configure_rx(adapter); 1840 } 1841 1842 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 1843 { 1844 /* Only save pre-reset stats if there are some */ 1845 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 1846 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 1847 adapter->stats.base_vfgprc; 1848 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 1849 adapter->stats.base_vfgptc; 1850 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 1851 adapter->stats.base_vfgorc; 1852 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 1853 adapter->stats.base_vfgotc; 1854 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 1855 adapter->stats.base_vfmprc; 1856 } 1857 } 1858 1859 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 1860 { 1861 struct ixgbe_hw *hw = &adapter->hw; 1862 1863 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 1864 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 1865 adapter->stats.last_vfgorc |= 1866 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 1867 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 1868 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 1869 adapter->stats.last_vfgotc |= 1870 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 1871 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 1872 1873 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 1874 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 1875 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 1876 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 1877 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 1878 } 1879 1880 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) 1881 { 1882 struct ixgbe_hw *hw = &adapter->hw; 1883 int api[] = { ixgbe_mbox_api_11, 1884 ixgbe_mbox_api_10, 1885 ixgbe_mbox_api_unknown }; 1886 int err = 0, idx = 0; 1887 1888 spin_lock_bh(&adapter->mbx_lock); 1889 1890 while (api[idx] != ixgbe_mbox_api_unknown) { 1891 err = ixgbevf_negotiate_api_version(hw, api[idx]); 1892 if (!err) 1893 break; 1894 idx++; 1895 } 1896 1897 spin_unlock_bh(&adapter->mbx_lock); 1898 } 1899 1900 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 1901 { 1902 struct net_device *netdev = adapter->netdev; 1903 struct ixgbe_hw *hw = &adapter->hw; 1904 1905 ixgbevf_configure_msix(adapter); 1906 1907 spin_lock_bh(&adapter->mbx_lock); 1908 1909 if (is_valid_ether_addr(hw->mac.addr)) 1910 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 1911 else 1912 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 1913 1914 spin_unlock_bh(&adapter->mbx_lock); 1915 1916 smp_mb__before_atomic(); 1917 clear_bit(__IXGBEVF_DOWN, &adapter->state); 1918 ixgbevf_napi_enable_all(adapter); 1919 1920 /* enable transmits */ 1921 netif_tx_start_all_queues(netdev); 1922 1923 ixgbevf_save_reset_stats(adapter); 1924 ixgbevf_init_last_counter_stats(adapter); 1925 1926 hw->mac.get_link_status = 1; 1927 mod_timer(&adapter->watchdog_timer, jiffies); 1928 } 1929 1930 void ixgbevf_up(struct ixgbevf_adapter *adapter) 1931 { 1932 struct ixgbe_hw *hw = &adapter->hw; 1933 1934 ixgbevf_configure(adapter); 1935 1936 ixgbevf_up_complete(adapter); 1937 1938 /* clear any pending interrupts, may auto mask */ 1939 IXGBE_READ_REG(hw, IXGBE_VTEICR); 1940 1941 ixgbevf_irq_enable(adapter); 1942 } 1943 1944 /** 1945 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 1946 * @rx_ring: ring to free buffers from 1947 **/ 1948 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) 1949 { 1950 struct device *dev = rx_ring->dev; 1951 unsigned long size; 1952 unsigned int i; 1953 1954 /* Free Rx ring sk_buff */ 1955 if (rx_ring->skb) { 1956 dev_kfree_skb(rx_ring->skb); 1957 rx_ring->skb = NULL; 1958 } 1959 1960 /* ring already cleared, nothing to do */ 1961 if (!rx_ring->rx_buffer_info) 1962 return; 1963 1964 /* Free all the Rx ring pages */ 1965 for (i = 0; i < rx_ring->count; i++) { 1966 struct ixgbevf_rx_buffer *rx_buffer; 1967 1968 rx_buffer = &rx_ring->rx_buffer_info[i]; 1969 if (rx_buffer->dma) 1970 dma_unmap_page(dev, rx_buffer->dma, 1971 PAGE_SIZE, DMA_FROM_DEVICE); 1972 rx_buffer->dma = 0; 1973 if (rx_buffer->page) 1974 __free_page(rx_buffer->page); 1975 rx_buffer->page = NULL; 1976 } 1977 1978 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 1979 memset(rx_ring->rx_buffer_info, 0, size); 1980 1981 /* Zero out the descriptor ring */ 1982 memset(rx_ring->desc, 0, rx_ring->size); 1983 } 1984 1985 /** 1986 * ixgbevf_clean_tx_ring - Free Tx Buffers 1987 * @tx_ring: ring to be cleaned 1988 **/ 1989 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) 1990 { 1991 struct ixgbevf_tx_buffer *tx_buffer_info; 1992 unsigned long size; 1993 unsigned int i; 1994 1995 if (!tx_ring->tx_buffer_info) 1996 return; 1997 1998 /* Free all the Tx ring sk_buffs */ 1999 for (i = 0; i < tx_ring->count; i++) { 2000 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2001 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 2002 } 2003 2004 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2005 memset(tx_ring->tx_buffer_info, 0, size); 2006 2007 memset(tx_ring->desc, 0, tx_ring->size); 2008 } 2009 2010 /** 2011 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 2012 * @adapter: board private structure 2013 **/ 2014 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 2015 { 2016 int i; 2017 2018 for (i = 0; i < adapter->num_rx_queues; i++) 2019 ixgbevf_clean_rx_ring(adapter->rx_ring[i]); 2020 } 2021 2022 /** 2023 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 2024 * @adapter: board private structure 2025 **/ 2026 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 2027 { 2028 int i; 2029 2030 for (i = 0; i < adapter->num_tx_queues; i++) 2031 ixgbevf_clean_tx_ring(adapter->tx_ring[i]); 2032 } 2033 2034 void ixgbevf_down(struct ixgbevf_adapter *adapter) 2035 { 2036 struct net_device *netdev = adapter->netdev; 2037 struct ixgbe_hw *hw = &adapter->hw; 2038 int i; 2039 2040 /* signal that we are down to the interrupt handler */ 2041 if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state)) 2042 return; /* do nothing if already down */ 2043 2044 /* disable all enabled rx queues */ 2045 for (i = 0; i < adapter->num_rx_queues; i++) 2046 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]); 2047 2048 netif_tx_disable(netdev); 2049 2050 msleep(10); 2051 2052 netif_tx_stop_all_queues(netdev); 2053 2054 ixgbevf_irq_disable(adapter); 2055 2056 ixgbevf_napi_disable_all(adapter); 2057 2058 del_timer_sync(&adapter->watchdog_timer); 2059 /* can't call flush scheduled work here because it can deadlock 2060 * if linkwatch_event tries to acquire the rtnl_lock which we are 2061 * holding */ 2062 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK) 2063 msleep(1); 2064 2065 /* disable transmits in the hardware now that interrupts are off */ 2066 for (i = 0; i < adapter->num_tx_queues; i++) { 2067 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 2068 2069 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), 2070 IXGBE_TXDCTL_SWFLSH); 2071 } 2072 2073 netif_carrier_off(netdev); 2074 2075 if (!pci_channel_offline(adapter->pdev)) 2076 ixgbevf_reset(adapter); 2077 2078 ixgbevf_clean_all_tx_rings(adapter); 2079 ixgbevf_clean_all_rx_rings(adapter); 2080 } 2081 2082 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 2083 { 2084 WARN_ON(in_interrupt()); 2085 2086 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 2087 msleep(1); 2088 2089 ixgbevf_down(adapter); 2090 ixgbevf_up(adapter); 2091 2092 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 2093 } 2094 2095 void ixgbevf_reset(struct ixgbevf_adapter *adapter) 2096 { 2097 struct ixgbe_hw *hw = &adapter->hw; 2098 struct net_device *netdev = adapter->netdev; 2099 2100 if (hw->mac.ops.reset_hw(hw)) { 2101 hw_dbg(hw, "PF still resetting\n"); 2102 } else { 2103 hw->mac.ops.init_hw(hw); 2104 ixgbevf_negotiate_api(adapter); 2105 } 2106 2107 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 2108 memcpy(netdev->dev_addr, adapter->hw.mac.addr, 2109 netdev->addr_len); 2110 memcpy(netdev->perm_addr, adapter->hw.mac.addr, 2111 netdev->addr_len); 2112 } 2113 } 2114 2115 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 2116 int vectors) 2117 { 2118 int vector_threshold; 2119 2120 /* We'll want at least 2 (vector_threshold): 2121 * 1) TxQ[0] + RxQ[0] handler 2122 * 2) Other (Link Status Change, etc.) 2123 */ 2124 vector_threshold = MIN_MSIX_COUNT; 2125 2126 /* The more we get, the more we will assign to Tx/Rx Cleanup 2127 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 2128 * Right now, we simply care about how many we'll get; we'll 2129 * set them up later while requesting irq's. 2130 */ 2131 vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 2132 vector_threshold, vectors); 2133 2134 if (vectors < 0) { 2135 dev_err(&adapter->pdev->dev, 2136 "Unable to allocate MSI-X interrupts\n"); 2137 kfree(adapter->msix_entries); 2138 adapter->msix_entries = NULL; 2139 return vectors; 2140 } 2141 2142 /* Adjust for only the vectors we'll use, which is minimum 2143 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 2144 * vectors we were allocated. 2145 */ 2146 adapter->num_msix_vectors = vectors; 2147 2148 return 0; 2149 } 2150 2151 /** 2152 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent 2153 * @adapter: board private structure to initialize 2154 * 2155 * This is the top level queue allocation routine. The order here is very 2156 * important, starting with the "most" number of features turned on at once, 2157 * and ending with the smallest set of features. This way large combinations 2158 * can be allocated if they're turned on, and smaller combinations are the 2159 * fallthrough conditions. 2160 * 2161 **/ 2162 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 2163 { 2164 struct ixgbe_hw *hw = &adapter->hw; 2165 unsigned int def_q = 0; 2166 unsigned int num_tcs = 0; 2167 int err; 2168 2169 /* Start with base case */ 2170 adapter->num_rx_queues = 1; 2171 adapter->num_tx_queues = 1; 2172 2173 spin_lock_bh(&adapter->mbx_lock); 2174 2175 /* fetch queue configuration from the PF */ 2176 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2177 2178 spin_unlock_bh(&adapter->mbx_lock); 2179 2180 if (err) 2181 return; 2182 2183 /* we need as many queues as traffic classes */ 2184 if (num_tcs > 1) 2185 adapter->num_rx_queues = num_tcs; 2186 } 2187 2188 /** 2189 * ixgbevf_alloc_queues - Allocate memory for all rings 2190 * @adapter: board private structure to initialize 2191 * 2192 * We allocate one ring per queue at run-time since we don't know the 2193 * number of queues at compile-time. The polling_netdev array is 2194 * intended for Multiqueue, but should work fine with a single queue. 2195 **/ 2196 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) 2197 { 2198 struct ixgbevf_ring *ring; 2199 int rx = 0, tx = 0; 2200 2201 for (; tx < adapter->num_tx_queues; tx++) { 2202 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 2203 if (!ring) 2204 goto err_allocation; 2205 2206 ring->dev = &adapter->pdev->dev; 2207 ring->netdev = adapter->netdev; 2208 ring->count = adapter->tx_ring_count; 2209 ring->queue_index = tx; 2210 ring->reg_idx = tx; 2211 2212 adapter->tx_ring[tx] = ring; 2213 } 2214 2215 for (; rx < adapter->num_rx_queues; rx++) { 2216 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 2217 if (!ring) 2218 goto err_allocation; 2219 2220 ring->dev = &adapter->pdev->dev; 2221 ring->netdev = adapter->netdev; 2222 2223 ring->count = adapter->rx_ring_count; 2224 ring->queue_index = rx; 2225 ring->reg_idx = rx; 2226 2227 adapter->rx_ring[rx] = ring; 2228 } 2229 2230 return 0; 2231 2232 err_allocation: 2233 while (tx) { 2234 kfree(adapter->tx_ring[--tx]); 2235 adapter->tx_ring[tx] = NULL; 2236 } 2237 2238 while (rx) { 2239 kfree(adapter->rx_ring[--rx]); 2240 adapter->rx_ring[rx] = NULL; 2241 } 2242 return -ENOMEM; 2243 } 2244 2245 /** 2246 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 2247 * @adapter: board private structure to initialize 2248 * 2249 * Attempt to configure the interrupts using the best available 2250 * capabilities of the hardware and the kernel. 2251 **/ 2252 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 2253 { 2254 struct net_device *netdev = adapter->netdev; 2255 int err = 0; 2256 int vector, v_budget; 2257 2258 /* 2259 * It's easy to be greedy for MSI-X vectors, but it really 2260 * doesn't do us much good if we have a lot more vectors 2261 * than CPU's. So let's be conservative and only ask for 2262 * (roughly) the same number of vectors as there are CPU's. 2263 * The default is to use pairs of vectors. 2264 */ 2265 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); 2266 v_budget = min_t(int, v_budget, num_online_cpus()); 2267 v_budget += NON_Q_VECTORS; 2268 2269 /* A failure in MSI-X entry allocation isn't fatal, but it does 2270 * mean we disable MSI-X capabilities of the adapter. */ 2271 adapter->msix_entries = kcalloc(v_budget, 2272 sizeof(struct msix_entry), GFP_KERNEL); 2273 if (!adapter->msix_entries) { 2274 err = -ENOMEM; 2275 goto out; 2276 } 2277 2278 for (vector = 0; vector < v_budget; vector++) 2279 adapter->msix_entries[vector].entry = vector; 2280 2281 err = ixgbevf_acquire_msix_vectors(adapter, v_budget); 2282 if (err) 2283 goto out; 2284 2285 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 2286 if (err) 2287 goto out; 2288 2289 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 2290 2291 out: 2292 return err; 2293 } 2294 2295 /** 2296 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 2297 * @adapter: board private structure to initialize 2298 * 2299 * We allocate one q_vector per queue interrupt. If allocation fails we 2300 * return -ENOMEM. 2301 **/ 2302 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 2303 { 2304 int q_idx, num_q_vectors; 2305 struct ixgbevf_q_vector *q_vector; 2306 2307 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2308 2309 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2310 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL); 2311 if (!q_vector) 2312 goto err_out; 2313 q_vector->adapter = adapter; 2314 q_vector->v_idx = q_idx; 2315 netif_napi_add(adapter->netdev, &q_vector->napi, 2316 ixgbevf_poll, 64); 2317 #ifdef CONFIG_NET_RX_BUSY_POLL 2318 napi_hash_add(&q_vector->napi); 2319 #endif 2320 adapter->q_vector[q_idx] = q_vector; 2321 } 2322 2323 return 0; 2324 2325 err_out: 2326 while (q_idx) { 2327 q_idx--; 2328 q_vector = adapter->q_vector[q_idx]; 2329 #ifdef CONFIG_NET_RX_BUSY_POLL 2330 napi_hash_del(&q_vector->napi); 2331 #endif 2332 netif_napi_del(&q_vector->napi); 2333 kfree(q_vector); 2334 adapter->q_vector[q_idx] = NULL; 2335 } 2336 return -ENOMEM; 2337 } 2338 2339 /** 2340 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 2341 * @adapter: board private structure to initialize 2342 * 2343 * This function frees the memory allocated to the q_vectors. In addition if 2344 * NAPI is enabled it will delete any references to the NAPI struct prior 2345 * to freeing the q_vector. 2346 **/ 2347 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 2348 { 2349 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2350 2351 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2352 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; 2353 2354 adapter->q_vector[q_idx] = NULL; 2355 #ifdef CONFIG_NET_RX_BUSY_POLL 2356 napi_hash_del(&q_vector->napi); 2357 #endif 2358 netif_napi_del(&q_vector->napi); 2359 kfree(q_vector); 2360 } 2361 } 2362 2363 /** 2364 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 2365 * @adapter: board private structure 2366 * 2367 **/ 2368 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 2369 { 2370 pci_disable_msix(adapter->pdev); 2371 kfree(adapter->msix_entries); 2372 adapter->msix_entries = NULL; 2373 } 2374 2375 /** 2376 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 2377 * @adapter: board private structure to initialize 2378 * 2379 **/ 2380 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 2381 { 2382 int err; 2383 2384 /* Number of supported queues */ 2385 ixgbevf_set_num_queues(adapter); 2386 2387 err = ixgbevf_set_interrupt_capability(adapter); 2388 if (err) { 2389 hw_dbg(&adapter->hw, 2390 "Unable to setup interrupt capabilities\n"); 2391 goto err_set_interrupt; 2392 } 2393 2394 err = ixgbevf_alloc_q_vectors(adapter); 2395 if (err) { 2396 hw_dbg(&adapter->hw, "Unable to allocate memory for queue " 2397 "vectors\n"); 2398 goto err_alloc_q_vectors; 2399 } 2400 2401 err = ixgbevf_alloc_queues(adapter); 2402 if (err) { 2403 pr_err("Unable to allocate memory for queues\n"); 2404 goto err_alloc_queues; 2405 } 2406 2407 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, " 2408 "Tx Queue count = %u\n", 2409 (adapter->num_rx_queues > 1) ? "Enabled" : 2410 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues); 2411 2412 set_bit(__IXGBEVF_DOWN, &adapter->state); 2413 2414 return 0; 2415 err_alloc_queues: 2416 ixgbevf_free_q_vectors(adapter); 2417 err_alloc_q_vectors: 2418 ixgbevf_reset_interrupt_capability(adapter); 2419 err_set_interrupt: 2420 return err; 2421 } 2422 2423 /** 2424 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings 2425 * @adapter: board private structure to clear interrupt scheme on 2426 * 2427 * We go through and clear interrupt specific resources and reset the structure 2428 * to pre-load conditions 2429 **/ 2430 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) 2431 { 2432 int i; 2433 2434 for (i = 0; i < adapter->num_tx_queues; i++) { 2435 kfree(adapter->tx_ring[i]); 2436 adapter->tx_ring[i] = NULL; 2437 } 2438 for (i = 0; i < adapter->num_rx_queues; i++) { 2439 kfree(adapter->rx_ring[i]); 2440 adapter->rx_ring[i] = NULL; 2441 } 2442 2443 adapter->num_tx_queues = 0; 2444 adapter->num_rx_queues = 0; 2445 2446 ixgbevf_free_q_vectors(adapter); 2447 ixgbevf_reset_interrupt_capability(adapter); 2448 } 2449 2450 /** 2451 * ixgbevf_sw_init - Initialize general software structures 2452 * (struct ixgbevf_adapter) 2453 * @adapter: board private structure to initialize 2454 * 2455 * ixgbevf_sw_init initializes the Adapter private data structure. 2456 * Fields are initialized based on PCI device information and 2457 * OS network device settings (MTU size). 2458 **/ 2459 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 2460 { 2461 struct ixgbe_hw *hw = &adapter->hw; 2462 struct pci_dev *pdev = adapter->pdev; 2463 struct net_device *netdev = adapter->netdev; 2464 int err; 2465 2466 /* PCI config space info */ 2467 2468 hw->vendor_id = pdev->vendor; 2469 hw->device_id = pdev->device; 2470 hw->revision_id = pdev->revision; 2471 hw->subsystem_vendor_id = pdev->subsystem_vendor; 2472 hw->subsystem_device_id = pdev->subsystem_device; 2473 2474 hw->mbx.ops.init_params(hw); 2475 2476 /* assume legacy case in which PF would only give VF 2 queues */ 2477 hw->mac.max_tx_queues = 2; 2478 hw->mac.max_rx_queues = 2; 2479 2480 /* lock to protect mailbox accesses */ 2481 spin_lock_init(&adapter->mbx_lock); 2482 2483 err = hw->mac.ops.reset_hw(hw); 2484 if (err) { 2485 dev_info(&pdev->dev, 2486 "PF still in reset state. Is the PF interface up?\n"); 2487 } else { 2488 err = hw->mac.ops.init_hw(hw); 2489 if (err) { 2490 pr_err("init_shared_code failed: %d\n", err); 2491 goto out; 2492 } 2493 ixgbevf_negotiate_api(adapter); 2494 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr); 2495 if (err) 2496 dev_info(&pdev->dev, "Error reading MAC address\n"); 2497 else if (is_zero_ether_addr(adapter->hw.mac.addr)) 2498 dev_info(&pdev->dev, 2499 "MAC address not assigned by administrator.\n"); 2500 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 2501 } 2502 2503 if (!is_valid_ether_addr(netdev->dev_addr)) { 2504 dev_info(&pdev->dev, "Assigning random MAC address\n"); 2505 eth_hw_addr_random(netdev); 2506 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len); 2507 } 2508 2509 /* Enable dynamic interrupt throttling rates */ 2510 adapter->rx_itr_setting = 1; 2511 adapter->tx_itr_setting = 1; 2512 2513 /* set default ring sizes */ 2514 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 2515 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 2516 2517 set_bit(__IXGBEVF_DOWN, &adapter->state); 2518 return 0; 2519 2520 out: 2521 return err; 2522 } 2523 2524 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 2525 { \ 2526 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 2527 if (current_counter < last_counter) \ 2528 counter += 0x100000000LL; \ 2529 last_counter = current_counter; \ 2530 counter &= 0xFFFFFFFF00000000LL; \ 2531 counter |= current_counter; \ 2532 } 2533 2534 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 2535 { \ 2536 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 2537 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 2538 u64 current_counter = (current_counter_msb << 32) | \ 2539 current_counter_lsb; \ 2540 if (current_counter < last_counter) \ 2541 counter += 0x1000000000LL; \ 2542 last_counter = current_counter; \ 2543 counter &= 0xFFFFFFF000000000LL; \ 2544 counter |= current_counter; \ 2545 } 2546 /** 2547 * ixgbevf_update_stats - Update the board statistics counters. 2548 * @adapter: board private structure 2549 **/ 2550 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 2551 { 2552 struct ixgbe_hw *hw = &adapter->hw; 2553 int i; 2554 2555 if (!adapter->link_up) 2556 return; 2557 2558 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 2559 adapter->stats.vfgprc); 2560 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 2561 adapter->stats.vfgptc); 2562 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2563 adapter->stats.last_vfgorc, 2564 adapter->stats.vfgorc); 2565 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2566 adapter->stats.last_vfgotc, 2567 adapter->stats.vfgotc); 2568 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 2569 adapter->stats.vfmprc); 2570 2571 for (i = 0; i < adapter->num_rx_queues; i++) { 2572 adapter->hw_csum_rx_error += 2573 adapter->rx_ring[i]->hw_csum_rx_error; 2574 adapter->rx_ring[i]->hw_csum_rx_error = 0; 2575 } 2576 } 2577 2578 /** 2579 * ixgbevf_watchdog - Timer Call-back 2580 * @data: pointer to adapter cast into an unsigned long 2581 **/ 2582 static void ixgbevf_watchdog(unsigned long data) 2583 { 2584 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data; 2585 struct ixgbe_hw *hw = &adapter->hw; 2586 u32 eics = 0; 2587 int i; 2588 2589 /* 2590 * Do the watchdog outside of interrupt context due to the lovely 2591 * delays that some of the newer hardware requires 2592 */ 2593 2594 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 2595 goto watchdog_short_circuit; 2596 2597 /* get one bit for every active tx/rx interrupt vector */ 2598 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 2599 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 2600 if (qv->rx.ring || qv->tx.ring) 2601 eics |= 1 << i; 2602 } 2603 2604 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics); 2605 2606 watchdog_short_circuit: 2607 schedule_work(&adapter->watchdog_task); 2608 } 2609 2610 /** 2611 * ixgbevf_tx_timeout - Respond to a Tx Hang 2612 * @netdev: network interface device structure 2613 **/ 2614 static void ixgbevf_tx_timeout(struct net_device *netdev) 2615 { 2616 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2617 2618 /* Do the reset outside of interrupt context */ 2619 schedule_work(&adapter->reset_task); 2620 } 2621 2622 static void ixgbevf_reset_task(struct work_struct *work) 2623 { 2624 struct ixgbevf_adapter *adapter; 2625 adapter = container_of(work, struct ixgbevf_adapter, reset_task); 2626 2627 /* If we're already down or resetting, just bail */ 2628 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2629 test_bit(__IXGBEVF_REMOVING, &adapter->state) || 2630 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2631 return; 2632 2633 adapter->tx_timeout_count++; 2634 2635 ixgbevf_reinit_locked(adapter); 2636 } 2637 2638 /** 2639 * ixgbevf_watchdog_task - worker thread to bring link up 2640 * @work: pointer to work_struct containing our data 2641 **/ 2642 static void ixgbevf_watchdog_task(struct work_struct *work) 2643 { 2644 struct ixgbevf_adapter *adapter = container_of(work, 2645 struct ixgbevf_adapter, 2646 watchdog_task); 2647 struct net_device *netdev = adapter->netdev; 2648 struct ixgbe_hw *hw = &adapter->hw; 2649 u32 link_speed = adapter->link_speed; 2650 bool link_up = adapter->link_up; 2651 s32 need_reset; 2652 2653 if (IXGBE_REMOVED(hw->hw_addr)) { 2654 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 2655 rtnl_lock(); 2656 ixgbevf_down(adapter); 2657 rtnl_unlock(); 2658 } 2659 return; 2660 } 2661 ixgbevf_queue_reset_subtask(adapter); 2662 2663 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK; 2664 2665 /* 2666 * Always check the link on the watchdog because we have 2667 * no LSC interrupt 2668 */ 2669 spin_lock_bh(&adapter->mbx_lock); 2670 2671 need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 2672 2673 spin_unlock_bh(&adapter->mbx_lock); 2674 2675 if (need_reset) { 2676 adapter->link_up = link_up; 2677 adapter->link_speed = link_speed; 2678 netif_carrier_off(netdev); 2679 netif_tx_stop_all_queues(netdev); 2680 schedule_work(&adapter->reset_task); 2681 goto pf_has_reset; 2682 } 2683 adapter->link_up = link_up; 2684 adapter->link_speed = link_speed; 2685 2686 if (link_up) { 2687 if (!netif_carrier_ok(netdev)) { 2688 char *link_speed_string; 2689 switch (link_speed) { 2690 case IXGBE_LINK_SPEED_10GB_FULL: 2691 link_speed_string = "10 Gbps"; 2692 break; 2693 case IXGBE_LINK_SPEED_1GB_FULL: 2694 link_speed_string = "1 Gbps"; 2695 break; 2696 case IXGBE_LINK_SPEED_100_FULL: 2697 link_speed_string = "100 Mbps"; 2698 break; 2699 default: 2700 link_speed_string = "unknown speed"; 2701 break; 2702 } 2703 dev_info(&adapter->pdev->dev, 2704 "NIC Link is Up, %s\n", link_speed_string); 2705 netif_carrier_on(netdev); 2706 netif_tx_wake_all_queues(netdev); 2707 } 2708 } else { 2709 adapter->link_up = false; 2710 adapter->link_speed = 0; 2711 if (netif_carrier_ok(netdev)) { 2712 dev_info(&adapter->pdev->dev, "NIC Link is Down\n"); 2713 netif_carrier_off(netdev); 2714 netif_tx_stop_all_queues(netdev); 2715 } 2716 } 2717 2718 ixgbevf_update_stats(adapter); 2719 2720 pf_has_reset: 2721 /* Reset the timer */ 2722 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 2723 !test_bit(__IXGBEVF_REMOVING, &adapter->state)) 2724 mod_timer(&adapter->watchdog_timer, 2725 round_jiffies(jiffies + (2 * HZ))); 2726 2727 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK; 2728 } 2729 2730 /** 2731 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 2732 * @tx_ring: Tx descriptor ring for a specific queue 2733 * 2734 * Free all transmit software resources 2735 **/ 2736 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) 2737 { 2738 ixgbevf_clean_tx_ring(tx_ring); 2739 2740 vfree(tx_ring->tx_buffer_info); 2741 tx_ring->tx_buffer_info = NULL; 2742 2743 /* if not set, then don't free */ 2744 if (!tx_ring->desc) 2745 return; 2746 2747 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, 2748 tx_ring->dma); 2749 2750 tx_ring->desc = NULL; 2751 } 2752 2753 /** 2754 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 2755 * @adapter: board private structure 2756 * 2757 * Free all transmit software resources 2758 **/ 2759 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 2760 { 2761 int i; 2762 2763 for (i = 0; i < adapter->num_tx_queues; i++) 2764 if (adapter->tx_ring[i]->desc) 2765 ixgbevf_free_tx_resources(adapter->tx_ring[i]); 2766 } 2767 2768 /** 2769 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 2770 * @tx_ring: tx descriptor ring (for a specific queue) to setup 2771 * 2772 * Return 0 on success, negative on failure 2773 **/ 2774 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) 2775 { 2776 int size; 2777 2778 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2779 tx_ring->tx_buffer_info = vzalloc(size); 2780 if (!tx_ring->tx_buffer_info) 2781 goto err; 2782 2783 /* round up to nearest 4K */ 2784 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 2785 tx_ring->size = ALIGN(tx_ring->size, 4096); 2786 2787 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, 2788 &tx_ring->dma, GFP_KERNEL); 2789 if (!tx_ring->desc) 2790 goto err; 2791 2792 return 0; 2793 2794 err: 2795 vfree(tx_ring->tx_buffer_info); 2796 tx_ring->tx_buffer_info = NULL; 2797 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit " 2798 "descriptor ring\n"); 2799 return -ENOMEM; 2800 } 2801 2802 /** 2803 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 2804 * @adapter: board private structure 2805 * 2806 * If this function returns with an error, then it's possible one or 2807 * more of the rings is populated (while the rest are not). It is the 2808 * callers duty to clean those orphaned rings. 2809 * 2810 * Return 0 on success, negative on failure 2811 **/ 2812 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 2813 { 2814 int i, err = 0; 2815 2816 for (i = 0; i < adapter->num_tx_queues; i++) { 2817 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]); 2818 if (!err) 2819 continue; 2820 hw_dbg(&adapter->hw, 2821 "Allocation for Tx Queue %u failed\n", i); 2822 break; 2823 } 2824 2825 return err; 2826 } 2827 2828 /** 2829 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 2830 * @rx_ring: rx descriptor ring (for a specific queue) to setup 2831 * 2832 * Returns 0 on success, negative on failure 2833 **/ 2834 int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring) 2835 { 2836 int size; 2837 2838 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 2839 rx_ring->rx_buffer_info = vzalloc(size); 2840 if (!rx_ring->rx_buffer_info) 2841 goto err; 2842 2843 /* Round up to nearest 4K */ 2844 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 2845 rx_ring->size = ALIGN(rx_ring->size, 4096); 2846 2847 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, 2848 &rx_ring->dma, GFP_KERNEL); 2849 2850 if (!rx_ring->desc) 2851 goto err; 2852 2853 return 0; 2854 err: 2855 vfree(rx_ring->rx_buffer_info); 2856 rx_ring->rx_buffer_info = NULL; 2857 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); 2858 return -ENOMEM; 2859 } 2860 2861 /** 2862 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 2863 * @adapter: board private structure 2864 * 2865 * If this function returns with an error, then it's possible one or 2866 * more of the rings is populated (while the rest are not). It is the 2867 * callers duty to clean those orphaned rings. 2868 * 2869 * Return 0 on success, negative on failure 2870 **/ 2871 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 2872 { 2873 int i, err = 0; 2874 2875 for (i = 0; i < adapter->num_rx_queues; i++) { 2876 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]); 2877 if (!err) 2878 continue; 2879 hw_dbg(&adapter->hw, 2880 "Allocation for Rx Queue %u failed\n", i); 2881 break; 2882 } 2883 return err; 2884 } 2885 2886 /** 2887 * ixgbevf_free_rx_resources - Free Rx Resources 2888 * @rx_ring: ring to clean the resources from 2889 * 2890 * Free all receive software resources 2891 **/ 2892 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) 2893 { 2894 ixgbevf_clean_rx_ring(rx_ring); 2895 2896 vfree(rx_ring->rx_buffer_info); 2897 rx_ring->rx_buffer_info = NULL; 2898 2899 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, 2900 rx_ring->dma); 2901 2902 rx_ring->desc = NULL; 2903 } 2904 2905 /** 2906 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 2907 * @adapter: board private structure 2908 * 2909 * Free all receive software resources 2910 **/ 2911 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 2912 { 2913 int i; 2914 2915 for (i = 0; i < adapter->num_rx_queues; i++) 2916 if (adapter->rx_ring[i]->desc) 2917 ixgbevf_free_rx_resources(adapter->rx_ring[i]); 2918 } 2919 2920 /** 2921 * ixgbevf_open - Called when a network interface is made active 2922 * @netdev: network interface device structure 2923 * 2924 * Returns 0 on success, negative value on failure 2925 * 2926 * The open entry point is called when a network interface is made 2927 * active by the system (IFF_UP). At this point all resources needed 2928 * for transmit and receive operations are allocated, the interrupt 2929 * handler is registered with the OS, the watchdog timer is started, 2930 * and the stack is notified that the interface is ready. 2931 **/ 2932 static int ixgbevf_open(struct net_device *netdev) 2933 { 2934 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2935 struct ixgbe_hw *hw = &adapter->hw; 2936 int err; 2937 2938 /* A previous failure to open the device because of a lack of 2939 * available MSIX vector resources may have reset the number 2940 * of msix vectors variable to zero. The only way to recover 2941 * is to unload/reload the driver and hope that the system has 2942 * been able to recover some MSIX vector resources. 2943 */ 2944 if (!adapter->num_msix_vectors) 2945 return -ENOMEM; 2946 2947 /* disallow open during test */ 2948 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 2949 return -EBUSY; 2950 2951 if (hw->adapter_stopped) { 2952 ixgbevf_reset(adapter); 2953 /* if adapter is still stopped then PF isn't up and 2954 * the vf can't start. */ 2955 if (hw->adapter_stopped) { 2956 err = IXGBE_ERR_MBX; 2957 pr_err("Unable to start - perhaps the PF Driver isn't " 2958 "up yet\n"); 2959 goto err_setup_reset; 2960 } 2961 } 2962 2963 /* allocate transmit descriptors */ 2964 err = ixgbevf_setup_all_tx_resources(adapter); 2965 if (err) 2966 goto err_setup_tx; 2967 2968 /* allocate receive descriptors */ 2969 err = ixgbevf_setup_all_rx_resources(adapter); 2970 if (err) 2971 goto err_setup_rx; 2972 2973 ixgbevf_configure(adapter); 2974 2975 /* 2976 * Map the Tx/Rx rings to the vectors we were allotted. 2977 * if request_irq will be called in this function map_rings 2978 * must be called *before* up_complete 2979 */ 2980 ixgbevf_map_rings_to_vectors(adapter); 2981 2982 ixgbevf_up_complete(adapter); 2983 2984 /* clear any pending interrupts, may auto mask */ 2985 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2986 err = ixgbevf_request_irq(adapter); 2987 if (err) 2988 goto err_req_irq; 2989 2990 ixgbevf_irq_enable(adapter); 2991 2992 return 0; 2993 2994 err_req_irq: 2995 ixgbevf_down(adapter); 2996 err_setup_rx: 2997 ixgbevf_free_all_rx_resources(adapter); 2998 err_setup_tx: 2999 ixgbevf_free_all_tx_resources(adapter); 3000 ixgbevf_reset(adapter); 3001 3002 err_setup_reset: 3003 3004 return err; 3005 } 3006 3007 /** 3008 * ixgbevf_close - Disables a network interface 3009 * @netdev: network interface device structure 3010 * 3011 * Returns 0, this is not allowed to fail 3012 * 3013 * The close entry point is called when an interface is de-activated 3014 * by the OS. The hardware is still under the drivers control, but 3015 * needs to be disabled. A global MAC reset is issued to stop the 3016 * hardware, and all transmit and receive resources are freed. 3017 **/ 3018 static int ixgbevf_close(struct net_device *netdev) 3019 { 3020 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3021 3022 ixgbevf_down(adapter); 3023 ixgbevf_free_irq(adapter); 3024 3025 ixgbevf_free_all_tx_resources(adapter); 3026 ixgbevf_free_all_rx_resources(adapter); 3027 3028 return 0; 3029 } 3030 3031 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter) 3032 { 3033 struct net_device *dev = adapter->netdev; 3034 3035 if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED)) 3036 return; 3037 3038 adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED; 3039 3040 /* if interface is down do nothing */ 3041 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3042 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3043 return; 3044 3045 /* Hardware has to reinitialize queues and interrupts to 3046 * match packet buffer alignment. Unfortunately, the 3047 * hardware is not flexible enough to do this dynamically. 3048 */ 3049 if (netif_running(dev)) 3050 ixgbevf_close(dev); 3051 3052 ixgbevf_clear_interrupt_scheme(adapter); 3053 ixgbevf_init_interrupt_scheme(adapter); 3054 3055 if (netif_running(dev)) 3056 ixgbevf_open(dev); 3057 } 3058 3059 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, 3060 u32 vlan_macip_lens, u32 type_tucmd, 3061 u32 mss_l4len_idx) 3062 { 3063 struct ixgbe_adv_tx_context_desc *context_desc; 3064 u16 i = tx_ring->next_to_use; 3065 3066 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i); 3067 3068 i++; 3069 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 3070 3071 /* set bits to identify this as an advanced context descriptor */ 3072 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3073 3074 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 3075 context_desc->seqnum_seed = 0; 3076 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 3077 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 3078 } 3079 3080 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, 3081 struct ixgbevf_tx_buffer *first, 3082 u8 *hdr_len) 3083 { 3084 struct sk_buff *skb = first->skb; 3085 u32 vlan_macip_lens, type_tucmd; 3086 u32 mss_l4len_idx, l4len; 3087 int err; 3088 3089 if (skb->ip_summed != CHECKSUM_PARTIAL) 3090 return 0; 3091 3092 if (!skb_is_gso(skb)) 3093 return 0; 3094 3095 err = skb_cow_head(skb, 0); 3096 if (err < 0) 3097 return err; 3098 3099 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 3100 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3101 3102 if (skb->protocol == htons(ETH_P_IP)) { 3103 struct iphdr *iph = ip_hdr(skb); 3104 iph->tot_len = 0; 3105 iph->check = 0; 3106 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 3107 iph->daddr, 0, 3108 IPPROTO_TCP, 3109 0); 3110 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3111 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3112 IXGBE_TX_FLAGS_CSUM | 3113 IXGBE_TX_FLAGS_IPV4; 3114 } else if (skb_is_gso_v6(skb)) { 3115 ipv6_hdr(skb)->payload_len = 0; 3116 tcp_hdr(skb)->check = 3117 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 3118 &ipv6_hdr(skb)->daddr, 3119 0, IPPROTO_TCP, 0); 3120 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3121 IXGBE_TX_FLAGS_CSUM; 3122 } 3123 3124 /* compute header lengths */ 3125 l4len = tcp_hdrlen(skb); 3126 *hdr_len += l4len; 3127 *hdr_len = skb_transport_offset(skb) + l4len; 3128 3129 /* update gso size and bytecount with header size */ 3130 first->gso_segs = skb_shinfo(skb)->gso_segs; 3131 first->bytecount += (first->gso_segs - 1) * *hdr_len; 3132 3133 /* mss_l4len_id: use 1 as index for TSO */ 3134 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; 3135 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 3136 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT; 3137 3138 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 3139 vlan_macip_lens = skb_network_header_len(skb); 3140 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3141 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3142 3143 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3144 type_tucmd, mss_l4len_idx); 3145 3146 return 1; 3147 } 3148 3149 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, 3150 struct ixgbevf_tx_buffer *first) 3151 { 3152 struct sk_buff *skb = first->skb; 3153 u32 vlan_macip_lens = 0; 3154 u32 mss_l4len_idx = 0; 3155 u32 type_tucmd = 0; 3156 3157 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3158 u8 l4_hdr = 0; 3159 switch (skb->protocol) { 3160 case htons(ETH_P_IP): 3161 vlan_macip_lens |= skb_network_header_len(skb); 3162 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3163 l4_hdr = ip_hdr(skb)->protocol; 3164 break; 3165 case htons(ETH_P_IPV6): 3166 vlan_macip_lens |= skb_network_header_len(skb); 3167 l4_hdr = ipv6_hdr(skb)->nexthdr; 3168 break; 3169 default: 3170 if (unlikely(net_ratelimit())) { 3171 dev_warn(tx_ring->dev, 3172 "partial checksum but proto=%x!\n", 3173 first->protocol); 3174 } 3175 break; 3176 } 3177 3178 switch (l4_hdr) { 3179 case IPPROTO_TCP: 3180 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3181 mss_l4len_idx = tcp_hdrlen(skb) << 3182 IXGBE_ADVTXD_L4LEN_SHIFT; 3183 break; 3184 case IPPROTO_SCTP: 3185 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 3186 mss_l4len_idx = sizeof(struct sctphdr) << 3187 IXGBE_ADVTXD_L4LEN_SHIFT; 3188 break; 3189 case IPPROTO_UDP: 3190 mss_l4len_idx = sizeof(struct udphdr) << 3191 IXGBE_ADVTXD_L4LEN_SHIFT; 3192 break; 3193 default: 3194 if (unlikely(net_ratelimit())) { 3195 dev_warn(tx_ring->dev, 3196 "partial checksum but l4 proto=%x!\n", 3197 l4_hdr); 3198 } 3199 break; 3200 } 3201 3202 /* update TX checksum flag */ 3203 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 3204 } 3205 3206 /* vlan_macip_lens: MACLEN, VLAN tag */ 3207 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3208 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3209 3210 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3211 type_tucmd, mss_l4len_idx); 3212 } 3213 3214 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags) 3215 { 3216 /* set type for advanced descriptor with frame checksum insertion */ 3217 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | 3218 IXGBE_ADVTXD_DCMD_IFCS | 3219 IXGBE_ADVTXD_DCMD_DEXT); 3220 3221 /* set HW vlan bit if vlan is present */ 3222 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 3223 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); 3224 3225 /* set segmentation enable bits for TSO/FSO */ 3226 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3227 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); 3228 3229 return cmd_type; 3230 } 3231 3232 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 3233 u32 tx_flags, unsigned int paylen) 3234 { 3235 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT); 3236 3237 /* enable L4 checksum for TSO and TX checksum offload */ 3238 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 3239 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM); 3240 3241 /* enble IPv4 checksum for TSO */ 3242 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 3243 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM); 3244 3245 /* use index 1 context for TSO/FSO/FCOE */ 3246 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3247 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT); 3248 3249 /* Check Context must be set if Tx switch is enabled, which it 3250 * always is for case where virtual functions are running 3251 */ 3252 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC); 3253 3254 tx_desc->read.olinfo_status = olinfo_status; 3255 } 3256 3257 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, 3258 struct ixgbevf_tx_buffer *first, 3259 const u8 hdr_len) 3260 { 3261 dma_addr_t dma; 3262 struct sk_buff *skb = first->skb; 3263 struct ixgbevf_tx_buffer *tx_buffer; 3264 union ixgbe_adv_tx_desc *tx_desc; 3265 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 3266 unsigned int data_len = skb->data_len; 3267 unsigned int size = skb_headlen(skb); 3268 unsigned int paylen = skb->len - hdr_len; 3269 u32 tx_flags = first->tx_flags; 3270 __le32 cmd_type; 3271 u16 i = tx_ring->next_to_use; 3272 3273 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 3274 3275 ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen); 3276 cmd_type = ixgbevf_tx_cmd_type(tx_flags); 3277 3278 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 3279 if (dma_mapping_error(tx_ring->dev, dma)) 3280 goto dma_error; 3281 3282 /* record length, and DMA address */ 3283 dma_unmap_len_set(first, len, size); 3284 dma_unmap_addr_set(first, dma, dma); 3285 3286 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3287 3288 for (;;) { 3289 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 3290 tx_desc->read.cmd_type_len = 3291 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); 3292 3293 i++; 3294 tx_desc++; 3295 if (i == tx_ring->count) { 3296 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3297 i = 0; 3298 } 3299 3300 dma += IXGBE_MAX_DATA_PER_TXD; 3301 size -= IXGBE_MAX_DATA_PER_TXD; 3302 3303 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3304 tx_desc->read.olinfo_status = 0; 3305 } 3306 3307 if (likely(!data_len)) 3308 break; 3309 3310 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); 3311 3312 i++; 3313 tx_desc++; 3314 if (i == tx_ring->count) { 3315 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3316 i = 0; 3317 } 3318 3319 size = skb_frag_size(frag); 3320 data_len -= size; 3321 3322 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 3323 DMA_TO_DEVICE); 3324 if (dma_mapping_error(tx_ring->dev, dma)) 3325 goto dma_error; 3326 3327 tx_buffer = &tx_ring->tx_buffer_info[i]; 3328 dma_unmap_len_set(tx_buffer, len, size); 3329 dma_unmap_addr_set(tx_buffer, dma, dma); 3330 3331 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3332 tx_desc->read.olinfo_status = 0; 3333 3334 frag++; 3335 } 3336 3337 /* write last descriptor with RS and EOP bits */ 3338 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); 3339 tx_desc->read.cmd_type_len = cmd_type; 3340 3341 /* set the timestamp */ 3342 first->time_stamp = jiffies; 3343 3344 /* Force memory writes to complete before letting h/w know there 3345 * are new descriptors to fetch. (Only applicable for weak-ordered 3346 * memory model archs, such as IA-64). 3347 * 3348 * We also need this memory barrier (wmb) to make certain all of the 3349 * status bits have been updated before next_to_watch is written. 3350 */ 3351 wmb(); 3352 3353 /* set next_to_watch value indicating a packet is present */ 3354 first->next_to_watch = tx_desc; 3355 3356 i++; 3357 if (i == tx_ring->count) 3358 i = 0; 3359 3360 tx_ring->next_to_use = i; 3361 3362 /* notify HW of packet */ 3363 ixgbevf_write_tail(tx_ring, i); 3364 3365 return; 3366 dma_error: 3367 dev_err(tx_ring->dev, "TX DMA map failed\n"); 3368 3369 /* clear dma mappings for failed tx_buffer_info map */ 3370 for (;;) { 3371 tx_buffer = &tx_ring->tx_buffer_info[i]; 3372 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer); 3373 if (tx_buffer == first) 3374 break; 3375 if (i == 0) 3376 i = tx_ring->count; 3377 i--; 3378 } 3379 3380 tx_ring->next_to_use = i; 3381 } 3382 3383 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 3384 { 3385 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 3386 /* Herbert's original patch had: 3387 * smp_mb__after_netif_stop_queue(); 3388 * but since that doesn't exist yet, just open code it. */ 3389 smp_mb(); 3390 3391 /* We need to check again in a case another CPU has just 3392 * made room available. */ 3393 if (likely(ixgbevf_desc_unused(tx_ring) < size)) 3394 return -EBUSY; 3395 3396 /* A reprieve! - use start_queue because it doesn't call schedule */ 3397 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 3398 ++tx_ring->tx_stats.restart_queue; 3399 3400 return 0; 3401 } 3402 3403 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 3404 { 3405 if (likely(ixgbevf_desc_unused(tx_ring) >= size)) 3406 return 0; 3407 return __ixgbevf_maybe_stop_tx(tx_ring, size); 3408 } 3409 3410 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 3411 { 3412 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3413 struct ixgbevf_tx_buffer *first; 3414 struct ixgbevf_ring *tx_ring; 3415 int tso; 3416 u32 tx_flags = 0; 3417 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 3418 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3419 unsigned short f; 3420 #endif 3421 u8 hdr_len = 0; 3422 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL); 3423 3424 if (!dst_mac || is_link_local_ether_addr(dst_mac)) { 3425 dev_kfree_skb(skb); 3426 return NETDEV_TX_OK; 3427 } 3428 3429 tx_ring = adapter->tx_ring[skb->queue_mapping]; 3430 3431 /* 3432 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 3433 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 3434 * + 2 desc gap to keep tail from touching head, 3435 * + 1 desc for context descriptor, 3436 * otherwise try next time 3437 */ 3438 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3439 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 3440 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 3441 #else 3442 count += skb_shinfo(skb)->nr_frags; 3443 #endif 3444 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { 3445 tx_ring->tx_stats.tx_busy++; 3446 return NETDEV_TX_BUSY; 3447 } 3448 3449 /* record the location of the first descriptor for this packet */ 3450 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 3451 first->skb = skb; 3452 first->bytecount = skb->len; 3453 first->gso_segs = 1; 3454 3455 if (vlan_tx_tag_present(skb)) { 3456 tx_flags |= vlan_tx_tag_get(skb); 3457 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 3458 tx_flags |= IXGBE_TX_FLAGS_VLAN; 3459 } 3460 3461 /* record initial flags and protocol */ 3462 first->tx_flags = tx_flags; 3463 first->protocol = vlan_get_protocol(skb); 3464 3465 tso = ixgbevf_tso(tx_ring, first, &hdr_len); 3466 if (tso < 0) 3467 goto out_drop; 3468 else if (!tso) 3469 ixgbevf_tx_csum(tx_ring, first); 3470 3471 ixgbevf_tx_map(tx_ring, first, hdr_len); 3472 3473 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); 3474 3475 return NETDEV_TX_OK; 3476 3477 out_drop: 3478 dev_kfree_skb_any(first->skb); 3479 first->skb = NULL; 3480 3481 return NETDEV_TX_OK; 3482 } 3483 3484 /** 3485 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 3486 * @netdev: network interface device structure 3487 * @p: pointer to an address structure 3488 * 3489 * Returns 0 on success, negative on failure 3490 **/ 3491 static int ixgbevf_set_mac(struct net_device *netdev, void *p) 3492 { 3493 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3494 struct ixgbe_hw *hw = &adapter->hw; 3495 struct sockaddr *addr = p; 3496 3497 if (!is_valid_ether_addr(addr->sa_data)) 3498 return -EADDRNOTAVAIL; 3499 3500 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 3501 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 3502 3503 spin_lock_bh(&adapter->mbx_lock); 3504 3505 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 3506 3507 spin_unlock_bh(&adapter->mbx_lock); 3508 3509 return 0; 3510 } 3511 3512 /** 3513 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 3514 * @netdev: network interface device structure 3515 * @new_mtu: new value for maximum frame size 3516 * 3517 * Returns 0 on success, negative on failure 3518 **/ 3519 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 3520 { 3521 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3522 struct ixgbe_hw *hw = &adapter->hw; 3523 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 3524 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; 3525 3526 switch (adapter->hw.api_version) { 3527 case ixgbe_mbox_api_11: 3528 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3529 break; 3530 default: 3531 if (adapter->hw.mac.type != ixgbe_mac_82599_vf) 3532 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3533 break; 3534 } 3535 3536 /* MTU < 68 is an error and causes problems on some kernels */ 3537 if ((new_mtu < 68) || (max_frame > max_possible_frame)) 3538 return -EINVAL; 3539 3540 hw_dbg(hw, "changing MTU from %d to %d\n", 3541 netdev->mtu, new_mtu); 3542 /* must set new MTU before calling down or up */ 3543 netdev->mtu = new_mtu; 3544 3545 /* notify the PF of our intent to use this size of frame */ 3546 ixgbevf_rlpml_set_vf(hw, max_frame); 3547 3548 return 0; 3549 } 3550 3551 #ifdef CONFIG_NET_POLL_CONTROLLER 3552 /* Polling 'interrupt' - used by things like netconsole to send skbs 3553 * without having to re-enable interrupts. It's not called while 3554 * the interrupt routine is executing. 3555 */ 3556 static void ixgbevf_netpoll(struct net_device *netdev) 3557 { 3558 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3559 int i; 3560 3561 /* if interface is down do nothing */ 3562 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 3563 return; 3564 for (i = 0; i < adapter->num_rx_queues; i++) 3565 ixgbevf_msix_clean_rings(0, adapter->q_vector[i]); 3566 } 3567 #endif /* CONFIG_NET_POLL_CONTROLLER */ 3568 3569 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) 3570 { 3571 struct net_device *netdev = pci_get_drvdata(pdev); 3572 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3573 #ifdef CONFIG_PM 3574 int retval = 0; 3575 #endif 3576 3577 netif_device_detach(netdev); 3578 3579 if (netif_running(netdev)) { 3580 rtnl_lock(); 3581 ixgbevf_down(adapter); 3582 ixgbevf_free_irq(adapter); 3583 ixgbevf_free_all_tx_resources(adapter); 3584 ixgbevf_free_all_rx_resources(adapter); 3585 rtnl_unlock(); 3586 } 3587 3588 ixgbevf_clear_interrupt_scheme(adapter); 3589 3590 #ifdef CONFIG_PM 3591 retval = pci_save_state(pdev); 3592 if (retval) 3593 return retval; 3594 3595 #endif 3596 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 3597 pci_disable_device(pdev); 3598 3599 return 0; 3600 } 3601 3602 #ifdef CONFIG_PM 3603 static int ixgbevf_resume(struct pci_dev *pdev) 3604 { 3605 struct net_device *netdev = pci_get_drvdata(pdev); 3606 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3607 u32 err; 3608 3609 pci_restore_state(pdev); 3610 /* 3611 * pci_restore_state clears dev->state_saved so call 3612 * pci_save_state to restore it. 3613 */ 3614 pci_save_state(pdev); 3615 3616 err = pci_enable_device_mem(pdev); 3617 if (err) { 3618 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 3619 return err; 3620 } 3621 smp_mb__before_atomic(); 3622 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 3623 pci_set_master(pdev); 3624 3625 ixgbevf_reset(adapter); 3626 3627 rtnl_lock(); 3628 err = ixgbevf_init_interrupt_scheme(adapter); 3629 rtnl_unlock(); 3630 if (err) { 3631 dev_err(&pdev->dev, "Cannot initialize interrupts\n"); 3632 return err; 3633 } 3634 3635 if (netif_running(netdev)) { 3636 err = ixgbevf_open(netdev); 3637 if (err) 3638 return err; 3639 } 3640 3641 netif_device_attach(netdev); 3642 3643 return err; 3644 } 3645 3646 #endif /* CONFIG_PM */ 3647 static void ixgbevf_shutdown(struct pci_dev *pdev) 3648 { 3649 ixgbevf_suspend(pdev, PMSG_SUSPEND); 3650 } 3651 3652 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, 3653 struct rtnl_link_stats64 *stats) 3654 { 3655 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3656 unsigned int start; 3657 u64 bytes, packets; 3658 const struct ixgbevf_ring *ring; 3659 int i; 3660 3661 ixgbevf_update_stats(adapter); 3662 3663 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3664 3665 for (i = 0; i < adapter->num_rx_queues; i++) { 3666 ring = adapter->rx_ring[i]; 3667 do { 3668 start = u64_stats_fetch_begin_irq(&ring->syncp); 3669 bytes = ring->stats.bytes; 3670 packets = ring->stats.packets; 3671 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 3672 stats->rx_bytes += bytes; 3673 stats->rx_packets += packets; 3674 } 3675 3676 for (i = 0; i < adapter->num_tx_queues; i++) { 3677 ring = adapter->tx_ring[i]; 3678 do { 3679 start = u64_stats_fetch_begin_irq(&ring->syncp); 3680 bytes = ring->stats.bytes; 3681 packets = ring->stats.packets; 3682 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 3683 stats->tx_bytes += bytes; 3684 stats->tx_packets += packets; 3685 } 3686 3687 return stats; 3688 } 3689 3690 static const struct net_device_ops ixgbevf_netdev_ops = { 3691 .ndo_open = ixgbevf_open, 3692 .ndo_stop = ixgbevf_close, 3693 .ndo_start_xmit = ixgbevf_xmit_frame, 3694 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 3695 .ndo_get_stats64 = ixgbevf_get_stats, 3696 .ndo_validate_addr = eth_validate_addr, 3697 .ndo_set_mac_address = ixgbevf_set_mac, 3698 .ndo_change_mtu = ixgbevf_change_mtu, 3699 .ndo_tx_timeout = ixgbevf_tx_timeout, 3700 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 3701 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 3702 #ifdef CONFIG_NET_RX_BUSY_POLL 3703 .ndo_busy_poll = ixgbevf_busy_poll_recv, 3704 #endif 3705 #ifdef CONFIG_NET_POLL_CONTROLLER 3706 .ndo_poll_controller = ixgbevf_netpoll, 3707 #endif 3708 }; 3709 3710 static void ixgbevf_assign_netdev_ops(struct net_device *dev) 3711 { 3712 dev->netdev_ops = &ixgbevf_netdev_ops; 3713 ixgbevf_set_ethtool_ops(dev); 3714 dev->watchdog_timeo = 5 * HZ; 3715 } 3716 3717 /** 3718 * ixgbevf_probe - Device Initialization Routine 3719 * @pdev: PCI device information struct 3720 * @ent: entry in ixgbevf_pci_tbl 3721 * 3722 * Returns 0 on success, negative on failure 3723 * 3724 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 3725 * The OS initialization, configuring of the adapter private structure, 3726 * and a hardware reset occur. 3727 **/ 3728 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3729 { 3730 struct net_device *netdev; 3731 struct ixgbevf_adapter *adapter = NULL; 3732 struct ixgbe_hw *hw = NULL; 3733 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 3734 int err, pci_using_dac; 3735 bool disable_dev = false; 3736 3737 err = pci_enable_device(pdev); 3738 if (err) 3739 return err; 3740 3741 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 3742 pci_using_dac = 1; 3743 } else { 3744 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 3745 if (err) { 3746 dev_err(&pdev->dev, "No usable DMA " 3747 "configuration, aborting\n"); 3748 goto err_dma; 3749 } 3750 pci_using_dac = 0; 3751 } 3752 3753 err = pci_request_regions(pdev, ixgbevf_driver_name); 3754 if (err) { 3755 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 3756 goto err_pci_reg; 3757 } 3758 3759 pci_set_master(pdev); 3760 3761 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 3762 MAX_TX_QUEUES); 3763 if (!netdev) { 3764 err = -ENOMEM; 3765 goto err_alloc_etherdev; 3766 } 3767 3768 SET_NETDEV_DEV(netdev, &pdev->dev); 3769 3770 adapter = netdev_priv(netdev); 3771 3772 adapter->netdev = netdev; 3773 adapter->pdev = pdev; 3774 hw = &adapter->hw; 3775 hw->back = adapter; 3776 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 3777 3778 /* 3779 * call save state here in standalone driver because it relies on 3780 * adapter struct to exist, and needs to call netdev_priv 3781 */ 3782 pci_save_state(pdev); 3783 3784 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3785 pci_resource_len(pdev, 0)); 3786 adapter->io_addr = hw->hw_addr; 3787 if (!hw->hw_addr) { 3788 err = -EIO; 3789 goto err_ioremap; 3790 } 3791 3792 ixgbevf_assign_netdev_ops(netdev); 3793 3794 /* Setup hw api */ 3795 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 3796 hw->mac.type = ii->mac; 3797 3798 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 3799 sizeof(struct ixgbe_mbx_operations)); 3800 3801 /* setup the private structure */ 3802 err = ixgbevf_sw_init(adapter); 3803 if (err) 3804 goto err_sw_init; 3805 3806 /* The HW MAC address was set and/or determined in sw_init */ 3807 if (!is_valid_ether_addr(netdev->dev_addr)) { 3808 pr_err("invalid MAC address\n"); 3809 err = -EIO; 3810 goto err_sw_init; 3811 } 3812 3813 netdev->hw_features = NETIF_F_SG | 3814 NETIF_F_IP_CSUM | 3815 NETIF_F_IPV6_CSUM | 3816 NETIF_F_TSO | 3817 NETIF_F_TSO6 | 3818 NETIF_F_RXCSUM; 3819 3820 netdev->features = netdev->hw_features | 3821 NETIF_F_HW_VLAN_CTAG_TX | 3822 NETIF_F_HW_VLAN_CTAG_RX | 3823 NETIF_F_HW_VLAN_CTAG_FILTER; 3824 3825 netdev->vlan_features |= NETIF_F_TSO; 3826 netdev->vlan_features |= NETIF_F_TSO6; 3827 netdev->vlan_features |= NETIF_F_IP_CSUM; 3828 netdev->vlan_features |= NETIF_F_IPV6_CSUM; 3829 netdev->vlan_features |= NETIF_F_SG; 3830 3831 if (pci_using_dac) 3832 netdev->features |= NETIF_F_HIGHDMA; 3833 3834 netdev->priv_flags |= IFF_UNICAST_FLT; 3835 3836 init_timer(&adapter->watchdog_timer); 3837 adapter->watchdog_timer.function = ixgbevf_watchdog; 3838 adapter->watchdog_timer.data = (unsigned long)adapter; 3839 3840 if (IXGBE_REMOVED(hw->hw_addr)) { 3841 err = -EIO; 3842 goto err_sw_init; 3843 } 3844 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task); 3845 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task); 3846 set_bit(__IXGBEVF_WORK_INIT, &adapter->state); 3847 3848 err = ixgbevf_init_interrupt_scheme(adapter); 3849 if (err) 3850 goto err_sw_init; 3851 3852 strcpy(netdev->name, "eth%d"); 3853 3854 err = register_netdev(netdev); 3855 if (err) 3856 goto err_register; 3857 3858 pci_set_drvdata(pdev, netdev); 3859 netif_carrier_off(netdev); 3860 3861 ixgbevf_init_last_counter_stats(adapter); 3862 3863 /* print the VF info */ 3864 dev_info(&pdev->dev, "%pM\n", netdev->dev_addr); 3865 dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type); 3866 3867 switch (hw->mac.type) { 3868 case ixgbe_mac_X550_vf: 3869 dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n"); 3870 break; 3871 case ixgbe_mac_X540_vf: 3872 dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n"); 3873 break; 3874 case ixgbe_mac_82599_vf: 3875 default: 3876 dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n"); 3877 break; 3878 } 3879 3880 return 0; 3881 3882 err_register: 3883 ixgbevf_clear_interrupt_scheme(adapter); 3884 err_sw_init: 3885 ixgbevf_reset_interrupt_capability(adapter); 3886 iounmap(adapter->io_addr); 3887 err_ioremap: 3888 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 3889 free_netdev(netdev); 3890 err_alloc_etherdev: 3891 pci_release_regions(pdev); 3892 err_pci_reg: 3893 err_dma: 3894 if (!adapter || disable_dev) 3895 pci_disable_device(pdev); 3896 return err; 3897 } 3898 3899 /** 3900 * ixgbevf_remove - Device Removal Routine 3901 * @pdev: PCI device information struct 3902 * 3903 * ixgbevf_remove is called by the PCI subsystem to alert the driver 3904 * that it should release a PCI device. The could be caused by a 3905 * Hot-Plug event, or because the driver is going to be removed from 3906 * memory. 3907 **/ 3908 static void ixgbevf_remove(struct pci_dev *pdev) 3909 { 3910 struct net_device *netdev = pci_get_drvdata(pdev); 3911 struct ixgbevf_adapter *adapter; 3912 bool disable_dev; 3913 3914 if (!netdev) 3915 return; 3916 3917 adapter = netdev_priv(netdev); 3918 3919 set_bit(__IXGBEVF_REMOVING, &adapter->state); 3920 3921 del_timer_sync(&adapter->watchdog_timer); 3922 3923 cancel_work_sync(&adapter->reset_task); 3924 cancel_work_sync(&adapter->watchdog_task); 3925 3926 if (netdev->reg_state == NETREG_REGISTERED) 3927 unregister_netdev(netdev); 3928 3929 ixgbevf_clear_interrupt_scheme(adapter); 3930 ixgbevf_reset_interrupt_capability(adapter); 3931 3932 iounmap(adapter->io_addr); 3933 pci_release_regions(pdev); 3934 3935 hw_dbg(&adapter->hw, "Remove complete\n"); 3936 3937 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 3938 free_netdev(netdev); 3939 3940 if (disable_dev) 3941 pci_disable_device(pdev); 3942 } 3943 3944 /** 3945 * ixgbevf_io_error_detected - called when PCI error is detected 3946 * @pdev: Pointer to PCI device 3947 * @state: The current pci connection state 3948 * 3949 * This function is called after a PCI bus error affecting 3950 * this device has been detected. 3951 */ 3952 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, 3953 pci_channel_state_t state) 3954 { 3955 struct net_device *netdev = pci_get_drvdata(pdev); 3956 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3957 3958 if (!test_bit(__IXGBEVF_WORK_INIT, &adapter->state)) 3959 return PCI_ERS_RESULT_DISCONNECT; 3960 3961 rtnl_lock(); 3962 netif_device_detach(netdev); 3963 3964 if (state == pci_channel_io_perm_failure) { 3965 rtnl_unlock(); 3966 return PCI_ERS_RESULT_DISCONNECT; 3967 } 3968 3969 if (netif_running(netdev)) 3970 ixgbevf_down(adapter); 3971 3972 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 3973 pci_disable_device(pdev); 3974 rtnl_unlock(); 3975 3976 /* Request a slot slot reset. */ 3977 return PCI_ERS_RESULT_NEED_RESET; 3978 } 3979 3980 /** 3981 * ixgbevf_io_slot_reset - called after the pci bus has been reset. 3982 * @pdev: Pointer to PCI device 3983 * 3984 * Restart the card from scratch, as if from a cold-boot. Implementation 3985 * resembles the first-half of the ixgbevf_resume routine. 3986 */ 3987 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) 3988 { 3989 struct net_device *netdev = pci_get_drvdata(pdev); 3990 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3991 3992 if (pci_enable_device_mem(pdev)) { 3993 dev_err(&pdev->dev, 3994 "Cannot re-enable PCI device after reset.\n"); 3995 return PCI_ERS_RESULT_DISCONNECT; 3996 } 3997 3998 smp_mb__before_atomic(); 3999 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 4000 pci_set_master(pdev); 4001 4002 ixgbevf_reset(adapter); 4003 4004 return PCI_ERS_RESULT_RECOVERED; 4005 } 4006 4007 /** 4008 * ixgbevf_io_resume - called when traffic can start flowing again. 4009 * @pdev: Pointer to PCI device 4010 * 4011 * This callback is called when the error recovery driver tells us that 4012 * its OK to resume normal operation. Implementation resembles the 4013 * second-half of the ixgbevf_resume routine. 4014 */ 4015 static void ixgbevf_io_resume(struct pci_dev *pdev) 4016 { 4017 struct net_device *netdev = pci_get_drvdata(pdev); 4018 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4019 4020 if (netif_running(netdev)) 4021 ixgbevf_up(adapter); 4022 4023 netif_device_attach(netdev); 4024 } 4025 4026 /* PCI Error Recovery (ERS) */ 4027 static const struct pci_error_handlers ixgbevf_err_handler = { 4028 .error_detected = ixgbevf_io_error_detected, 4029 .slot_reset = ixgbevf_io_slot_reset, 4030 .resume = ixgbevf_io_resume, 4031 }; 4032 4033 static struct pci_driver ixgbevf_driver = { 4034 .name = ixgbevf_driver_name, 4035 .id_table = ixgbevf_pci_tbl, 4036 .probe = ixgbevf_probe, 4037 .remove = ixgbevf_remove, 4038 #ifdef CONFIG_PM 4039 /* Power Management Hooks */ 4040 .suspend = ixgbevf_suspend, 4041 .resume = ixgbevf_resume, 4042 #endif 4043 .shutdown = ixgbevf_shutdown, 4044 .err_handler = &ixgbevf_err_handler 4045 }; 4046 4047 /** 4048 * ixgbevf_init_module - Driver Registration Routine 4049 * 4050 * ixgbevf_init_module is the first routine called when the driver is 4051 * loaded. All it does is register with the PCI subsystem. 4052 **/ 4053 static int __init ixgbevf_init_module(void) 4054 { 4055 int ret; 4056 pr_info("%s - version %s\n", ixgbevf_driver_string, 4057 ixgbevf_driver_version); 4058 4059 pr_info("%s\n", ixgbevf_copyright); 4060 4061 ret = pci_register_driver(&ixgbevf_driver); 4062 return ret; 4063 } 4064 4065 module_init(ixgbevf_init_module); 4066 4067 /** 4068 * ixgbevf_exit_module - Driver Exit Cleanup Routine 4069 * 4070 * ixgbevf_exit_module is called just before the driver is removed 4071 * from memory. 4072 **/ 4073 static void __exit ixgbevf_exit_module(void) 4074 { 4075 pci_unregister_driver(&ixgbevf_driver); 4076 } 4077 4078 #ifdef DEBUG 4079 /** 4080 * ixgbevf_get_hw_dev_name - return device name string 4081 * used by hardware layer to print debugging information 4082 **/ 4083 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 4084 { 4085 struct ixgbevf_adapter *adapter = hw->back; 4086 return adapter->netdev->name; 4087 } 4088 4089 #endif 4090 module_exit(ixgbevf_exit_module); 4091 4092 /* ixgbevf_main.c */ 4093