1 /******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2012 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 26 *******************************************************************************/ 27 28 29 /****************************************************************************** 30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 31 ******************************************************************************/ 32 33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35 #include <linux/types.h> 36 #include <linux/bitops.h> 37 #include <linux/module.h> 38 #include <linux/pci.h> 39 #include <linux/netdevice.h> 40 #include <linux/vmalloc.h> 41 #include <linux/string.h> 42 #include <linux/in.h> 43 #include <linux/ip.h> 44 #include <linux/tcp.h> 45 #include <linux/sctp.h> 46 #include <linux/ipv6.h> 47 #include <linux/slab.h> 48 #include <net/checksum.h> 49 #include <net/ip6_checksum.h> 50 #include <linux/ethtool.h> 51 #include <linux/if.h> 52 #include <linux/if_vlan.h> 53 #include <linux/prefetch.h> 54 55 #include "ixgbevf.h" 56 57 const char ixgbevf_driver_name[] = "ixgbevf"; 58 static const char ixgbevf_driver_string[] = 59 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 60 61 #define DRV_VERSION "2.7.12-k" 62 const char ixgbevf_driver_version[] = DRV_VERSION; 63 static char ixgbevf_copyright[] = 64 "Copyright (c) 2009 - 2012 Intel Corporation."; 65 66 static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 67 [board_82599_vf] = &ixgbevf_82599_vf_info, 68 [board_X540_vf] = &ixgbevf_X540_vf_info, 69 }; 70 71 /* ixgbevf_pci_tbl - PCI Device ID Table 72 * 73 * Wildcard entries (PCI_ANY_ID) should come last 74 * Last entry must be all 0s 75 * 76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 77 * Class, Class Mask, private data (not used) } 78 */ 79 static struct pci_device_id ixgbevf_pci_tbl[] = { 80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), 81 board_82599_vf}, 82 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), 83 board_X540_vf}, 84 85 /* required last entry */ 86 {0, } 87 }; 88 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 89 90 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 91 MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver"); 92 MODULE_LICENSE("GPL"); 93 MODULE_VERSION(DRV_VERSION); 94 95 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 96 static int debug = -1; 97 module_param(debug, int, 0); 98 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 99 100 /* forward decls */ 101 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); 102 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); 103 104 static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw, 105 struct ixgbevf_ring *rx_ring, 106 u32 val) 107 { 108 /* 109 * Force memory writes to complete before letting h/w 110 * know there are new descriptors to fetch. (Only 111 * applicable for weak-ordered memory model archs, 112 * such as IA-64). 113 */ 114 wmb(); 115 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val); 116 } 117 118 /** 119 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 120 * @adapter: pointer to adapter struct 121 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 122 * @queue: queue to map the corresponding interrupt to 123 * @msix_vector: the vector to map to the corresponding queue 124 */ 125 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 126 u8 queue, u8 msix_vector) 127 { 128 u32 ivar, index; 129 struct ixgbe_hw *hw = &adapter->hw; 130 if (direction == -1) { 131 /* other causes */ 132 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 133 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 134 ivar &= ~0xFF; 135 ivar |= msix_vector; 136 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 137 } else { 138 /* tx or rx causes */ 139 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 140 index = ((16 * (queue & 1)) + (8 * direction)); 141 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 142 ivar &= ~(0xFF << index); 143 ivar |= (msix_vector << index); 144 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 145 } 146 } 147 148 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring, 149 struct ixgbevf_tx_buffer 150 *tx_buffer_info) 151 { 152 if (tx_buffer_info->dma) { 153 if (tx_buffer_info->mapped_as_page) 154 dma_unmap_page(tx_ring->dev, 155 tx_buffer_info->dma, 156 tx_buffer_info->length, 157 DMA_TO_DEVICE); 158 else 159 dma_unmap_single(tx_ring->dev, 160 tx_buffer_info->dma, 161 tx_buffer_info->length, 162 DMA_TO_DEVICE); 163 tx_buffer_info->dma = 0; 164 } 165 if (tx_buffer_info->skb) { 166 dev_kfree_skb_any(tx_buffer_info->skb); 167 tx_buffer_info->skb = NULL; 168 } 169 tx_buffer_info->time_stamp = 0; 170 /* tx_buffer_info must be completely set up in the transmit path */ 171 } 172 173 #define IXGBE_MAX_TXD_PWR 14 174 #define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR) 175 176 /* Tx Descriptors needed, worst case */ 177 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD) 178 #define DESC_NEEDED (MAX_SKB_FRAGS + 4) 179 180 static void ixgbevf_tx_timeout(struct net_device *netdev); 181 182 /** 183 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 184 * @q_vector: board private structure 185 * @tx_ring: tx ring to clean 186 **/ 187 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, 188 struct ixgbevf_ring *tx_ring) 189 { 190 struct ixgbevf_adapter *adapter = q_vector->adapter; 191 union ixgbe_adv_tx_desc *tx_desc, *eop_desc; 192 struct ixgbevf_tx_buffer *tx_buffer_info; 193 unsigned int i, eop, count = 0; 194 unsigned int total_bytes = 0, total_packets = 0; 195 196 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 197 return true; 198 199 i = tx_ring->next_to_clean; 200 eop = tx_ring->tx_buffer_info[i].next_to_watch; 201 eop_desc = IXGBEVF_TX_DESC(tx_ring, eop); 202 203 while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) && 204 (count < tx_ring->count)) { 205 bool cleaned = false; 206 rmb(); /* read buffer_info after eop_desc */ 207 /* eop could change between read and DD-check */ 208 if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch)) 209 goto cont_loop; 210 for ( ; !cleaned; count++) { 211 struct sk_buff *skb; 212 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 213 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 214 cleaned = (i == eop); 215 skb = tx_buffer_info->skb; 216 217 if (cleaned && skb) { 218 unsigned int segs, bytecount; 219 220 /* gso_segs is currently only valid for tcp */ 221 segs = skb_shinfo(skb)->gso_segs ?: 1; 222 /* multiply data chunks by size of headers */ 223 bytecount = ((segs - 1) * skb_headlen(skb)) + 224 skb->len; 225 total_packets += segs; 226 total_bytes += bytecount; 227 } 228 229 ixgbevf_unmap_and_free_tx_resource(tx_ring, 230 tx_buffer_info); 231 232 tx_desc->wb.status = 0; 233 234 i++; 235 if (i == tx_ring->count) 236 i = 0; 237 } 238 239 cont_loop: 240 eop = tx_ring->tx_buffer_info[i].next_to_watch; 241 eop_desc = IXGBEVF_TX_DESC(tx_ring, eop); 242 } 243 244 tx_ring->next_to_clean = i; 245 246 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 247 if (unlikely(count && netif_carrier_ok(tx_ring->netdev) && 248 (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 249 /* Make sure that anybody stopping the queue after this 250 * sees the new next_to_clean. 251 */ 252 smp_mb(); 253 if (__netif_subqueue_stopped(tx_ring->netdev, 254 tx_ring->queue_index) && 255 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 256 netif_wake_subqueue(tx_ring->netdev, 257 tx_ring->queue_index); 258 ++adapter->restart_queue; 259 } 260 } 261 262 u64_stats_update_begin(&tx_ring->syncp); 263 tx_ring->total_bytes += total_bytes; 264 tx_ring->total_packets += total_packets; 265 u64_stats_update_end(&tx_ring->syncp); 266 q_vector->tx.total_bytes += total_bytes; 267 q_vector->tx.total_packets += total_packets; 268 269 return count < tx_ring->count; 270 } 271 272 /** 273 * ixgbevf_receive_skb - Send a completed packet up the stack 274 * @q_vector: structure containing interrupt and ring information 275 * @skb: packet to send up 276 * @status: hardware indication of status of receive 277 * @rx_desc: rx descriptor 278 **/ 279 static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector, 280 struct sk_buff *skb, u8 status, 281 union ixgbe_adv_rx_desc *rx_desc) 282 { 283 struct ixgbevf_adapter *adapter = q_vector->adapter; 284 bool is_vlan = (status & IXGBE_RXD_STAT_VP); 285 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan); 286 287 if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans)) 288 __vlan_hwaccel_put_tag(skb, tag); 289 290 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL)) 291 napi_gro_receive(&q_vector->napi, skb); 292 else 293 netif_rx(skb); 294 } 295 296 /** 297 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 298 * @ring: pointer to Rx descriptor ring structure 299 * @status_err: hardware indication of status of receive 300 * @skb: skb currently being received and modified 301 **/ 302 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, 303 u32 status_err, struct sk_buff *skb) 304 { 305 skb_checksum_none_assert(skb); 306 307 /* Rx csum disabled */ 308 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 309 return; 310 311 /* if IP and error */ 312 if ((status_err & IXGBE_RXD_STAT_IPCS) && 313 (status_err & IXGBE_RXDADV_ERR_IPE)) { 314 ring->hw_csum_rx_error++; 315 return; 316 } 317 318 if (!(status_err & IXGBE_RXD_STAT_L4CS)) 319 return; 320 321 if (status_err & IXGBE_RXDADV_ERR_TCPE) { 322 ring->hw_csum_rx_error++; 323 return; 324 } 325 326 /* It must be a TCP or UDP packet with a valid checksum */ 327 skb->ip_summed = CHECKSUM_UNNECESSARY; 328 ring->hw_csum_rx_good++; 329 } 330 331 /** 332 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 333 * @adapter: address of board private structure 334 **/ 335 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter, 336 struct ixgbevf_ring *rx_ring, 337 int cleaned_count) 338 { 339 struct pci_dev *pdev = adapter->pdev; 340 union ixgbe_adv_rx_desc *rx_desc; 341 struct ixgbevf_rx_buffer *bi; 342 unsigned int i = rx_ring->next_to_use; 343 344 bi = &rx_ring->rx_buffer_info[i]; 345 346 while (cleaned_count--) { 347 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 348 349 if (!bi->skb) { 350 struct sk_buff *skb; 351 352 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 353 rx_ring->rx_buf_len); 354 if (!skb) { 355 adapter->alloc_rx_buff_failed++; 356 goto no_buffers; 357 } 358 bi->skb = skb; 359 360 bi->dma = dma_map_single(&pdev->dev, skb->data, 361 rx_ring->rx_buf_len, 362 DMA_FROM_DEVICE); 363 if (dma_mapping_error(&pdev->dev, bi->dma)) { 364 dev_kfree_skb(skb); 365 bi->skb = NULL; 366 dev_err(&pdev->dev, "RX DMA map failed\n"); 367 break; 368 } 369 } 370 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); 371 372 i++; 373 if (i == rx_ring->count) 374 i = 0; 375 bi = &rx_ring->rx_buffer_info[i]; 376 } 377 378 no_buffers: 379 if (rx_ring->next_to_use != i) { 380 rx_ring->next_to_use = i; 381 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i); 382 } 383 } 384 385 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 386 u32 qmask) 387 { 388 struct ixgbe_hw *hw = &adapter->hw; 389 390 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); 391 } 392 393 static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 394 struct ixgbevf_ring *rx_ring, 395 int budget) 396 { 397 struct ixgbevf_adapter *adapter = q_vector->adapter; 398 struct pci_dev *pdev = adapter->pdev; 399 union ixgbe_adv_rx_desc *rx_desc, *next_rxd; 400 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer; 401 struct sk_buff *skb; 402 unsigned int i; 403 u32 len, staterr; 404 int cleaned_count = 0; 405 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 406 407 i = rx_ring->next_to_clean; 408 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 409 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 410 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 411 412 while (staterr & IXGBE_RXD_STAT_DD) { 413 if (!budget) 414 break; 415 budget--; 416 417 rmb(); /* read descriptor and rx_buffer_info after status DD */ 418 len = le16_to_cpu(rx_desc->wb.upper.length); 419 skb = rx_buffer_info->skb; 420 prefetch(skb->data - NET_IP_ALIGN); 421 rx_buffer_info->skb = NULL; 422 423 if (rx_buffer_info->dma) { 424 dma_unmap_single(&pdev->dev, rx_buffer_info->dma, 425 rx_ring->rx_buf_len, 426 DMA_FROM_DEVICE); 427 rx_buffer_info->dma = 0; 428 skb_put(skb, len); 429 } 430 431 i++; 432 if (i == rx_ring->count) 433 i = 0; 434 435 next_rxd = IXGBEVF_RX_DESC(rx_ring, i); 436 prefetch(next_rxd); 437 cleaned_count++; 438 439 next_buffer = &rx_ring->rx_buffer_info[i]; 440 441 if (!(staterr & IXGBE_RXD_STAT_EOP)) { 442 skb->next = next_buffer->skb; 443 IXGBE_CB(skb->next)->prev = skb; 444 adapter->non_eop_descs++; 445 goto next_desc; 446 } 447 448 /* we should not be chaining buffers, if we did drop the skb */ 449 if (IXGBE_CB(skb)->prev) { 450 do { 451 struct sk_buff *this = skb; 452 skb = IXGBE_CB(skb)->prev; 453 dev_kfree_skb(this); 454 } while (skb); 455 goto next_desc; 456 } 457 458 /* ERR_MASK will only have valid bits if EOP set */ 459 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) { 460 dev_kfree_skb_irq(skb); 461 goto next_desc; 462 } 463 464 ixgbevf_rx_checksum(rx_ring, staterr, skb); 465 466 /* probably a little skewed due to removing CRC */ 467 total_rx_bytes += skb->len; 468 total_rx_packets++; 469 470 /* 471 * Work around issue of some types of VM to VM loop back 472 * packets not getting split correctly 473 */ 474 if (staterr & IXGBE_RXD_STAT_LB) { 475 u32 header_fixup_len = skb_headlen(skb); 476 if (header_fixup_len < 14) 477 skb_push(skb, header_fixup_len); 478 } 479 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 480 481 /* Workaround hardware that can't do proper VEPA multicast 482 * source pruning. 483 */ 484 if ((skb->pkt_type & (PACKET_BROADCAST | PACKET_MULTICAST)) && 485 !(compare_ether_addr(adapter->netdev->dev_addr, 486 eth_hdr(skb)->h_source))) { 487 dev_kfree_skb_irq(skb); 488 goto next_desc; 489 } 490 491 ixgbevf_receive_skb(q_vector, skb, staterr, rx_desc); 492 493 next_desc: 494 rx_desc->wb.upper.status_error = 0; 495 496 /* return some buffers to hardware, one at a time is too slow */ 497 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 498 ixgbevf_alloc_rx_buffers(adapter, rx_ring, 499 cleaned_count); 500 cleaned_count = 0; 501 } 502 503 /* use prefetched values */ 504 rx_desc = next_rxd; 505 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 506 507 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 508 } 509 510 rx_ring->next_to_clean = i; 511 cleaned_count = IXGBE_DESC_UNUSED(rx_ring); 512 513 if (cleaned_count) 514 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count); 515 516 u64_stats_update_begin(&rx_ring->syncp); 517 rx_ring->total_packets += total_rx_packets; 518 rx_ring->total_bytes += total_rx_bytes; 519 u64_stats_update_end(&rx_ring->syncp); 520 q_vector->rx.total_packets += total_rx_packets; 521 q_vector->rx.total_bytes += total_rx_bytes; 522 523 return !!budget; 524 } 525 526 /** 527 * ixgbevf_poll - NAPI polling calback 528 * @napi: napi struct with our devices info in it 529 * @budget: amount of work driver is allowed to do this pass, in packets 530 * 531 * This function will clean more than one or more rings associated with a 532 * q_vector. 533 **/ 534 static int ixgbevf_poll(struct napi_struct *napi, int budget) 535 { 536 struct ixgbevf_q_vector *q_vector = 537 container_of(napi, struct ixgbevf_q_vector, napi); 538 struct ixgbevf_adapter *adapter = q_vector->adapter; 539 struct ixgbevf_ring *ring; 540 int per_ring_budget; 541 bool clean_complete = true; 542 543 ixgbevf_for_each_ring(ring, q_vector->tx) 544 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring); 545 546 /* attempt to distribute budget to each queue fairly, but don't allow 547 * the budget to go below 1 because we'll exit polling */ 548 if (q_vector->rx.count > 1) 549 per_ring_budget = max(budget/q_vector->rx.count, 1); 550 else 551 per_ring_budget = budget; 552 553 adapter->flags |= IXGBE_FLAG_IN_NETPOLL; 554 ixgbevf_for_each_ring(ring, q_vector->rx) 555 clean_complete &= ixgbevf_clean_rx_irq(q_vector, ring, 556 per_ring_budget); 557 adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL; 558 559 /* If all work not completed, return budget and keep polling */ 560 if (!clean_complete) 561 return budget; 562 /* all work done, exit the polling mode */ 563 napi_complete(napi); 564 if (adapter->rx_itr_setting & 1) 565 ixgbevf_set_itr(q_vector); 566 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 567 ixgbevf_irq_enable_queues(adapter, 568 1 << q_vector->v_idx); 569 570 return 0; 571 } 572 573 /** 574 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 575 * @q_vector: structure containing interrupt and ring information 576 */ 577 static void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) 578 { 579 struct ixgbevf_adapter *adapter = q_vector->adapter; 580 struct ixgbe_hw *hw = &adapter->hw; 581 int v_idx = q_vector->v_idx; 582 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 583 584 /* 585 * set the WDIS bit to not clear the timer bits and cause an 586 * immediate assertion of the interrupt 587 */ 588 itr_reg |= IXGBE_EITR_CNT_WDIS; 589 590 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 591 } 592 593 /** 594 * ixgbevf_configure_msix - Configure MSI-X hardware 595 * @adapter: board private structure 596 * 597 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 598 * interrupts. 599 **/ 600 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 601 { 602 struct ixgbevf_q_vector *q_vector; 603 int q_vectors, v_idx; 604 605 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 606 adapter->eims_enable_mask = 0; 607 608 /* 609 * Populate the IVAR table and set the ITR values to the 610 * corresponding register. 611 */ 612 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 613 struct ixgbevf_ring *ring; 614 q_vector = adapter->q_vector[v_idx]; 615 616 ixgbevf_for_each_ring(ring, q_vector->rx) 617 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx); 618 619 ixgbevf_for_each_ring(ring, q_vector->tx) 620 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx); 621 622 if (q_vector->tx.ring && !q_vector->rx.ring) { 623 /* tx only vector */ 624 if (adapter->tx_itr_setting == 1) 625 q_vector->itr = IXGBE_10K_ITR; 626 else 627 q_vector->itr = adapter->tx_itr_setting; 628 } else { 629 /* rx or rx/tx vector */ 630 if (adapter->rx_itr_setting == 1) 631 q_vector->itr = IXGBE_20K_ITR; 632 else 633 q_vector->itr = adapter->rx_itr_setting; 634 } 635 636 /* add q_vector eims value to global eims_enable_mask */ 637 adapter->eims_enable_mask |= 1 << v_idx; 638 639 ixgbevf_write_eitr(q_vector); 640 } 641 642 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 643 /* setup eims_other and add value to global eims_enable_mask */ 644 adapter->eims_other = 1 << v_idx; 645 adapter->eims_enable_mask |= adapter->eims_other; 646 } 647 648 enum latency_range { 649 lowest_latency = 0, 650 low_latency = 1, 651 bulk_latency = 2, 652 latency_invalid = 255 653 }; 654 655 /** 656 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 657 * @q_vector: structure containing interrupt and ring information 658 * @ring_container: structure containing ring performance data 659 * 660 * Stores a new ITR value based on packets and byte 661 * counts during the last interrupt. The advantage of per interrupt 662 * computation is faster updates and more accurate ITR for the current 663 * traffic pattern. Constants in this function were computed 664 * based on theoretical maximum wire speed and thresholds were set based 665 * on testing data as well as attempting to minimize response time 666 * while increasing bulk throughput. 667 **/ 668 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector, 669 struct ixgbevf_ring_container *ring_container) 670 { 671 int bytes = ring_container->total_bytes; 672 int packets = ring_container->total_packets; 673 u32 timepassed_us; 674 u64 bytes_perint; 675 u8 itr_setting = ring_container->itr; 676 677 if (packets == 0) 678 return; 679 680 /* simple throttlerate management 681 * 0-20MB/s lowest (100000 ints/s) 682 * 20-100MB/s low (20000 ints/s) 683 * 100-1249MB/s bulk (8000 ints/s) 684 */ 685 /* what was last interrupt timeslice? */ 686 timepassed_us = q_vector->itr >> 2; 687 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 688 689 switch (itr_setting) { 690 case lowest_latency: 691 if (bytes_perint > 10) 692 itr_setting = low_latency; 693 break; 694 case low_latency: 695 if (bytes_perint > 20) 696 itr_setting = bulk_latency; 697 else if (bytes_perint <= 10) 698 itr_setting = lowest_latency; 699 break; 700 case bulk_latency: 701 if (bytes_perint <= 20) 702 itr_setting = low_latency; 703 break; 704 } 705 706 /* clear work counters since we have the values we need */ 707 ring_container->total_bytes = 0; 708 ring_container->total_packets = 0; 709 710 /* write updated itr to ring container */ 711 ring_container->itr = itr_setting; 712 } 713 714 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector) 715 { 716 u32 new_itr = q_vector->itr; 717 u8 current_itr; 718 719 ixgbevf_update_itr(q_vector, &q_vector->tx); 720 ixgbevf_update_itr(q_vector, &q_vector->rx); 721 722 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 723 724 switch (current_itr) { 725 /* counts and packets in update_itr are dependent on these numbers */ 726 case lowest_latency: 727 new_itr = IXGBE_100K_ITR; 728 break; 729 case low_latency: 730 new_itr = IXGBE_20K_ITR; 731 break; 732 case bulk_latency: 733 default: 734 new_itr = IXGBE_8K_ITR; 735 break; 736 } 737 738 if (new_itr != q_vector->itr) { 739 /* do an exponential smoothing */ 740 new_itr = (10 * new_itr * q_vector->itr) / 741 ((9 * new_itr) + q_vector->itr); 742 743 /* save the algorithm value here */ 744 q_vector->itr = new_itr; 745 746 ixgbevf_write_eitr(q_vector); 747 } 748 } 749 750 static irqreturn_t ixgbevf_msix_other(int irq, void *data) 751 { 752 struct ixgbevf_adapter *adapter = data; 753 struct pci_dev *pdev = adapter->pdev; 754 struct ixgbe_hw *hw = &adapter->hw; 755 u32 msg; 756 bool got_ack = false; 757 758 hw->mac.get_link_status = 1; 759 if (!hw->mbx.ops.check_for_ack(hw)) 760 got_ack = true; 761 762 if (!hw->mbx.ops.check_for_msg(hw)) { 763 hw->mbx.ops.read(hw, &msg, 1); 764 765 if ((msg & IXGBE_MBVFICR_VFREQ_MASK) == IXGBE_PF_CONTROL_MSG) { 766 mod_timer(&adapter->watchdog_timer, 767 round_jiffies(jiffies + 1)); 768 adapter->link_up = false; 769 } 770 771 if (msg & IXGBE_VT_MSGTYPE_NACK) 772 dev_info(&pdev->dev, 773 "Last Request of type %2.2x to PF Nacked\n", 774 msg & 0xFF); 775 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFSTS; 776 } 777 778 /* checking for the ack clears the PFACK bit. Place 779 * it back in the v2p_mailbox cache so that anyone 780 * polling for an ack will not miss it 781 */ 782 if (got_ack) 783 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFACK; 784 785 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); 786 787 return IRQ_HANDLED; 788 } 789 790 /** 791 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) 792 * @irq: unused 793 * @data: pointer to our q_vector struct for this interrupt vector 794 **/ 795 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data) 796 { 797 struct ixgbevf_q_vector *q_vector = data; 798 799 /* EIAM disabled interrupts (on this vector) for us */ 800 if (q_vector->rx.ring || q_vector->tx.ring) 801 napi_schedule(&q_vector->napi); 802 803 return IRQ_HANDLED; 804 } 805 806 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, 807 int r_idx) 808 { 809 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 810 811 a->rx_ring[r_idx].next = q_vector->rx.ring; 812 q_vector->rx.ring = &a->rx_ring[r_idx]; 813 q_vector->rx.count++; 814 } 815 816 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, 817 int t_idx) 818 { 819 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 820 821 a->tx_ring[t_idx].next = q_vector->tx.ring; 822 q_vector->tx.ring = &a->tx_ring[t_idx]; 823 q_vector->tx.count++; 824 } 825 826 /** 827 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors 828 * @adapter: board private structure to initialize 829 * 830 * This function maps descriptor rings to the queue-specific vectors 831 * we were allotted through the MSI-X enabling code. Ideally, we'd have 832 * one vector per ring/queue, but on a constrained vector budget, we 833 * group the rings as "efficiently" as possible. You would add new 834 * mapping configurations in here. 835 **/ 836 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter) 837 { 838 int q_vectors; 839 int v_start = 0; 840 int rxr_idx = 0, txr_idx = 0; 841 int rxr_remaining = adapter->num_rx_queues; 842 int txr_remaining = adapter->num_tx_queues; 843 int i, j; 844 int rqpv, tqpv; 845 int err = 0; 846 847 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 848 849 /* 850 * The ideal configuration... 851 * We have enough vectors to map one per queue. 852 */ 853 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) { 854 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) 855 map_vector_to_rxq(adapter, v_start, rxr_idx); 856 857 for (; txr_idx < txr_remaining; v_start++, txr_idx++) 858 map_vector_to_txq(adapter, v_start, txr_idx); 859 goto out; 860 } 861 862 /* 863 * If we don't have enough vectors for a 1-to-1 864 * mapping, we'll have to group them so there are 865 * multiple queues per vector. 866 */ 867 /* Re-adjusting *qpv takes care of the remainder. */ 868 for (i = v_start; i < q_vectors; i++) { 869 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i); 870 for (j = 0; j < rqpv; j++) { 871 map_vector_to_rxq(adapter, i, rxr_idx); 872 rxr_idx++; 873 rxr_remaining--; 874 } 875 } 876 for (i = v_start; i < q_vectors; i++) { 877 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i); 878 for (j = 0; j < tqpv; j++) { 879 map_vector_to_txq(adapter, i, txr_idx); 880 txr_idx++; 881 txr_remaining--; 882 } 883 } 884 885 out: 886 return err; 887 } 888 889 /** 890 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 891 * @adapter: board private structure 892 * 893 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 894 * interrupts from the kernel. 895 **/ 896 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 897 { 898 struct net_device *netdev = adapter->netdev; 899 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 900 int vector, err; 901 int ri = 0, ti = 0; 902 903 for (vector = 0; vector < q_vectors; vector++) { 904 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector]; 905 struct msix_entry *entry = &adapter->msix_entries[vector]; 906 907 if (q_vector->tx.ring && q_vector->rx.ring) { 908 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 909 "%s-%s-%d", netdev->name, "TxRx", ri++); 910 ti++; 911 } else if (q_vector->rx.ring) { 912 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 913 "%s-%s-%d", netdev->name, "rx", ri++); 914 } else if (q_vector->tx.ring) { 915 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 916 "%s-%s-%d", netdev->name, "tx", ti++); 917 } else { 918 /* skip this unused q_vector */ 919 continue; 920 } 921 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0, 922 q_vector->name, q_vector); 923 if (err) { 924 hw_dbg(&adapter->hw, 925 "request_irq failed for MSIX interrupt " 926 "Error: %d\n", err); 927 goto free_queue_irqs; 928 } 929 } 930 931 err = request_irq(adapter->msix_entries[vector].vector, 932 &ixgbevf_msix_other, 0, netdev->name, adapter); 933 if (err) { 934 hw_dbg(&adapter->hw, 935 "request_irq for msix_other failed: %d\n", err); 936 goto free_queue_irqs; 937 } 938 939 return 0; 940 941 free_queue_irqs: 942 while (vector) { 943 vector--; 944 free_irq(adapter->msix_entries[vector].vector, 945 adapter->q_vector[vector]); 946 } 947 pci_disable_msix(adapter->pdev); 948 kfree(adapter->msix_entries); 949 adapter->msix_entries = NULL; 950 return err; 951 } 952 953 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter) 954 { 955 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 956 957 for (i = 0; i < q_vectors; i++) { 958 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i]; 959 q_vector->rx.ring = NULL; 960 q_vector->tx.ring = NULL; 961 q_vector->rx.count = 0; 962 q_vector->tx.count = 0; 963 } 964 } 965 966 /** 967 * ixgbevf_request_irq - initialize interrupts 968 * @adapter: board private structure 969 * 970 * Attempts to configure interrupts using the best available 971 * capabilities of the hardware and kernel. 972 **/ 973 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 974 { 975 int err = 0; 976 977 err = ixgbevf_request_msix_irqs(adapter); 978 979 if (err) 980 hw_dbg(&adapter->hw, 981 "request_irq failed, Error %d\n", err); 982 983 return err; 984 } 985 986 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 987 { 988 int i, q_vectors; 989 990 q_vectors = adapter->num_msix_vectors; 991 i = q_vectors - 1; 992 993 free_irq(adapter->msix_entries[i].vector, adapter); 994 i--; 995 996 for (; i >= 0; i--) { 997 /* free only the irqs that were actually requested */ 998 if (!adapter->q_vector[i]->rx.ring && 999 !adapter->q_vector[i]->tx.ring) 1000 continue; 1001 1002 free_irq(adapter->msix_entries[i].vector, 1003 adapter->q_vector[i]); 1004 } 1005 1006 ixgbevf_reset_q_vectors(adapter); 1007 } 1008 1009 /** 1010 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1011 * @adapter: board private structure 1012 **/ 1013 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1014 { 1015 struct ixgbe_hw *hw = &adapter->hw; 1016 int i; 1017 1018 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0); 1019 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1020 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0); 1021 1022 IXGBE_WRITE_FLUSH(hw); 1023 1024 for (i = 0; i < adapter->num_msix_vectors; i++) 1025 synchronize_irq(adapter->msix_entries[i].vector); 1026 } 1027 1028 /** 1029 * ixgbevf_irq_enable - Enable default interrupt generation settings 1030 * @adapter: board private structure 1031 **/ 1032 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) 1033 { 1034 struct ixgbe_hw *hw = &adapter->hw; 1035 1036 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask); 1037 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask); 1038 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); 1039 } 1040 1041 /** 1042 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1043 * @adapter: board private structure 1044 * 1045 * Configure the Tx unit of the MAC after a reset. 1046 **/ 1047 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1048 { 1049 u64 tdba; 1050 struct ixgbe_hw *hw = &adapter->hw; 1051 u32 i, j, tdlen, txctrl; 1052 1053 /* Setup the HW Tx Head and Tail descriptor pointers */ 1054 for (i = 0; i < adapter->num_tx_queues; i++) { 1055 struct ixgbevf_ring *ring = &adapter->tx_ring[i]; 1056 j = ring->reg_idx; 1057 tdba = ring->dma; 1058 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc); 1059 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1060 (tdba & DMA_BIT_MASK(32))); 1061 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1062 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen); 1063 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0); 1064 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0); 1065 adapter->tx_ring[i].head = IXGBE_VFTDH(j); 1066 adapter->tx_ring[i].tail = IXGBE_VFTDT(j); 1067 /* Disable Tx Head Writeback RO bit, since this hoses 1068 * bookkeeping if things aren't delivered in order. 1069 */ 1070 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1071 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 1072 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1073 } 1074 } 1075 1076 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1077 1078 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) 1079 { 1080 struct ixgbevf_ring *rx_ring; 1081 struct ixgbe_hw *hw = &adapter->hw; 1082 u32 srrctl; 1083 1084 rx_ring = &adapter->rx_ring[index]; 1085 1086 srrctl = IXGBE_SRRCTL_DROP_EN; 1087 1088 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1089 1090 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >> 1091 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1092 1093 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1094 } 1095 1096 static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter) 1097 { 1098 struct ixgbe_hw *hw = &adapter->hw; 1099 struct net_device *netdev = adapter->netdev; 1100 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 1101 int i; 1102 u16 rx_buf_len; 1103 1104 /* notify the PF of our intent to use this size of frame */ 1105 ixgbevf_rlpml_set_vf(hw, max_frame); 1106 1107 /* PF will allow an extra 4 bytes past for vlan tagged frames */ 1108 max_frame += VLAN_HLEN; 1109 1110 /* 1111 * Allocate buffer sizes that fit well into 32K and 1112 * take into account max frame size of 9.5K 1113 */ 1114 if ((hw->mac.type == ixgbe_mac_X540_vf) && 1115 (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)) 1116 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE; 1117 else if (max_frame <= IXGBEVF_RXBUFFER_2K) 1118 rx_buf_len = IXGBEVF_RXBUFFER_2K; 1119 else if (max_frame <= IXGBEVF_RXBUFFER_4K) 1120 rx_buf_len = IXGBEVF_RXBUFFER_4K; 1121 else if (max_frame <= IXGBEVF_RXBUFFER_8K) 1122 rx_buf_len = IXGBEVF_RXBUFFER_8K; 1123 else 1124 rx_buf_len = IXGBEVF_RXBUFFER_10K; 1125 1126 for (i = 0; i < adapter->num_rx_queues; i++) 1127 adapter->rx_ring[i].rx_buf_len = rx_buf_len; 1128 } 1129 1130 /** 1131 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1132 * @adapter: board private structure 1133 * 1134 * Configure the Rx unit of the MAC after a reset. 1135 **/ 1136 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1137 { 1138 u64 rdba; 1139 struct ixgbe_hw *hw = &adapter->hw; 1140 int i, j; 1141 u32 rdlen; 1142 1143 /* PSRTYPE must be initialized in 82599 */ 1144 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0); 1145 1146 /* set_rx_buffer_len must be called before ring initialization */ 1147 ixgbevf_set_rx_buffer_len(adapter); 1148 1149 rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc); 1150 /* Setup the HW Rx Head and Tail Descriptor Pointers and 1151 * the Base and Length of the Rx Descriptor Ring */ 1152 for (i = 0; i < adapter->num_rx_queues; i++) { 1153 rdba = adapter->rx_ring[i].dma; 1154 j = adapter->rx_ring[i].reg_idx; 1155 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1156 (rdba & DMA_BIT_MASK(32))); 1157 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1158 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen); 1159 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0); 1160 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0); 1161 adapter->rx_ring[i].head = IXGBE_VFRDH(j); 1162 adapter->rx_ring[i].tail = IXGBE_VFRDT(j); 1163 1164 ixgbevf_configure_srrctl(adapter, j); 1165 } 1166 } 1167 1168 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid) 1169 { 1170 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1171 struct ixgbe_hw *hw = &adapter->hw; 1172 int err; 1173 1174 spin_lock_bh(&adapter->mbx_lock); 1175 1176 /* add VID to filter table */ 1177 err = hw->mac.ops.set_vfta(hw, vid, 0, true); 1178 1179 spin_unlock_bh(&adapter->mbx_lock); 1180 1181 /* translate error return types so error makes sense */ 1182 if (err == IXGBE_ERR_MBX) 1183 return -EIO; 1184 1185 if (err == IXGBE_ERR_INVALID_ARGUMENT) 1186 return -EACCES; 1187 1188 set_bit(vid, adapter->active_vlans); 1189 1190 return err; 1191 } 1192 1193 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) 1194 { 1195 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1196 struct ixgbe_hw *hw = &adapter->hw; 1197 int err = -EOPNOTSUPP; 1198 1199 spin_lock_bh(&adapter->mbx_lock); 1200 1201 /* remove VID from filter table */ 1202 err = hw->mac.ops.set_vfta(hw, vid, 0, false); 1203 1204 spin_unlock_bh(&adapter->mbx_lock); 1205 1206 clear_bit(vid, adapter->active_vlans); 1207 1208 return err; 1209 } 1210 1211 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 1212 { 1213 u16 vid; 1214 1215 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1216 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid); 1217 } 1218 1219 static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 1220 { 1221 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1222 struct ixgbe_hw *hw = &adapter->hw; 1223 int count = 0; 1224 1225 if ((netdev_uc_count(netdev)) > 10) { 1226 pr_err("Too many unicast filters - No Space\n"); 1227 return -ENOSPC; 1228 } 1229 1230 if (!netdev_uc_empty(netdev)) { 1231 struct netdev_hw_addr *ha; 1232 netdev_for_each_uc_addr(ha, netdev) { 1233 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 1234 udelay(200); 1235 } 1236 } else { 1237 /* 1238 * If the list is empty then send message to PF driver to 1239 * clear all macvlans on this VF. 1240 */ 1241 hw->mac.ops.set_uc_addr(hw, 0, NULL); 1242 } 1243 1244 return count; 1245 } 1246 1247 /** 1248 * ixgbevf_set_rx_mode - Multicast and unicast set 1249 * @netdev: network interface device structure 1250 * 1251 * The set_rx_method entry point is called whenever the multicast address 1252 * list, unicast address list or the network interface flags are updated. 1253 * This routine is responsible for configuring the hardware for proper 1254 * multicast mode and configuring requested unicast filters. 1255 **/ 1256 static void ixgbevf_set_rx_mode(struct net_device *netdev) 1257 { 1258 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1259 struct ixgbe_hw *hw = &adapter->hw; 1260 1261 spin_lock_bh(&adapter->mbx_lock); 1262 1263 /* reprogram multicast list */ 1264 hw->mac.ops.update_mc_addr_list(hw, netdev); 1265 1266 ixgbevf_write_uc_addr_list(netdev); 1267 1268 spin_unlock_bh(&adapter->mbx_lock); 1269 } 1270 1271 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 1272 { 1273 int q_idx; 1274 struct ixgbevf_q_vector *q_vector; 1275 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1276 1277 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1278 q_vector = adapter->q_vector[q_idx]; 1279 napi_enable(&q_vector->napi); 1280 } 1281 } 1282 1283 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 1284 { 1285 int q_idx; 1286 struct ixgbevf_q_vector *q_vector; 1287 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1288 1289 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1290 q_vector = adapter->q_vector[q_idx]; 1291 napi_disable(&q_vector->napi); 1292 } 1293 } 1294 1295 static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 1296 { 1297 struct net_device *netdev = adapter->netdev; 1298 int i; 1299 1300 ixgbevf_set_rx_mode(netdev); 1301 1302 ixgbevf_restore_vlan(adapter); 1303 1304 ixgbevf_configure_tx(adapter); 1305 ixgbevf_configure_rx(adapter); 1306 for (i = 0; i < adapter->num_rx_queues; i++) { 1307 struct ixgbevf_ring *ring = &adapter->rx_ring[i]; 1308 ixgbevf_alloc_rx_buffers(adapter, ring, 1309 IXGBE_DESC_UNUSED(ring)); 1310 } 1311 } 1312 1313 #define IXGBE_MAX_RX_DESC_POLL 10 1314 static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1315 int rxr) 1316 { 1317 struct ixgbe_hw *hw = &adapter->hw; 1318 int j = adapter->rx_ring[rxr].reg_idx; 1319 int k; 1320 1321 for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) { 1322 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE) 1323 break; 1324 else 1325 msleep(1); 1326 } 1327 if (k >= IXGBE_MAX_RX_DESC_POLL) { 1328 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d " 1329 "not set within the polling period\n", rxr); 1330 } 1331 1332 ixgbevf_release_rx_desc(hw, &adapter->rx_ring[rxr], 1333 adapter->rx_ring[rxr].count - 1); 1334 } 1335 1336 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 1337 { 1338 /* Only save pre-reset stats if there are some */ 1339 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 1340 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 1341 adapter->stats.base_vfgprc; 1342 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 1343 adapter->stats.base_vfgptc; 1344 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 1345 adapter->stats.base_vfgorc; 1346 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 1347 adapter->stats.base_vfgotc; 1348 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 1349 adapter->stats.base_vfmprc; 1350 } 1351 } 1352 1353 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 1354 { 1355 struct ixgbe_hw *hw = &adapter->hw; 1356 1357 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 1358 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 1359 adapter->stats.last_vfgorc |= 1360 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 1361 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 1362 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 1363 adapter->stats.last_vfgotc |= 1364 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 1365 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 1366 1367 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 1368 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 1369 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 1370 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 1371 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 1372 } 1373 1374 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) 1375 { 1376 struct ixgbe_hw *hw = &adapter->hw; 1377 int api[] = { ixgbe_mbox_api_11, 1378 ixgbe_mbox_api_10, 1379 ixgbe_mbox_api_unknown }; 1380 int err = 0, idx = 0; 1381 1382 spin_lock_bh(&adapter->mbx_lock); 1383 1384 while (api[idx] != ixgbe_mbox_api_unknown) { 1385 err = ixgbevf_negotiate_api_version(hw, api[idx]); 1386 if (!err) 1387 break; 1388 idx++; 1389 } 1390 1391 spin_unlock_bh(&adapter->mbx_lock); 1392 } 1393 1394 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 1395 { 1396 struct net_device *netdev = adapter->netdev; 1397 struct ixgbe_hw *hw = &adapter->hw; 1398 int i, j = 0; 1399 int num_rx_rings = adapter->num_rx_queues; 1400 u32 txdctl, rxdctl; 1401 1402 for (i = 0; i < adapter->num_tx_queues; i++) { 1403 j = adapter->tx_ring[i].reg_idx; 1404 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1405 /* enable WTHRESH=8 descriptors, to encourage burst writeback */ 1406 txdctl |= (8 << 16); 1407 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1408 } 1409 1410 for (i = 0; i < adapter->num_tx_queues; i++) { 1411 j = adapter->tx_ring[i].reg_idx; 1412 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1413 txdctl |= IXGBE_TXDCTL_ENABLE; 1414 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1415 } 1416 1417 for (i = 0; i < num_rx_rings; i++) { 1418 j = adapter->rx_ring[i].reg_idx; 1419 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1420 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1421 if (hw->mac.type == ixgbe_mac_X540_vf) { 1422 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK; 1423 rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) | 1424 IXGBE_RXDCTL_RLPML_EN); 1425 } 1426 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1427 ixgbevf_rx_desc_queue_enable(adapter, i); 1428 } 1429 1430 ixgbevf_configure_msix(adapter); 1431 1432 spin_lock_bh(&adapter->mbx_lock); 1433 1434 if (is_valid_ether_addr(hw->mac.addr)) 1435 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 1436 else 1437 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 1438 1439 spin_unlock_bh(&adapter->mbx_lock); 1440 1441 clear_bit(__IXGBEVF_DOWN, &adapter->state); 1442 ixgbevf_napi_enable_all(adapter); 1443 1444 /* enable transmits */ 1445 netif_tx_start_all_queues(netdev); 1446 1447 ixgbevf_save_reset_stats(adapter); 1448 ixgbevf_init_last_counter_stats(adapter); 1449 1450 hw->mac.get_link_status = 1; 1451 mod_timer(&adapter->watchdog_timer, jiffies); 1452 } 1453 1454 static int ixgbevf_reset_queues(struct ixgbevf_adapter *adapter) 1455 { 1456 struct ixgbe_hw *hw = &adapter->hw; 1457 struct ixgbevf_ring *rx_ring; 1458 unsigned int def_q = 0; 1459 unsigned int num_tcs = 0; 1460 unsigned int num_rx_queues = 1; 1461 int err, i; 1462 1463 spin_lock_bh(&adapter->mbx_lock); 1464 1465 /* fetch queue configuration from the PF */ 1466 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 1467 1468 spin_unlock_bh(&adapter->mbx_lock); 1469 1470 if (err) 1471 return err; 1472 1473 if (num_tcs > 1) { 1474 /* update default Tx ring register index */ 1475 adapter->tx_ring[0].reg_idx = def_q; 1476 1477 /* we need as many queues as traffic classes */ 1478 num_rx_queues = num_tcs; 1479 } 1480 1481 /* nothing to do if we have the correct number of queues */ 1482 if (adapter->num_rx_queues == num_rx_queues) 1483 return 0; 1484 1485 /* allocate new rings */ 1486 rx_ring = kcalloc(num_rx_queues, 1487 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1488 if (!rx_ring) 1489 return -ENOMEM; 1490 1491 /* setup ring fields */ 1492 for (i = 0; i < num_rx_queues; i++) { 1493 rx_ring[i].count = adapter->rx_ring_count; 1494 rx_ring[i].queue_index = i; 1495 rx_ring[i].reg_idx = i; 1496 rx_ring[i].dev = &adapter->pdev->dev; 1497 rx_ring[i].netdev = adapter->netdev; 1498 1499 /* allocate resources on the ring */ 1500 err = ixgbevf_setup_rx_resources(adapter, &rx_ring[i]); 1501 if (err) { 1502 while (i) { 1503 i--; 1504 ixgbevf_free_rx_resources(adapter, &rx_ring[i]); 1505 } 1506 kfree(rx_ring); 1507 return err; 1508 } 1509 } 1510 1511 /* free the existing rings and queues */ 1512 ixgbevf_free_all_rx_resources(adapter); 1513 adapter->num_rx_queues = 0; 1514 kfree(adapter->rx_ring); 1515 1516 /* move new rings into position on the adapter struct */ 1517 adapter->rx_ring = rx_ring; 1518 adapter->num_rx_queues = num_rx_queues; 1519 1520 /* reset ring to vector mapping */ 1521 ixgbevf_reset_q_vectors(adapter); 1522 ixgbevf_map_rings_to_vectors(adapter); 1523 1524 return 0; 1525 } 1526 1527 void ixgbevf_up(struct ixgbevf_adapter *adapter) 1528 { 1529 struct ixgbe_hw *hw = &adapter->hw; 1530 1531 ixgbevf_negotiate_api(adapter); 1532 1533 ixgbevf_reset_queues(adapter); 1534 1535 ixgbevf_configure(adapter); 1536 1537 ixgbevf_up_complete(adapter); 1538 1539 /* clear any pending interrupts, may auto mask */ 1540 IXGBE_READ_REG(hw, IXGBE_VTEICR); 1541 1542 ixgbevf_irq_enable(adapter); 1543 } 1544 1545 /** 1546 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 1547 * @adapter: board private structure 1548 * @rx_ring: ring to free buffers from 1549 **/ 1550 static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter, 1551 struct ixgbevf_ring *rx_ring) 1552 { 1553 struct pci_dev *pdev = adapter->pdev; 1554 unsigned long size; 1555 unsigned int i; 1556 1557 if (!rx_ring->rx_buffer_info) 1558 return; 1559 1560 /* Free all the Rx ring sk_buffs */ 1561 for (i = 0; i < rx_ring->count; i++) { 1562 struct ixgbevf_rx_buffer *rx_buffer_info; 1563 1564 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 1565 if (rx_buffer_info->dma) { 1566 dma_unmap_single(&pdev->dev, rx_buffer_info->dma, 1567 rx_ring->rx_buf_len, 1568 DMA_FROM_DEVICE); 1569 rx_buffer_info->dma = 0; 1570 } 1571 if (rx_buffer_info->skb) { 1572 struct sk_buff *skb = rx_buffer_info->skb; 1573 rx_buffer_info->skb = NULL; 1574 do { 1575 struct sk_buff *this = skb; 1576 skb = IXGBE_CB(skb)->prev; 1577 dev_kfree_skb(this); 1578 } while (skb); 1579 } 1580 } 1581 1582 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 1583 memset(rx_ring->rx_buffer_info, 0, size); 1584 1585 /* Zero out the descriptor ring */ 1586 memset(rx_ring->desc, 0, rx_ring->size); 1587 1588 rx_ring->next_to_clean = 0; 1589 rx_ring->next_to_use = 0; 1590 1591 if (rx_ring->head) 1592 writel(0, adapter->hw.hw_addr + rx_ring->head); 1593 if (rx_ring->tail) 1594 writel(0, adapter->hw.hw_addr + rx_ring->tail); 1595 } 1596 1597 /** 1598 * ixgbevf_clean_tx_ring - Free Tx Buffers 1599 * @adapter: board private structure 1600 * @tx_ring: ring to be cleaned 1601 **/ 1602 static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter, 1603 struct ixgbevf_ring *tx_ring) 1604 { 1605 struct ixgbevf_tx_buffer *tx_buffer_info; 1606 unsigned long size; 1607 unsigned int i; 1608 1609 if (!tx_ring->tx_buffer_info) 1610 return; 1611 1612 /* Free all the Tx ring sk_buffs */ 1613 for (i = 0; i < tx_ring->count; i++) { 1614 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 1615 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 1616 } 1617 1618 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 1619 memset(tx_ring->tx_buffer_info, 0, size); 1620 1621 memset(tx_ring->desc, 0, tx_ring->size); 1622 1623 tx_ring->next_to_use = 0; 1624 tx_ring->next_to_clean = 0; 1625 1626 if (tx_ring->head) 1627 writel(0, adapter->hw.hw_addr + tx_ring->head); 1628 if (tx_ring->tail) 1629 writel(0, adapter->hw.hw_addr + tx_ring->tail); 1630 } 1631 1632 /** 1633 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 1634 * @adapter: board private structure 1635 **/ 1636 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 1637 { 1638 int i; 1639 1640 for (i = 0; i < adapter->num_rx_queues; i++) 1641 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]); 1642 } 1643 1644 /** 1645 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 1646 * @adapter: board private structure 1647 **/ 1648 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 1649 { 1650 int i; 1651 1652 for (i = 0; i < adapter->num_tx_queues; i++) 1653 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]); 1654 } 1655 1656 void ixgbevf_down(struct ixgbevf_adapter *adapter) 1657 { 1658 struct net_device *netdev = adapter->netdev; 1659 struct ixgbe_hw *hw = &adapter->hw; 1660 u32 txdctl; 1661 int i, j; 1662 1663 /* signal that we are down to the interrupt handler */ 1664 set_bit(__IXGBEVF_DOWN, &adapter->state); 1665 /* disable receives */ 1666 1667 netif_tx_disable(netdev); 1668 1669 msleep(10); 1670 1671 netif_tx_stop_all_queues(netdev); 1672 1673 ixgbevf_irq_disable(adapter); 1674 1675 ixgbevf_napi_disable_all(adapter); 1676 1677 del_timer_sync(&adapter->watchdog_timer); 1678 /* can't call flush scheduled work here because it can deadlock 1679 * if linkwatch_event tries to acquire the rtnl_lock which we are 1680 * holding */ 1681 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK) 1682 msleep(1); 1683 1684 /* disable transmits in the hardware now that interrupts are off */ 1685 for (i = 0; i < adapter->num_tx_queues; i++) { 1686 j = adapter->tx_ring[i].reg_idx; 1687 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1688 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), 1689 (txdctl & ~IXGBE_TXDCTL_ENABLE)); 1690 } 1691 1692 netif_carrier_off(netdev); 1693 1694 if (!pci_channel_offline(adapter->pdev)) 1695 ixgbevf_reset(adapter); 1696 1697 ixgbevf_clean_all_tx_rings(adapter); 1698 ixgbevf_clean_all_rx_rings(adapter); 1699 } 1700 1701 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 1702 { 1703 WARN_ON(in_interrupt()); 1704 1705 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 1706 msleep(1); 1707 1708 ixgbevf_down(adapter); 1709 ixgbevf_up(adapter); 1710 1711 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 1712 } 1713 1714 void ixgbevf_reset(struct ixgbevf_adapter *adapter) 1715 { 1716 struct ixgbe_hw *hw = &adapter->hw; 1717 struct net_device *netdev = adapter->netdev; 1718 1719 if (hw->mac.ops.reset_hw(hw)) 1720 hw_dbg(hw, "PF still resetting\n"); 1721 else 1722 hw->mac.ops.init_hw(hw); 1723 1724 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 1725 memcpy(netdev->dev_addr, adapter->hw.mac.addr, 1726 netdev->addr_len); 1727 memcpy(netdev->perm_addr, adapter->hw.mac.addr, 1728 netdev->addr_len); 1729 } 1730 } 1731 1732 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 1733 int vectors) 1734 { 1735 int err = 0; 1736 int vector_threshold; 1737 1738 /* We'll want at least 2 (vector_threshold): 1739 * 1) TxQ[0] + RxQ[0] handler 1740 * 2) Other (Link Status Change, etc.) 1741 */ 1742 vector_threshold = MIN_MSIX_COUNT; 1743 1744 /* The more we get, the more we will assign to Tx/Rx Cleanup 1745 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 1746 * Right now, we simply care about how many we'll get; we'll 1747 * set them up later while requesting irq's. 1748 */ 1749 while (vectors >= vector_threshold) { 1750 err = pci_enable_msix(adapter->pdev, adapter->msix_entries, 1751 vectors); 1752 if (!err || err < 0) /* Success or a nasty failure. */ 1753 break; 1754 else /* err == number of vectors we should try again with */ 1755 vectors = err; 1756 } 1757 1758 if (vectors < vector_threshold) 1759 err = -ENOMEM; 1760 1761 if (err) { 1762 dev_err(&adapter->pdev->dev, 1763 "Unable to allocate MSI-X interrupts\n"); 1764 kfree(adapter->msix_entries); 1765 adapter->msix_entries = NULL; 1766 } else { 1767 /* 1768 * Adjust for only the vectors we'll use, which is minimum 1769 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 1770 * vectors we were allocated. 1771 */ 1772 adapter->num_msix_vectors = vectors; 1773 } 1774 1775 return err; 1776 } 1777 1778 /** 1779 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent 1780 * @adapter: board private structure to initialize 1781 * 1782 * This is the top level queue allocation routine. The order here is very 1783 * important, starting with the "most" number of features turned on at once, 1784 * and ending with the smallest set of features. This way large combinations 1785 * can be allocated if they're turned on, and smaller combinations are the 1786 * fallthrough conditions. 1787 * 1788 **/ 1789 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 1790 { 1791 /* Start with base case */ 1792 adapter->num_rx_queues = 1; 1793 adapter->num_tx_queues = 1; 1794 } 1795 1796 /** 1797 * ixgbevf_alloc_queues - Allocate memory for all rings 1798 * @adapter: board private structure to initialize 1799 * 1800 * We allocate one ring per queue at run-time since we don't know the 1801 * number of queues at compile-time. The polling_netdev array is 1802 * intended for Multiqueue, but should work fine with a single queue. 1803 **/ 1804 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) 1805 { 1806 int i; 1807 1808 adapter->tx_ring = kcalloc(adapter->num_tx_queues, 1809 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1810 if (!adapter->tx_ring) 1811 goto err_tx_ring_allocation; 1812 1813 adapter->rx_ring = kcalloc(adapter->num_rx_queues, 1814 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1815 if (!adapter->rx_ring) 1816 goto err_rx_ring_allocation; 1817 1818 for (i = 0; i < adapter->num_tx_queues; i++) { 1819 adapter->tx_ring[i].count = adapter->tx_ring_count; 1820 adapter->tx_ring[i].queue_index = i; 1821 /* reg_idx may be remapped later by DCB config */ 1822 adapter->tx_ring[i].reg_idx = i; 1823 adapter->tx_ring[i].dev = &adapter->pdev->dev; 1824 adapter->tx_ring[i].netdev = adapter->netdev; 1825 } 1826 1827 for (i = 0; i < adapter->num_rx_queues; i++) { 1828 adapter->rx_ring[i].count = adapter->rx_ring_count; 1829 adapter->rx_ring[i].queue_index = i; 1830 adapter->rx_ring[i].reg_idx = i; 1831 adapter->rx_ring[i].dev = &adapter->pdev->dev; 1832 adapter->rx_ring[i].netdev = adapter->netdev; 1833 } 1834 1835 return 0; 1836 1837 err_rx_ring_allocation: 1838 kfree(adapter->tx_ring); 1839 err_tx_ring_allocation: 1840 return -ENOMEM; 1841 } 1842 1843 /** 1844 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 1845 * @adapter: board private structure to initialize 1846 * 1847 * Attempt to configure the interrupts using the best available 1848 * capabilities of the hardware and the kernel. 1849 **/ 1850 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 1851 { 1852 struct net_device *netdev = adapter->netdev; 1853 int err = 0; 1854 int vector, v_budget; 1855 1856 /* 1857 * It's easy to be greedy for MSI-X vectors, but it really 1858 * doesn't do us much good if we have a lot more vectors 1859 * than CPU's. So let's be conservative and only ask for 1860 * (roughly) the same number of vectors as there are CPU's. 1861 * The default is to use pairs of vectors. 1862 */ 1863 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); 1864 v_budget = min_t(int, v_budget, num_online_cpus()); 1865 v_budget += NON_Q_VECTORS; 1866 1867 /* A failure in MSI-X entry allocation isn't fatal, but it does 1868 * mean we disable MSI-X capabilities of the adapter. */ 1869 adapter->msix_entries = kcalloc(v_budget, 1870 sizeof(struct msix_entry), GFP_KERNEL); 1871 if (!adapter->msix_entries) { 1872 err = -ENOMEM; 1873 goto out; 1874 } 1875 1876 for (vector = 0; vector < v_budget; vector++) 1877 adapter->msix_entries[vector].entry = vector; 1878 1879 err = ixgbevf_acquire_msix_vectors(adapter, v_budget); 1880 if (err) 1881 goto out; 1882 1883 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 1884 if (err) 1885 goto out; 1886 1887 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 1888 1889 out: 1890 return err; 1891 } 1892 1893 /** 1894 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 1895 * @adapter: board private structure to initialize 1896 * 1897 * We allocate one q_vector per queue interrupt. If allocation fails we 1898 * return -ENOMEM. 1899 **/ 1900 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 1901 { 1902 int q_idx, num_q_vectors; 1903 struct ixgbevf_q_vector *q_vector; 1904 1905 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1906 1907 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1908 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL); 1909 if (!q_vector) 1910 goto err_out; 1911 q_vector->adapter = adapter; 1912 q_vector->v_idx = q_idx; 1913 netif_napi_add(adapter->netdev, &q_vector->napi, 1914 ixgbevf_poll, 64); 1915 adapter->q_vector[q_idx] = q_vector; 1916 } 1917 1918 return 0; 1919 1920 err_out: 1921 while (q_idx) { 1922 q_idx--; 1923 q_vector = adapter->q_vector[q_idx]; 1924 netif_napi_del(&q_vector->napi); 1925 kfree(q_vector); 1926 adapter->q_vector[q_idx] = NULL; 1927 } 1928 return -ENOMEM; 1929 } 1930 1931 /** 1932 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 1933 * @adapter: board private structure to initialize 1934 * 1935 * This function frees the memory allocated to the q_vectors. In addition if 1936 * NAPI is enabled it will delete any references to the NAPI struct prior 1937 * to freeing the q_vector. 1938 **/ 1939 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 1940 { 1941 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1942 1943 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1944 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; 1945 1946 adapter->q_vector[q_idx] = NULL; 1947 netif_napi_del(&q_vector->napi); 1948 kfree(q_vector); 1949 } 1950 } 1951 1952 /** 1953 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 1954 * @adapter: board private structure 1955 * 1956 **/ 1957 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 1958 { 1959 pci_disable_msix(adapter->pdev); 1960 kfree(adapter->msix_entries); 1961 adapter->msix_entries = NULL; 1962 } 1963 1964 /** 1965 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 1966 * @adapter: board private structure to initialize 1967 * 1968 **/ 1969 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 1970 { 1971 int err; 1972 1973 /* Number of supported queues */ 1974 ixgbevf_set_num_queues(adapter); 1975 1976 err = ixgbevf_set_interrupt_capability(adapter); 1977 if (err) { 1978 hw_dbg(&adapter->hw, 1979 "Unable to setup interrupt capabilities\n"); 1980 goto err_set_interrupt; 1981 } 1982 1983 err = ixgbevf_alloc_q_vectors(adapter); 1984 if (err) { 1985 hw_dbg(&adapter->hw, "Unable to allocate memory for queue " 1986 "vectors\n"); 1987 goto err_alloc_q_vectors; 1988 } 1989 1990 err = ixgbevf_alloc_queues(adapter); 1991 if (err) { 1992 pr_err("Unable to allocate memory for queues\n"); 1993 goto err_alloc_queues; 1994 } 1995 1996 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, " 1997 "Tx Queue count = %u\n", 1998 (adapter->num_rx_queues > 1) ? "Enabled" : 1999 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues); 2000 2001 set_bit(__IXGBEVF_DOWN, &adapter->state); 2002 2003 return 0; 2004 err_alloc_queues: 2005 ixgbevf_free_q_vectors(adapter); 2006 err_alloc_q_vectors: 2007 ixgbevf_reset_interrupt_capability(adapter); 2008 err_set_interrupt: 2009 return err; 2010 } 2011 2012 /** 2013 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings 2014 * @adapter: board private structure to clear interrupt scheme on 2015 * 2016 * We go through and clear interrupt specific resources and reset the structure 2017 * to pre-load conditions 2018 **/ 2019 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) 2020 { 2021 adapter->num_tx_queues = 0; 2022 adapter->num_rx_queues = 0; 2023 2024 ixgbevf_free_q_vectors(adapter); 2025 ixgbevf_reset_interrupt_capability(adapter); 2026 } 2027 2028 /** 2029 * ixgbevf_sw_init - Initialize general software structures 2030 * (struct ixgbevf_adapter) 2031 * @adapter: board private structure to initialize 2032 * 2033 * ixgbevf_sw_init initializes the Adapter private data structure. 2034 * Fields are initialized based on PCI device information and 2035 * OS network device settings (MTU size). 2036 **/ 2037 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 2038 { 2039 struct ixgbe_hw *hw = &adapter->hw; 2040 struct pci_dev *pdev = adapter->pdev; 2041 int err; 2042 2043 /* PCI config space info */ 2044 2045 hw->vendor_id = pdev->vendor; 2046 hw->device_id = pdev->device; 2047 hw->revision_id = pdev->revision; 2048 hw->subsystem_vendor_id = pdev->subsystem_vendor; 2049 hw->subsystem_device_id = pdev->subsystem_device; 2050 2051 hw->mbx.ops.init_params(hw); 2052 2053 /* assume legacy case in which PF would only give VF 2 queues */ 2054 hw->mac.max_tx_queues = 2; 2055 hw->mac.max_rx_queues = 2; 2056 2057 err = hw->mac.ops.reset_hw(hw); 2058 if (err) { 2059 dev_info(&pdev->dev, 2060 "PF still in reset state, assigning new address\n"); 2061 eth_hw_addr_random(adapter->netdev); 2062 memcpy(adapter->hw.mac.addr, adapter->netdev->dev_addr, 2063 adapter->netdev->addr_len); 2064 } else { 2065 err = hw->mac.ops.init_hw(hw); 2066 if (err) { 2067 pr_err("init_shared_code failed: %d\n", err); 2068 goto out; 2069 } 2070 memcpy(adapter->netdev->dev_addr, adapter->hw.mac.addr, 2071 adapter->netdev->addr_len); 2072 } 2073 2074 /* lock to protect mailbox accesses */ 2075 spin_lock_init(&adapter->mbx_lock); 2076 2077 /* Enable dynamic interrupt throttling rates */ 2078 adapter->rx_itr_setting = 1; 2079 adapter->tx_itr_setting = 1; 2080 2081 /* set default ring sizes */ 2082 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 2083 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 2084 2085 set_bit(__IXGBEVF_DOWN, &adapter->state); 2086 return 0; 2087 2088 out: 2089 return err; 2090 } 2091 2092 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 2093 { \ 2094 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 2095 if (current_counter < last_counter) \ 2096 counter += 0x100000000LL; \ 2097 last_counter = current_counter; \ 2098 counter &= 0xFFFFFFFF00000000LL; \ 2099 counter |= current_counter; \ 2100 } 2101 2102 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 2103 { \ 2104 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 2105 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 2106 u64 current_counter = (current_counter_msb << 32) | \ 2107 current_counter_lsb; \ 2108 if (current_counter < last_counter) \ 2109 counter += 0x1000000000LL; \ 2110 last_counter = current_counter; \ 2111 counter &= 0xFFFFFFF000000000LL; \ 2112 counter |= current_counter; \ 2113 } 2114 /** 2115 * ixgbevf_update_stats - Update the board statistics counters. 2116 * @adapter: board private structure 2117 **/ 2118 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 2119 { 2120 struct ixgbe_hw *hw = &adapter->hw; 2121 int i; 2122 2123 if (!adapter->link_up) 2124 return; 2125 2126 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 2127 adapter->stats.vfgprc); 2128 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 2129 adapter->stats.vfgptc); 2130 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2131 adapter->stats.last_vfgorc, 2132 adapter->stats.vfgorc); 2133 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2134 adapter->stats.last_vfgotc, 2135 adapter->stats.vfgotc); 2136 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 2137 adapter->stats.vfmprc); 2138 2139 for (i = 0; i < adapter->num_rx_queues; i++) { 2140 adapter->hw_csum_rx_error += 2141 adapter->rx_ring[i].hw_csum_rx_error; 2142 adapter->hw_csum_rx_good += 2143 adapter->rx_ring[i].hw_csum_rx_good; 2144 adapter->rx_ring[i].hw_csum_rx_error = 0; 2145 adapter->rx_ring[i].hw_csum_rx_good = 0; 2146 } 2147 } 2148 2149 /** 2150 * ixgbevf_watchdog - Timer Call-back 2151 * @data: pointer to adapter cast into an unsigned long 2152 **/ 2153 static void ixgbevf_watchdog(unsigned long data) 2154 { 2155 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data; 2156 struct ixgbe_hw *hw = &adapter->hw; 2157 u32 eics = 0; 2158 int i; 2159 2160 /* 2161 * Do the watchdog outside of interrupt context due to the lovely 2162 * delays that some of the newer hardware requires 2163 */ 2164 2165 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 2166 goto watchdog_short_circuit; 2167 2168 /* get one bit for every active tx/rx interrupt vector */ 2169 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 2170 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 2171 if (qv->rx.ring || qv->tx.ring) 2172 eics |= 1 << i; 2173 } 2174 2175 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics); 2176 2177 watchdog_short_circuit: 2178 schedule_work(&adapter->watchdog_task); 2179 } 2180 2181 /** 2182 * ixgbevf_tx_timeout - Respond to a Tx Hang 2183 * @netdev: network interface device structure 2184 **/ 2185 static void ixgbevf_tx_timeout(struct net_device *netdev) 2186 { 2187 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2188 2189 /* Do the reset outside of interrupt context */ 2190 schedule_work(&adapter->reset_task); 2191 } 2192 2193 static void ixgbevf_reset_task(struct work_struct *work) 2194 { 2195 struct ixgbevf_adapter *adapter; 2196 adapter = container_of(work, struct ixgbevf_adapter, reset_task); 2197 2198 /* If we're already down or resetting, just bail */ 2199 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2200 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2201 return; 2202 2203 adapter->tx_timeout_count++; 2204 2205 ixgbevf_reinit_locked(adapter); 2206 } 2207 2208 /** 2209 * ixgbevf_watchdog_task - worker thread to bring link up 2210 * @work: pointer to work_struct containing our data 2211 **/ 2212 static void ixgbevf_watchdog_task(struct work_struct *work) 2213 { 2214 struct ixgbevf_adapter *adapter = container_of(work, 2215 struct ixgbevf_adapter, 2216 watchdog_task); 2217 struct net_device *netdev = adapter->netdev; 2218 struct ixgbe_hw *hw = &adapter->hw; 2219 u32 link_speed = adapter->link_speed; 2220 bool link_up = adapter->link_up; 2221 s32 need_reset; 2222 2223 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK; 2224 2225 /* 2226 * Always check the link on the watchdog because we have 2227 * no LSC interrupt 2228 */ 2229 spin_lock_bh(&adapter->mbx_lock); 2230 2231 need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 2232 2233 spin_unlock_bh(&adapter->mbx_lock); 2234 2235 if (need_reset) { 2236 adapter->link_up = link_up; 2237 adapter->link_speed = link_speed; 2238 netif_carrier_off(netdev); 2239 netif_tx_stop_all_queues(netdev); 2240 schedule_work(&adapter->reset_task); 2241 goto pf_has_reset; 2242 } 2243 adapter->link_up = link_up; 2244 adapter->link_speed = link_speed; 2245 2246 if (link_up) { 2247 if (!netif_carrier_ok(netdev)) { 2248 char *link_speed_string; 2249 switch (link_speed) { 2250 case IXGBE_LINK_SPEED_10GB_FULL: 2251 link_speed_string = "10 Gbps"; 2252 break; 2253 case IXGBE_LINK_SPEED_1GB_FULL: 2254 link_speed_string = "1 Gbps"; 2255 break; 2256 case IXGBE_LINK_SPEED_100_FULL: 2257 link_speed_string = "100 Mbps"; 2258 break; 2259 default: 2260 link_speed_string = "unknown speed"; 2261 break; 2262 } 2263 dev_info(&adapter->pdev->dev, 2264 "NIC Link is Up, %s\n", link_speed_string); 2265 netif_carrier_on(netdev); 2266 netif_tx_wake_all_queues(netdev); 2267 } 2268 } else { 2269 adapter->link_up = false; 2270 adapter->link_speed = 0; 2271 if (netif_carrier_ok(netdev)) { 2272 dev_info(&adapter->pdev->dev, "NIC Link is Down\n"); 2273 netif_carrier_off(netdev); 2274 netif_tx_stop_all_queues(netdev); 2275 } 2276 } 2277 2278 ixgbevf_update_stats(adapter); 2279 2280 pf_has_reset: 2281 /* Reset the timer */ 2282 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 2283 mod_timer(&adapter->watchdog_timer, 2284 round_jiffies(jiffies + (2 * HZ))); 2285 2286 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK; 2287 } 2288 2289 /** 2290 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 2291 * @adapter: board private structure 2292 * @tx_ring: Tx descriptor ring for a specific queue 2293 * 2294 * Free all transmit software resources 2295 **/ 2296 void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter, 2297 struct ixgbevf_ring *tx_ring) 2298 { 2299 struct pci_dev *pdev = adapter->pdev; 2300 2301 ixgbevf_clean_tx_ring(adapter, tx_ring); 2302 2303 vfree(tx_ring->tx_buffer_info); 2304 tx_ring->tx_buffer_info = NULL; 2305 2306 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc, 2307 tx_ring->dma); 2308 2309 tx_ring->desc = NULL; 2310 } 2311 2312 /** 2313 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 2314 * @adapter: board private structure 2315 * 2316 * Free all transmit software resources 2317 **/ 2318 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 2319 { 2320 int i; 2321 2322 for (i = 0; i < adapter->num_tx_queues; i++) 2323 if (adapter->tx_ring[i].desc) 2324 ixgbevf_free_tx_resources(adapter, 2325 &adapter->tx_ring[i]); 2326 2327 } 2328 2329 /** 2330 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 2331 * @adapter: board private structure 2332 * @tx_ring: tx descriptor ring (for a specific queue) to setup 2333 * 2334 * Return 0 on success, negative on failure 2335 **/ 2336 int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter, 2337 struct ixgbevf_ring *tx_ring) 2338 { 2339 struct pci_dev *pdev = adapter->pdev; 2340 int size; 2341 2342 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2343 tx_ring->tx_buffer_info = vzalloc(size); 2344 if (!tx_ring->tx_buffer_info) 2345 goto err; 2346 2347 /* round up to nearest 4K */ 2348 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 2349 tx_ring->size = ALIGN(tx_ring->size, 4096); 2350 2351 tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size, 2352 &tx_ring->dma, GFP_KERNEL); 2353 if (!tx_ring->desc) 2354 goto err; 2355 2356 tx_ring->next_to_use = 0; 2357 tx_ring->next_to_clean = 0; 2358 return 0; 2359 2360 err: 2361 vfree(tx_ring->tx_buffer_info); 2362 tx_ring->tx_buffer_info = NULL; 2363 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit " 2364 "descriptor ring\n"); 2365 return -ENOMEM; 2366 } 2367 2368 /** 2369 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 2370 * @adapter: board private structure 2371 * 2372 * If this function returns with an error, then it's possible one or 2373 * more of the rings is populated (while the rest are not). It is the 2374 * callers duty to clean those orphaned rings. 2375 * 2376 * Return 0 on success, negative on failure 2377 **/ 2378 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 2379 { 2380 int i, err = 0; 2381 2382 for (i = 0; i < adapter->num_tx_queues; i++) { 2383 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]); 2384 if (!err) 2385 continue; 2386 hw_dbg(&adapter->hw, 2387 "Allocation for Tx Queue %u failed\n", i); 2388 break; 2389 } 2390 2391 return err; 2392 } 2393 2394 /** 2395 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 2396 * @adapter: board private structure 2397 * @rx_ring: rx descriptor ring (for a specific queue) to setup 2398 * 2399 * Returns 0 on success, negative on failure 2400 **/ 2401 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, 2402 struct ixgbevf_ring *rx_ring) 2403 { 2404 struct pci_dev *pdev = adapter->pdev; 2405 int size; 2406 2407 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 2408 rx_ring->rx_buffer_info = vzalloc(size); 2409 if (!rx_ring->rx_buffer_info) 2410 goto alloc_failed; 2411 2412 /* Round up to nearest 4K */ 2413 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 2414 rx_ring->size = ALIGN(rx_ring->size, 4096); 2415 2416 rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size, 2417 &rx_ring->dma, GFP_KERNEL); 2418 2419 if (!rx_ring->desc) { 2420 hw_dbg(&adapter->hw, 2421 "Unable to allocate memory for " 2422 "the receive descriptor ring\n"); 2423 vfree(rx_ring->rx_buffer_info); 2424 rx_ring->rx_buffer_info = NULL; 2425 goto alloc_failed; 2426 } 2427 2428 rx_ring->next_to_clean = 0; 2429 rx_ring->next_to_use = 0; 2430 2431 return 0; 2432 alloc_failed: 2433 return -ENOMEM; 2434 } 2435 2436 /** 2437 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 2438 * @adapter: board private structure 2439 * 2440 * If this function returns with an error, then it's possible one or 2441 * more of the rings is populated (while the rest are not). It is the 2442 * callers duty to clean those orphaned rings. 2443 * 2444 * Return 0 on success, negative on failure 2445 **/ 2446 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 2447 { 2448 int i, err = 0; 2449 2450 for (i = 0; i < adapter->num_rx_queues; i++) { 2451 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]); 2452 if (!err) 2453 continue; 2454 hw_dbg(&adapter->hw, 2455 "Allocation for Rx Queue %u failed\n", i); 2456 break; 2457 } 2458 return err; 2459 } 2460 2461 /** 2462 * ixgbevf_free_rx_resources - Free Rx Resources 2463 * @adapter: board private structure 2464 * @rx_ring: ring to clean the resources from 2465 * 2466 * Free all receive software resources 2467 **/ 2468 void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter, 2469 struct ixgbevf_ring *rx_ring) 2470 { 2471 struct pci_dev *pdev = adapter->pdev; 2472 2473 ixgbevf_clean_rx_ring(adapter, rx_ring); 2474 2475 vfree(rx_ring->rx_buffer_info); 2476 rx_ring->rx_buffer_info = NULL; 2477 2478 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, 2479 rx_ring->dma); 2480 2481 rx_ring->desc = NULL; 2482 } 2483 2484 /** 2485 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 2486 * @adapter: board private structure 2487 * 2488 * Free all receive software resources 2489 **/ 2490 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 2491 { 2492 int i; 2493 2494 for (i = 0; i < adapter->num_rx_queues; i++) 2495 if (adapter->rx_ring[i].desc) 2496 ixgbevf_free_rx_resources(adapter, 2497 &adapter->rx_ring[i]); 2498 } 2499 2500 static int ixgbevf_setup_queues(struct ixgbevf_adapter *adapter) 2501 { 2502 struct ixgbe_hw *hw = &adapter->hw; 2503 struct ixgbevf_ring *rx_ring; 2504 unsigned int def_q = 0; 2505 unsigned int num_tcs = 0; 2506 unsigned int num_rx_queues = 1; 2507 int err, i; 2508 2509 spin_lock_bh(&adapter->mbx_lock); 2510 2511 /* fetch queue configuration from the PF */ 2512 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2513 2514 spin_unlock_bh(&adapter->mbx_lock); 2515 2516 if (err) 2517 return err; 2518 2519 if (num_tcs > 1) { 2520 /* update default Tx ring register index */ 2521 adapter->tx_ring[0].reg_idx = def_q; 2522 2523 /* we need as many queues as traffic classes */ 2524 num_rx_queues = num_tcs; 2525 } 2526 2527 /* nothing to do if we have the correct number of queues */ 2528 if (adapter->num_rx_queues == num_rx_queues) 2529 return 0; 2530 2531 /* allocate new rings */ 2532 rx_ring = kcalloc(num_rx_queues, 2533 sizeof(struct ixgbevf_ring), GFP_KERNEL); 2534 if (!rx_ring) 2535 return -ENOMEM; 2536 2537 /* setup ring fields */ 2538 for (i = 0; i < num_rx_queues; i++) { 2539 rx_ring[i].count = adapter->rx_ring_count; 2540 rx_ring[i].queue_index = i; 2541 rx_ring[i].reg_idx = i; 2542 rx_ring[i].dev = &adapter->pdev->dev; 2543 rx_ring[i].netdev = adapter->netdev; 2544 } 2545 2546 /* free the existing ring and queues */ 2547 adapter->num_rx_queues = 0; 2548 kfree(adapter->rx_ring); 2549 2550 /* move new rings into position on the adapter struct */ 2551 adapter->rx_ring = rx_ring; 2552 adapter->num_rx_queues = num_rx_queues; 2553 2554 return 0; 2555 } 2556 2557 /** 2558 * ixgbevf_open - Called when a network interface is made active 2559 * @netdev: network interface device structure 2560 * 2561 * Returns 0 on success, negative value on failure 2562 * 2563 * The open entry point is called when a network interface is made 2564 * active by the system (IFF_UP). At this point all resources needed 2565 * for transmit and receive operations are allocated, the interrupt 2566 * handler is registered with the OS, the watchdog timer is started, 2567 * and the stack is notified that the interface is ready. 2568 **/ 2569 static int ixgbevf_open(struct net_device *netdev) 2570 { 2571 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2572 struct ixgbe_hw *hw = &adapter->hw; 2573 int err; 2574 2575 /* disallow open during test */ 2576 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 2577 return -EBUSY; 2578 2579 if (hw->adapter_stopped) { 2580 ixgbevf_reset(adapter); 2581 /* if adapter is still stopped then PF isn't up and 2582 * the vf can't start. */ 2583 if (hw->adapter_stopped) { 2584 err = IXGBE_ERR_MBX; 2585 pr_err("Unable to start - perhaps the PF Driver isn't " 2586 "up yet\n"); 2587 goto err_setup_reset; 2588 } 2589 } 2590 2591 ixgbevf_negotiate_api(adapter); 2592 2593 /* setup queue reg_idx and Rx queue count */ 2594 err = ixgbevf_setup_queues(adapter); 2595 if (err) 2596 goto err_setup_queues; 2597 2598 /* allocate transmit descriptors */ 2599 err = ixgbevf_setup_all_tx_resources(adapter); 2600 if (err) 2601 goto err_setup_tx; 2602 2603 /* allocate receive descriptors */ 2604 err = ixgbevf_setup_all_rx_resources(adapter); 2605 if (err) 2606 goto err_setup_rx; 2607 2608 ixgbevf_configure(adapter); 2609 2610 /* 2611 * Map the Tx/Rx rings to the vectors we were allotted. 2612 * if request_irq will be called in this function map_rings 2613 * must be called *before* up_complete 2614 */ 2615 ixgbevf_map_rings_to_vectors(adapter); 2616 2617 ixgbevf_up_complete(adapter); 2618 2619 /* clear any pending interrupts, may auto mask */ 2620 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2621 err = ixgbevf_request_irq(adapter); 2622 if (err) 2623 goto err_req_irq; 2624 2625 ixgbevf_irq_enable(adapter); 2626 2627 return 0; 2628 2629 err_req_irq: 2630 ixgbevf_down(adapter); 2631 ixgbevf_free_irq(adapter); 2632 err_setup_rx: 2633 ixgbevf_free_all_rx_resources(adapter); 2634 err_setup_tx: 2635 ixgbevf_free_all_tx_resources(adapter); 2636 err_setup_queues: 2637 ixgbevf_reset(adapter); 2638 2639 err_setup_reset: 2640 2641 return err; 2642 } 2643 2644 /** 2645 * ixgbevf_close - Disables a network interface 2646 * @netdev: network interface device structure 2647 * 2648 * Returns 0, this is not allowed to fail 2649 * 2650 * The close entry point is called when an interface is de-activated 2651 * by the OS. The hardware is still under the drivers control, but 2652 * needs to be disabled. A global MAC reset is issued to stop the 2653 * hardware, and all transmit and receive resources are freed. 2654 **/ 2655 static int ixgbevf_close(struct net_device *netdev) 2656 { 2657 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2658 2659 ixgbevf_down(adapter); 2660 ixgbevf_free_irq(adapter); 2661 2662 ixgbevf_free_all_tx_resources(adapter); 2663 ixgbevf_free_all_rx_resources(adapter); 2664 2665 return 0; 2666 } 2667 2668 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, 2669 u32 vlan_macip_lens, u32 type_tucmd, 2670 u32 mss_l4len_idx) 2671 { 2672 struct ixgbe_adv_tx_context_desc *context_desc; 2673 u16 i = tx_ring->next_to_use; 2674 2675 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i); 2676 2677 i++; 2678 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 2679 2680 /* set bits to identify this as an advanced context descriptor */ 2681 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2682 2683 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 2684 context_desc->seqnum_seed = 0; 2685 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 2686 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 2687 } 2688 2689 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, 2690 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len) 2691 { 2692 u32 vlan_macip_lens, type_tucmd; 2693 u32 mss_l4len_idx, l4len; 2694 2695 if (!skb_is_gso(skb)) 2696 return 0; 2697 2698 if (skb_header_cloned(skb)) { 2699 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2700 if (err) 2701 return err; 2702 } 2703 2704 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 2705 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 2706 2707 if (skb->protocol == htons(ETH_P_IP)) { 2708 struct iphdr *iph = ip_hdr(skb); 2709 iph->tot_len = 0; 2710 iph->check = 0; 2711 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 2712 iph->daddr, 0, 2713 IPPROTO_TCP, 2714 0); 2715 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 2716 } else if (skb_is_gso_v6(skb)) { 2717 ipv6_hdr(skb)->payload_len = 0; 2718 tcp_hdr(skb)->check = 2719 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 2720 &ipv6_hdr(skb)->daddr, 2721 0, IPPROTO_TCP, 0); 2722 } 2723 2724 /* compute header lengths */ 2725 l4len = tcp_hdrlen(skb); 2726 *hdr_len += l4len; 2727 *hdr_len = skb_transport_offset(skb) + l4len; 2728 2729 /* mss_l4len_id: use 1 as index for TSO */ 2730 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; 2731 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 2732 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT; 2733 2734 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 2735 vlan_macip_lens = skb_network_header_len(skb); 2736 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 2737 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 2738 2739 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 2740 type_tucmd, mss_l4len_idx); 2741 2742 return 1; 2743 } 2744 2745 static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, 2746 struct sk_buff *skb, u32 tx_flags) 2747 { 2748 u32 vlan_macip_lens = 0; 2749 u32 mss_l4len_idx = 0; 2750 u32 type_tucmd = 0; 2751 2752 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2753 u8 l4_hdr = 0; 2754 switch (skb->protocol) { 2755 case __constant_htons(ETH_P_IP): 2756 vlan_macip_lens |= skb_network_header_len(skb); 2757 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 2758 l4_hdr = ip_hdr(skb)->protocol; 2759 break; 2760 case __constant_htons(ETH_P_IPV6): 2761 vlan_macip_lens |= skb_network_header_len(skb); 2762 l4_hdr = ipv6_hdr(skb)->nexthdr; 2763 break; 2764 default: 2765 if (unlikely(net_ratelimit())) { 2766 dev_warn(tx_ring->dev, 2767 "partial checksum but proto=%x!\n", 2768 skb->protocol); 2769 } 2770 break; 2771 } 2772 2773 switch (l4_hdr) { 2774 case IPPROTO_TCP: 2775 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2776 mss_l4len_idx = tcp_hdrlen(skb) << 2777 IXGBE_ADVTXD_L4LEN_SHIFT; 2778 break; 2779 case IPPROTO_SCTP: 2780 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 2781 mss_l4len_idx = sizeof(struct sctphdr) << 2782 IXGBE_ADVTXD_L4LEN_SHIFT; 2783 break; 2784 case IPPROTO_UDP: 2785 mss_l4len_idx = sizeof(struct udphdr) << 2786 IXGBE_ADVTXD_L4LEN_SHIFT; 2787 break; 2788 default: 2789 if (unlikely(net_ratelimit())) { 2790 dev_warn(tx_ring->dev, 2791 "partial checksum but l4 proto=%x!\n", 2792 l4_hdr); 2793 } 2794 break; 2795 } 2796 } 2797 2798 /* vlan_macip_lens: MACLEN, VLAN tag */ 2799 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 2800 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 2801 2802 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 2803 type_tucmd, mss_l4len_idx); 2804 2805 return (skb->ip_summed == CHECKSUM_PARTIAL); 2806 } 2807 2808 static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, 2809 struct sk_buff *skb, u32 tx_flags, 2810 unsigned int first) 2811 { 2812 struct ixgbevf_tx_buffer *tx_buffer_info; 2813 unsigned int len; 2814 unsigned int total = skb->len; 2815 unsigned int offset = 0, size; 2816 int count = 0; 2817 unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 2818 unsigned int f; 2819 int i; 2820 2821 i = tx_ring->next_to_use; 2822 2823 len = min(skb_headlen(skb), total); 2824 while (len) { 2825 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2826 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); 2827 2828 tx_buffer_info->length = size; 2829 tx_buffer_info->mapped_as_page = false; 2830 tx_buffer_info->dma = dma_map_single(tx_ring->dev, 2831 skb->data + offset, 2832 size, DMA_TO_DEVICE); 2833 if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma)) 2834 goto dma_error; 2835 tx_buffer_info->next_to_watch = i; 2836 2837 len -= size; 2838 total -= size; 2839 offset += size; 2840 count++; 2841 i++; 2842 if (i == tx_ring->count) 2843 i = 0; 2844 } 2845 2846 for (f = 0; f < nr_frags; f++) { 2847 const struct skb_frag_struct *frag; 2848 2849 frag = &skb_shinfo(skb)->frags[f]; 2850 len = min((unsigned int)skb_frag_size(frag), total); 2851 offset = 0; 2852 2853 while (len) { 2854 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2855 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); 2856 2857 tx_buffer_info->length = size; 2858 tx_buffer_info->dma = 2859 skb_frag_dma_map(tx_ring->dev, frag, 2860 offset, size, DMA_TO_DEVICE); 2861 if (dma_mapping_error(tx_ring->dev, 2862 tx_buffer_info->dma)) 2863 goto dma_error; 2864 tx_buffer_info->mapped_as_page = true; 2865 tx_buffer_info->next_to_watch = i; 2866 2867 len -= size; 2868 total -= size; 2869 offset += size; 2870 count++; 2871 i++; 2872 if (i == tx_ring->count) 2873 i = 0; 2874 } 2875 if (total == 0) 2876 break; 2877 } 2878 2879 if (i == 0) 2880 i = tx_ring->count - 1; 2881 else 2882 i = i - 1; 2883 tx_ring->tx_buffer_info[i].skb = skb; 2884 tx_ring->tx_buffer_info[first].next_to_watch = i; 2885 tx_ring->tx_buffer_info[first].time_stamp = jiffies; 2886 2887 return count; 2888 2889 dma_error: 2890 dev_err(tx_ring->dev, "TX DMA map failed\n"); 2891 2892 /* clear timestamp and dma mappings for failed tx_buffer_info map */ 2893 tx_buffer_info->dma = 0; 2894 tx_buffer_info->next_to_watch = 0; 2895 count--; 2896 2897 /* clear timestamp and dma mappings for remaining portion of packet */ 2898 while (count >= 0) { 2899 count--; 2900 i--; 2901 if (i < 0) 2902 i += tx_ring->count; 2903 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2904 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 2905 } 2906 2907 return count; 2908 } 2909 2910 static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags, 2911 int count, u32 paylen, u8 hdr_len) 2912 { 2913 union ixgbe_adv_tx_desc *tx_desc = NULL; 2914 struct ixgbevf_tx_buffer *tx_buffer_info; 2915 u32 olinfo_status = 0, cmd_type_len = 0; 2916 unsigned int i; 2917 2918 u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS; 2919 2920 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; 2921 2922 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 2923 2924 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 2925 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 2926 2927 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 2928 olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM; 2929 2930 if (tx_flags & IXGBE_TX_FLAGS_TSO) { 2931 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 2932 2933 /* use index 1 context for tso */ 2934 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT); 2935 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 2936 olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM; 2937 } 2938 2939 /* 2940 * Check Context must be set if Tx switch is enabled, which it 2941 * always is for case where virtual functions are running 2942 */ 2943 olinfo_status |= IXGBE_ADVTXD_CC; 2944 2945 olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT); 2946 2947 i = tx_ring->next_to_use; 2948 while (count--) { 2949 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2950 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 2951 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma); 2952 tx_desc->read.cmd_type_len = 2953 cpu_to_le32(cmd_type_len | tx_buffer_info->length); 2954 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 2955 i++; 2956 if (i == tx_ring->count) 2957 i = 0; 2958 } 2959 2960 tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd); 2961 2962 tx_ring->next_to_use = i; 2963 } 2964 2965 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 2966 { 2967 struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); 2968 2969 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 2970 /* Herbert's original patch had: 2971 * smp_mb__after_netif_stop_queue(); 2972 * but since that doesn't exist yet, just open code it. */ 2973 smp_mb(); 2974 2975 /* We need to check again in a case another CPU has just 2976 * made room available. */ 2977 if (likely(IXGBE_DESC_UNUSED(tx_ring) < size)) 2978 return -EBUSY; 2979 2980 /* A reprieve! - use start_queue because it doesn't call schedule */ 2981 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 2982 ++adapter->restart_queue; 2983 return 0; 2984 } 2985 2986 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 2987 { 2988 if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size)) 2989 return 0; 2990 return __ixgbevf_maybe_stop_tx(tx_ring, size); 2991 } 2992 2993 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 2994 { 2995 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2996 struct ixgbevf_ring *tx_ring; 2997 unsigned int first; 2998 unsigned int tx_flags = 0; 2999 u8 hdr_len = 0; 3000 int r_idx = 0, tso; 3001 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 3002 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3003 unsigned short f; 3004 #endif 3005 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL); 3006 if (!dst_mac || is_link_local_ether_addr(dst_mac)) { 3007 dev_kfree_skb(skb); 3008 return NETDEV_TX_OK; 3009 } 3010 3011 tx_ring = &adapter->tx_ring[r_idx]; 3012 3013 /* 3014 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 3015 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 3016 * + 2 desc gap to keep tail from touching head, 3017 * + 1 desc for context descriptor, 3018 * otherwise try next time 3019 */ 3020 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3021 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 3022 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 3023 #else 3024 count += skb_shinfo(skb)->nr_frags; 3025 #endif 3026 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { 3027 adapter->tx_busy++; 3028 return NETDEV_TX_BUSY; 3029 } 3030 3031 if (vlan_tx_tag_present(skb)) { 3032 tx_flags |= vlan_tx_tag_get(skb); 3033 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 3034 tx_flags |= IXGBE_TX_FLAGS_VLAN; 3035 } 3036 3037 first = tx_ring->next_to_use; 3038 3039 if (skb->protocol == htons(ETH_P_IP)) 3040 tx_flags |= IXGBE_TX_FLAGS_IPV4; 3041 tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len); 3042 if (tso < 0) { 3043 dev_kfree_skb_any(skb); 3044 return NETDEV_TX_OK; 3045 } 3046 3047 if (tso) 3048 tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM; 3049 else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags)) 3050 tx_flags |= IXGBE_TX_FLAGS_CSUM; 3051 3052 ixgbevf_tx_queue(tx_ring, tx_flags, 3053 ixgbevf_tx_map(tx_ring, skb, tx_flags, first), 3054 skb->len, hdr_len); 3055 /* 3056 * Force memory writes to complete before letting h/w 3057 * know there are new descriptors to fetch. (Only 3058 * applicable for weak-ordered memory model archs, 3059 * such as IA-64). 3060 */ 3061 wmb(); 3062 3063 writel(tx_ring->next_to_use, adapter->hw.hw_addr + tx_ring->tail); 3064 3065 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); 3066 3067 return NETDEV_TX_OK; 3068 } 3069 3070 /** 3071 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 3072 * @netdev: network interface device structure 3073 * @p: pointer to an address structure 3074 * 3075 * Returns 0 on success, negative on failure 3076 **/ 3077 static int ixgbevf_set_mac(struct net_device *netdev, void *p) 3078 { 3079 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3080 struct ixgbe_hw *hw = &adapter->hw; 3081 struct sockaddr *addr = p; 3082 3083 if (!is_valid_ether_addr(addr->sa_data)) 3084 return -EADDRNOTAVAIL; 3085 3086 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 3087 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 3088 3089 spin_lock_bh(&adapter->mbx_lock); 3090 3091 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 3092 3093 spin_unlock_bh(&adapter->mbx_lock); 3094 3095 return 0; 3096 } 3097 3098 /** 3099 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 3100 * @netdev: network interface device structure 3101 * @new_mtu: new value for maximum frame size 3102 * 3103 * Returns 0 on success, negative on failure 3104 **/ 3105 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 3106 { 3107 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3108 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 3109 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; 3110 3111 switch (adapter->hw.api_version) { 3112 case ixgbe_mbox_api_11: 3113 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3114 break; 3115 default: 3116 if (adapter->hw.mac.type == ixgbe_mac_X540_vf) 3117 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3118 break; 3119 } 3120 3121 /* MTU < 68 is an error and causes problems on some kernels */ 3122 if ((new_mtu < 68) || (max_frame > max_possible_frame)) 3123 return -EINVAL; 3124 3125 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n", 3126 netdev->mtu, new_mtu); 3127 /* must set new MTU before calling down or up */ 3128 netdev->mtu = new_mtu; 3129 3130 if (netif_running(netdev)) 3131 ixgbevf_reinit_locked(adapter); 3132 3133 return 0; 3134 } 3135 3136 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) 3137 { 3138 struct net_device *netdev = pci_get_drvdata(pdev); 3139 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3140 #ifdef CONFIG_PM 3141 int retval = 0; 3142 #endif 3143 3144 netif_device_detach(netdev); 3145 3146 if (netif_running(netdev)) { 3147 rtnl_lock(); 3148 ixgbevf_down(adapter); 3149 ixgbevf_free_irq(adapter); 3150 ixgbevf_free_all_tx_resources(adapter); 3151 ixgbevf_free_all_rx_resources(adapter); 3152 rtnl_unlock(); 3153 } 3154 3155 ixgbevf_clear_interrupt_scheme(adapter); 3156 3157 #ifdef CONFIG_PM 3158 retval = pci_save_state(pdev); 3159 if (retval) 3160 return retval; 3161 3162 #endif 3163 pci_disable_device(pdev); 3164 3165 return 0; 3166 } 3167 3168 #ifdef CONFIG_PM 3169 static int ixgbevf_resume(struct pci_dev *pdev) 3170 { 3171 struct ixgbevf_adapter *adapter = pci_get_drvdata(pdev); 3172 struct net_device *netdev = adapter->netdev; 3173 u32 err; 3174 3175 pci_set_power_state(pdev, PCI_D0); 3176 pci_restore_state(pdev); 3177 /* 3178 * pci_restore_state clears dev->state_saved so call 3179 * pci_save_state to restore it. 3180 */ 3181 pci_save_state(pdev); 3182 3183 err = pci_enable_device_mem(pdev); 3184 if (err) { 3185 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 3186 return err; 3187 } 3188 pci_set_master(pdev); 3189 3190 rtnl_lock(); 3191 err = ixgbevf_init_interrupt_scheme(adapter); 3192 rtnl_unlock(); 3193 if (err) { 3194 dev_err(&pdev->dev, "Cannot initialize interrupts\n"); 3195 return err; 3196 } 3197 3198 ixgbevf_reset(adapter); 3199 3200 if (netif_running(netdev)) { 3201 err = ixgbevf_open(netdev); 3202 if (err) 3203 return err; 3204 } 3205 3206 netif_device_attach(netdev); 3207 3208 return err; 3209 } 3210 3211 #endif /* CONFIG_PM */ 3212 static void ixgbevf_shutdown(struct pci_dev *pdev) 3213 { 3214 ixgbevf_suspend(pdev, PMSG_SUSPEND); 3215 } 3216 3217 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, 3218 struct rtnl_link_stats64 *stats) 3219 { 3220 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3221 unsigned int start; 3222 u64 bytes, packets; 3223 const struct ixgbevf_ring *ring; 3224 int i; 3225 3226 ixgbevf_update_stats(adapter); 3227 3228 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3229 3230 for (i = 0; i < adapter->num_rx_queues; i++) { 3231 ring = &adapter->rx_ring[i]; 3232 do { 3233 start = u64_stats_fetch_begin_bh(&ring->syncp); 3234 bytes = ring->total_bytes; 3235 packets = ring->total_packets; 3236 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 3237 stats->rx_bytes += bytes; 3238 stats->rx_packets += packets; 3239 } 3240 3241 for (i = 0; i < adapter->num_tx_queues; i++) { 3242 ring = &adapter->tx_ring[i]; 3243 do { 3244 start = u64_stats_fetch_begin_bh(&ring->syncp); 3245 bytes = ring->total_bytes; 3246 packets = ring->total_packets; 3247 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 3248 stats->tx_bytes += bytes; 3249 stats->tx_packets += packets; 3250 } 3251 3252 return stats; 3253 } 3254 3255 static const struct net_device_ops ixgbevf_netdev_ops = { 3256 .ndo_open = ixgbevf_open, 3257 .ndo_stop = ixgbevf_close, 3258 .ndo_start_xmit = ixgbevf_xmit_frame, 3259 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 3260 .ndo_get_stats64 = ixgbevf_get_stats, 3261 .ndo_validate_addr = eth_validate_addr, 3262 .ndo_set_mac_address = ixgbevf_set_mac, 3263 .ndo_change_mtu = ixgbevf_change_mtu, 3264 .ndo_tx_timeout = ixgbevf_tx_timeout, 3265 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 3266 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 3267 }; 3268 3269 static void ixgbevf_assign_netdev_ops(struct net_device *dev) 3270 { 3271 dev->netdev_ops = &ixgbevf_netdev_ops; 3272 ixgbevf_set_ethtool_ops(dev); 3273 dev->watchdog_timeo = 5 * HZ; 3274 } 3275 3276 /** 3277 * ixgbevf_probe - Device Initialization Routine 3278 * @pdev: PCI device information struct 3279 * @ent: entry in ixgbevf_pci_tbl 3280 * 3281 * Returns 0 on success, negative on failure 3282 * 3283 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 3284 * The OS initialization, configuring of the adapter private structure, 3285 * and a hardware reset occur. 3286 **/ 3287 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3288 { 3289 struct net_device *netdev; 3290 struct ixgbevf_adapter *adapter = NULL; 3291 struct ixgbe_hw *hw = NULL; 3292 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 3293 static int cards_found; 3294 int err, pci_using_dac; 3295 3296 err = pci_enable_device(pdev); 3297 if (err) 3298 return err; 3299 3300 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && 3301 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { 3302 pci_using_dac = 1; 3303 } else { 3304 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 3305 if (err) { 3306 err = dma_set_coherent_mask(&pdev->dev, 3307 DMA_BIT_MASK(32)); 3308 if (err) { 3309 dev_err(&pdev->dev, "No usable DMA " 3310 "configuration, aborting\n"); 3311 goto err_dma; 3312 } 3313 } 3314 pci_using_dac = 0; 3315 } 3316 3317 err = pci_request_regions(pdev, ixgbevf_driver_name); 3318 if (err) { 3319 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 3320 goto err_pci_reg; 3321 } 3322 3323 pci_set_master(pdev); 3324 3325 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 3326 MAX_TX_QUEUES); 3327 if (!netdev) { 3328 err = -ENOMEM; 3329 goto err_alloc_etherdev; 3330 } 3331 3332 SET_NETDEV_DEV(netdev, &pdev->dev); 3333 3334 pci_set_drvdata(pdev, netdev); 3335 adapter = netdev_priv(netdev); 3336 3337 adapter->netdev = netdev; 3338 adapter->pdev = pdev; 3339 hw = &adapter->hw; 3340 hw->back = adapter; 3341 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 3342 3343 /* 3344 * call save state here in standalone driver because it relies on 3345 * adapter struct to exist, and needs to call netdev_priv 3346 */ 3347 pci_save_state(pdev); 3348 3349 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3350 pci_resource_len(pdev, 0)); 3351 if (!hw->hw_addr) { 3352 err = -EIO; 3353 goto err_ioremap; 3354 } 3355 3356 ixgbevf_assign_netdev_ops(netdev); 3357 3358 adapter->bd_number = cards_found; 3359 3360 /* Setup hw api */ 3361 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 3362 hw->mac.type = ii->mac; 3363 3364 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 3365 sizeof(struct ixgbe_mbx_operations)); 3366 3367 /* setup the private structure */ 3368 err = ixgbevf_sw_init(adapter); 3369 if (err) 3370 goto err_sw_init; 3371 3372 /* The HW MAC address was set and/or determined in sw_init */ 3373 if (!is_valid_ether_addr(netdev->dev_addr)) { 3374 pr_err("invalid MAC address\n"); 3375 err = -EIO; 3376 goto err_sw_init; 3377 } 3378 3379 netdev->hw_features = NETIF_F_SG | 3380 NETIF_F_IP_CSUM | 3381 NETIF_F_IPV6_CSUM | 3382 NETIF_F_TSO | 3383 NETIF_F_TSO6 | 3384 NETIF_F_RXCSUM; 3385 3386 netdev->features = netdev->hw_features | 3387 NETIF_F_HW_VLAN_TX | 3388 NETIF_F_HW_VLAN_RX | 3389 NETIF_F_HW_VLAN_FILTER; 3390 3391 netdev->vlan_features |= NETIF_F_TSO; 3392 netdev->vlan_features |= NETIF_F_TSO6; 3393 netdev->vlan_features |= NETIF_F_IP_CSUM; 3394 netdev->vlan_features |= NETIF_F_IPV6_CSUM; 3395 netdev->vlan_features |= NETIF_F_SG; 3396 3397 if (pci_using_dac) 3398 netdev->features |= NETIF_F_HIGHDMA; 3399 3400 netdev->priv_flags |= IFF_UNICAST_FLT; 3401 3402 init_timer(&adapter->watchdog_timer); 3403 adapter->watchdog_timer.function = ixgbevf_watchdog; 3404 adapter->watchdog_timer.data = (unsigned long)adapter; 3405 3406 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task); 3407 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task); 3408 3409 err = ixgbevf_init_interrupt_scheme(adapter); 3410 if (err) 3411 goto err_sw_init; 3412 3413 strcpy(netdev->name, "eth%d"); 3414 3415 err = register_netdev(netdev); 3416 if (err) 3417 goto err_register; 3418 3419 netif_carrier_off(netdev); 3420 3421 ixgbevf_init_last_counter_stats(adapter); 3422 3423 /* print the MAC address */ 3424 hw_dbg(hw, "%pM\n", netdev->dev_addr); 3425 3426 hw_dbg(hw, "MAC: %d\n", hw->mac.type); 3427 3428 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n"); 3429 cards_found++; 3430 return 0; 3431 3432 err_register: 3433 ixgbevf_clear_interrupt_scheme(adapter); 3434 err_sw_init: 3435 ixgbevf_reset_interrupt_capability(adapter); 3436 iounmap(hw->hw_addr); 3437 err_ioremap: 3438 free_netdev(netdev); 3439 err_alloc_etherdev: 3440 pci_release_regions(pdev); 3441 err_pci_reg: 3442 err_dma: 3443 pci_disable_device(pdev); 3444 return err; 3445 } 3446 3447 /** 3448 * ixgbevf_remove - Device Removal Routine 3449 * @pdev: PCI device information struct 3450 * 3451 * ixgbevf_remove is called by the PCI subsystem to alert the driver 3452 * that it should release a PCI device. The could be caused by a 3453 * Hot-Plug event, or because the driver is going to be removed from 3454 * memory. 3455 **/ 3456 static void ixgbevf_remove(struct pci_dev *pdev) 3457 { 3458 struct net_device *netdev = pci_get_drvdata(pdev); 3459 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3460 3461 set_bit(__IXGBEVF_DOWN, &adapter->state); 3462 3463 del_timer_sync(&adapter->watchdog_timer); 3464 3465 cancel_work_sync(&adapter->reset_task); 3466 cancel_work_sync(&adapter->watchdog_task); 3467 3468 if (netdev->reg_state == NETREG_REGISTERED) 3469 unregister_netdev(netdev); 3470 3471 ixgbevf_clear_interrupt_scheme(adapter); 3472 ixgbevf_reset_interrupt_capability(adapter); 3473 3474 iounmap(adapter->hw.hw_addr); 3475 pci_release_regions(pdev); 3476 3477 hw_dbg(&adapter->hw, "Remove complete\n"); 3478 3479 kfree(adapter->tx_ring); 3480 kfree(adapter->rx_ring); 3481 3482 free_netdev(netdev); 3483 3484 pci_disable_device(pdev); 3485 } 3486 3487 /** 3488 * ixgbevf_io_error_detected - called when PCI error is detected 3489 * @pdev: Pointer to PCI device 3490 * @state: The current pci connection state 3491 * 3492 * This function is called after a PCI bus error affecting 3493 * this device has been detected. 3494 */ 3495 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, 3496 pci_channel_state_t state) 3497 { 3498 struct net_device *netdev = pci_get_drvdata(pdev); 3499 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3500 3501 netif_device_detach(netdev); 3502 3503 if (state == pci_channel_io_perm_failure) 3504 return PCI_ERS_RESULT_DISCONNECT; 3505 3506 if (netif_running(netdev)) 3507 ixgbevf_down(adapter); 3508 3509 pci_disable_device(pdev); 3510 3511 /* Request a slot slot reset. */ 3512 return PCI_ERS_RESULT_NEED_RESET; 3513 } 3514 3515 /** 3516 * ixgbevf_io_slot_reset - called after the pci bus has been reset. 3517 * @pdev: Pointer to PCI device 3518 * 3519 * Restart the card from scratch, as if from a cold-boot. Implementation 3520 * resembles the first-half of the ixgbevf_resume routine. 3521 */ 3522 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) 3523 { 3524 struct net_device *netdev = pci_get_drvdata(pdev); 3525 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3526 3527 if (pci_enable_device_mem(pdev)) { 3528 dev_err(&pdev->dev, 3529 "Cannot re-enable PCI device after reset.\n"); 3530 return PCI_ERS_RESULT_DISCONNECT; 3531 } 3532 3533 pci_set_master(pdev); 3534 3535 ixgbevf_reset(adapter); 3536 3537 return PCI_ERS_RESULT_RECOVERED; 3538 } 3539 3540 /** 3541 * ixgbevf_io_resume - called when traffic can start flowing again. 3542 * @pdev: Pointer to PCI device 3543 * 3544 * This callback is called when the error recovery driver tells us that 3545 * its OK to resume normal operation. Implementation resembles the 3546 * second-half of the ixgbevf_resume routine. 3547 */ 3548 static void ixgbevf_io_resume(struct pci_dev *pdev) 3549 { 3550 struct net_device *netdev = pci_get_drvdata(pdev); 3551 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3552 3553 if (netif_running(netdev)) 3554 ixgbevf_up(adapter); 3555 3556 netif_device_attach(netdev); 3557 } 3558 3559 /* PCI Error Recovery (ERS) */ 3560 static const struct pci_error_handlers ixgbevf_err_handler = { 3561 .error_detected = ixgbevf_io_error_detected, 3562 .slot_reset = ixgbevf_io_slot_reset, 3563 .resume = ixgbevf_io_resume, 3564 }; 3565 3566 static struct pci_driver ixgbevf_driver = { 3567 .name = ixgbevf_driver_name, 3568 .id_table = ixgbevf_pci_tbl, 3569 .probe = ixgbevf_probe, 3570 .remove = ixgbevf_remove, 3571 #ifdef CONFIG_PM 3572 /* Power Management Hooks */ 3573 .suspend = ixgbevf_suspend, 3574 .resume = ixgbevf_resume, 3575 #endif 3576 .shutdown = ixgbevf_shutdown, 3577 .err_handler = &ixgbevf_err_handler 3578 }; 3579 3580 /** 3581 * ixgbevf_init_module - Driver Registration Routine 3582 * 3583 * ixgbevf_init_module is the first routine called when the driver is 3584 * loaded. All it does is register with the PCI subsystem. 3585 **/ 3586 static int __init ixgbevf_init_module(void) 3587 { 3588 int ret; 3589 pr_info("%s - version %s\n", ixgbevf_driver_string, 3590 ixgbevf_driver_version); 3591 3592 pr_info("%s\n", ixgbevf_copyright); 3593 3594 ret = pci_register_driver(&ixgbevf_driver); 3595 return ret; 3596 } 3597 3598 module_init(ixgbevf_init_module); 3599 3600 /** 3601 * ixgbevf_exit_module - Driver Exit Cleanup Routine 3602 * 3603 * ixgbevf_exit_module is called just before the driver is removed 3604 * from memory. 3605 **/ 3606 static void __exit ixgbevf_exit_module(void) 3607 { 3608 pci_unregister_driver(&ixgbevf_driver); 3609 } 3610 3611 #ifdef DEBUG 3612 /** 3613 * ixgbevf_get_hw_dev_name - return device name string 3614 * used by hardware layer to print debugging information 3615 **/ 3616 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 3617 { 3618 struct ixgbevf_adapter *adapter = hw->back; 3619 return adapter->netdev->name; 3620 } 3621 3622 #endif 3623 module_exit(ixgbevf_exit_module); 3624 3625 /* ixgbevf_main.c */ 3626