1 /******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2012 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 26 *******************************************************************************/ 27 28 29 /****************************************************************************** 30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 31 ******************************************************************************/ 32 33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35 #include <linux/types.h> 36 #include <linux/bitops.h> 37 #include <linux/module.h> 38 #include <linux/pci.h> 39 #include <linux/netdevice.h> 40 #include <linux/vmalloc.h> 41 #include <linux/string.h> 42 #include <linux/in.h> 43 #include <linux/ip.h> 44 #include <linux/tcp.h> 45 #include <linux/sctp.h> 46 #include <linux/ipv6.h> 47 #include <linux/slab.h> 48 #include <net/checksum.h> 49 #include <net/ip6_checksum.h> 50 #include <linux/ethtool.h> 51 #include <linux/if.h> 52 #include <linux/if_vlan.h> 53 #include <linux/prefetch.h> 54 55 #include "ixgbevf.h" 56 57 const char ixgbevf_driver_name[] = "ixgbevf"; 58 static const char ixgbevf_driver_string[] = 59 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 60 61 #define DRV_VERSION "2.6.0-k" 62 const char ixgbevf_driver_version[] = DRV_VERSION; 63 static char ixgbevf_copyright[] = 64 "Copyright (c) 2009 - 2012 Intel Corporation."; 65 66 static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 67 [board_82599_vf] = &ixgbevf_82599_vf_info, 68 [board_X540_vf] = &ixgbevf_X540_vf_info, 69 }; 70 71 /* ixgbevf_pci_tbl - PCI Device ID Table 72 * 73 * Wildcard entries (PCI_ANY_ID) should come last 74 * Last entry must be all 0s 75 * 76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 77 * Class, Class Mask, private data (not used) } 78 */ 79 static struct pci_device_id ixgbevf_pci_tbl[] = { 80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), 81 board_82599_vf}, 82 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), 83 board_X540_vf}, 84 85 /* required last entry */ 86 {0, } 87 }; 88 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 89 90 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 91 MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver"); 92 MODULE_LICENSE("GPL"); 93 MODULE_VERSION(DRV_VERSION); 94 95 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 96 static int debug = -1; 97 module_param(debug, int, 0); 98 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 99 100 /* forward decls */ 101 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); 102 103 static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw, 104 struct ixgbevf_ring *rx_ring, 105 u32 val) 106 { 107 /* 108 * Force memory writes to complete before letting h/w 109 * know there are new descriptors to fetch. (Only 110 * applicable for weak-ordered memory model archs, 111 * such as IA-64). 112 */ 113 wmb(); 114 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val); 115 } 116 117 /** 118 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 119 * @adapter: pointer to adapter struct 120 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 121 * @queue: queue to map the corresponding interrupt to 122 * @msix_vector: the vector to map to the corresponding queue 123 * 124 */ 125 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 126 u8 queue, u8 msix_vector) 127 { 128 u32 ivar, index; 129 struct ixgbe_hw *hw = &adapter->hw; 130 if (direction == -1) { 131 /* other causes */ 132 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 133 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 134 ivar &= ~0xFF; 135 ivar |= msix_vector; 136 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 137 } else { 138 /* tx or rx causes */ 139 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 140 index = ((16 * (queue & 1)) + (8 * direction)); 141 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 142 ivar &= ~(0xFF << index); 143 ivar |= (msix_vector << index); 144 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 145 } 146 } 147 148 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring, 149 struct ixgbevf_tx_buffer 150 *tx_buffer_info) 151 { 152 if (tx_buffer_info->dma) { 153 if (tx_buffer_info->mapped_as_page) 154 dma_unmap_page(tx_ring->dev, 155 tx_buffer_info->dma, 156 tx_buffer_info->length, 157 DMA_TO_DEVICE); 158 else 159 dma_unmap_single(tx_ring->dev, 160 tx_buffer_info->dma, 161 tx_buffer_info->length, 162 DMA_TO_DEVICE); 163 tx_buffer_info->dma = 0; 164 } 165 if (tx_buffer_info->skb) { 166 dev_kfree_skb_any(tx_buffer_info->skb); 167 tx_buffer_info->skb = NULL; 168 } 169 tx_buffer_info->time_stamp = 0; 170 /* tx_buffer_info must be completely set up in the transmit path */ 171 } 172 173 #define IXGBE_MAX_TXD_PWR 14 174 #define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR) 175 176 /* Tx Descriptors needed, worst case */ 177 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD) 178 #define DESC_NEEDED (MAX_SKB_FRAGS + 4) 179 180 static void ixgbevf_tx_timeout(struct net_device *netdev); 181 182 /** 183 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 184 * @q_vector: board private structure 185 * @tx_ring: tx ring to clean 186 **/ 187 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, 188 struct ixgbevf_ring *tx_ring) 189 { 190 struct ixgbevf_adapter *adapter = q_vector->adapter; 191 union ixgbe_adv_tx_desc *tx_desc, *eop_desc; 192 struct ixgbevf_tx_buffer *tx_buffer_info; 193 unsigned int i, eop, count = 0; 194 unsigned int total_bytes = 0, total_packets = 0; 195 196 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 197 return true; 198 199 i = tx_ring->next_to_clean; 200 eop = tx_ring->tx_buffer_info[i].next_to_watch; 201 eop_desc = IXGBEVF_TX_DESC(tx_ring, eop); 202 203 while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) && 204 (count < tx_ring->count)) { 205 bool cleaned = false; 206 rmb(); /* read buffer_info after eop_desc */ 207 /* eop could change between read and DD-check */ 208 if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch)) 209 goto cont_loop; 210 for ( ; !cleaned; count++) { 211 struct sk_buff *skb; 212 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 213 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 214 cleaned = (i == eop); 215 skb = tx_buffer_info->skb; 216 217 if (cleaned && skb) { 218 unsigned int segs, bytecount; 219 220 /* gso_segs is currently only valid for tcp */ 221 segs = skb_shinfo(skb)->gso_segs ?: 1; 222 /* multiply data chunks by size of headers */ 223 bytecount = ((segs - 1) * skb_headlen(skb)) + 224 skb->len; 225 total_packets += segs; 226 total_bytes += bytecount; 227 } 228 229 ixgbevf_unmap_and_free_tx_resource(tx_ring, 230 tx_buffer_info); 231 232 tx_desc->wb.status = 0; 233 234 i++; 235 if (i == tx_ring->count) 236 i = 0; 237 } 238 239 cont_loop: 240 eop = tx_ring->tx_buffer_info[i].next_to_watch; 241 eop_desc = IXGBEVF_TX_DESC(tx_ring, eop); 242 } 243 244 tx_ring->next_to_clean = i; 245 246 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 247 if (unlikely(count && netif_carrier_ok(tx_ring->netdev) && 248 (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 249 /* Make sure that anybody stopping the queue after this 250 * sees the new next_to_clean. 251 */ 252 smp_mb(); 253 if (__netif_subqueue_stopped(tx_ring->netdev, 254 tx_ring->queue_index) && 255 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 256 netif_wake_subqueue(tx_ring->netdev, 257 tx_ring->queue_index); 258 ++adapter->restart_queue; 259 } 260 } 261 262 u64_stats_update_begin(&tx_ring->syncp); 263 tx_ring->total_bytes += total_bytes; 264 tx_ring->total_packets += total_packets; 265 u64_stats_update_end(&tx_ring->syncp); 266 q_vector->tx.total_bytes += total_bytes; 267 q_vector->tx.total_packets += total_packets; 268 269 return count < tx_ring->count; 270 } 271 272 /** 273 * ixgbevf_receive_skb - Send a completed packet up the stack 274 * @q_vector: structure containing interrupt and ring information 275 * @skb: packet to send up 276 * @status: hardware indication of status of receive 277 * @rx_desc: rx descriptor 278 **/ 279 static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector, 280 struct sk_buff *skb, u8 status, 281 union ixgbe_adv_rx_desc *rx_desc) 282 { 283 struct ixgbevf_adapter *adapter = q_vector->adapter; 284 bool is_vlan = (status & IXGBE_RXD_STAT_VP); 285 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan); 286 287 if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans)) 288 __vlan_hwaccel_put_tag(skb, tag); 289 290 napi_gro_receive(&q_vector->napi, skb); 291 } 292 293 /** 294 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 295 * @adapter: address of board private structure 296 * @status_err: hardware indication of status of receive 297 * @skb: skb currently being received and modified 298 **/ 299 static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter, 300 struct ixgbevf_ring *ring, 301 u32 status_err, struct sk_buff *skb) 302 { 303 skb_checksum_none_assert(skb); 304 305 /* Rx csum disabled */ 306 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 307 return; 308 309 /* if IP and error */ 310 if ((status_err & IXGBE_RXD_STAT_IPCS) && 311 (status_err & IXGBE_RXDADV_ERR_IPE)) { 312 adapter->hw_csum_rx_error++; 313 return; 314 } 315 316 if (!(status_err & IXGBE_RXD_STAT_L4CS)) 317 return; 318 319 if (status_err & IXGBE_RXDADV_ERR_TCPE) { 320 adapter->hw_csum_rx_error++; 321 return; 322 } 323 324 /* It must be a TCP or UDP packet with a valid checksum */ 325 skb->ip_summed = CHECKSUM_UNNECESSARY; 326 adapter->hw_csum_rx_good++; 327 } 328 329 /** 330 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 331 * @adapter: address of board private structure 332 **/ 333 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter, 334 struct ixgbevf_ring *rx_ring, 335 int cleaned_count) 336 { 337 struct pci_dev *pdev = adapter->pdev; 338 union ixgbe_adv_rx_desc *rx_desc; 339 struct ixgbevf_rx_buffer *bi; 340 struct sk_buff *skb; 341 unsigned int i = rx_ring->next_to_use; 342 343 bi = &rx_ring->rx_buffer_info[i]; 344 345 while (cleaned_count--) { 346 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 347 skb = bi->skb; 348 if (!skb) { 349 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 350 rx_ring->rx_buf_len); 351 if (!skb) { 352 adapter->alloc_rx_buff_failed++; 353 goto no_buffers; 354 } 355 bi->skb = skb; 356 } 357 if (!bi->dma) { 358 bi->dma = dma_map_single(&pdev->dev, skb->data, 359 rx_ring->rx_buf_len, 360 DMA_FROM_DEVICE); 361 } 362 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); 363 364 i++; 365 if (i == rx_ring->count) 366 i = 0; 367 bi = &rx_ring->rx_buffer_info[i]; 368 } 369 370 no_buffers: 371 if (rx_ring->next_to_use != i) { 372 rx_ring->next_to_use = i; 373 374 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i); 375 } 376 } 377 378 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 379 u32 qmask) 380 { 381 struct ixgbe_hw *hw = &adapter->hw; 382 383 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); 384 } 385 386 static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 387 struct ixgbevf_ring *rx_ring, 388 int budget) 389 { 390 struct ixgbevf_adapter *adapter = q_vector->adapter; 391 struct pci_dev *pdev = adapter->pdev; 392 union ixgbe_adv_rx_desc *rx_desc, *next_rxd; 393 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer; 394 struct sk_buff *skb; 395 unsigned int i; 396 u32 len, staterr; 397 int cleaned_count = 0; 398 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 399 400 i = rx_ring->next_to_clean; 401 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 402 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 403 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 404 405 while (staterr & IXGBE_RXD_STAT_DD) { 406 if (!budget) 407 break; 408 budget--; 409 410 rmb(); /* read descriptor and rx_buffer_info after status DD */ 411 len = le16_to_cpu(rx_desc->wb.upper.length); 412 skb = rx_buffer_info->skb; 413 prefetch(skb->data - NET_IP_ALIGN); 414 rx_buffer_info->skb = NULL; 415 416 if (rx_buffer_info->dma) { 417 dma_unmap_single(&pdev->dev, rx_buffer_info->dma, 418 rx_ring->rx_buf_len, 419 DMA_FROM_DEVICE); 420 rx_buffer_info->dma = 0; 421 skb_put(skb, len); 422 } 423 424 i++; 425 if (i == rx_ring->count) 426 i = 0; 427 428 next_rxd = IXGBEVF_RX_DESC(rx_ring, i); 429 prefetch(next_rxd); 430 cleaned_count++; 431 432 next_buffer = &rx_ring->rx_buffer_info[i]; 433 434 if (!(staterr & IXGBE_RXD_STAT_EOP)) { 435 skb->next = next_buffer->skb; 436 IXGBE_CB(skb->next)->prev = skb; 437 adapter->non_eop_descs++; 438 goto next_desc; 439 } 440 441 /* we should not be chaining buffers, if we did drop the skb */ 442 if (IXGBE_CB(skb)->prev) { 443 do { 444 struct sk_buff *this = skb; 445 skb = IXGBE_CB(skb)->prev; 446 dev_kfree_skb(this); 447 } while (skb); 448 goto next_desc; 449 } 450 451 /* ERR_MASK will only have valid bits if EOP set */ 452 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) { 453 dev_kfree_skb_irq(skb); 454 goto next_desc; 455 } 456 457 ixgbevf_rx_checksum(adapter, rx_ring, staterr, skb); 458 459 /* probably a little skewed due to removing CRC */ 460 total_rx_bytes += skb->len; 461 total_rx_packets++; 462 463 /* 464 * Work around issue of some types of VM to VM loop back 465 * packets not getting split correctly 466 */ 467 if (staterr & IXGBE_RXD_STAT_LB) { 468 u32 header_fixup_len = skb_headlen(skb); 469 if (header_fixup_len < 14) 470 skb_push(skb, header_fixup_len); 471 } 472 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 473 474 ixgbevf_receive_skb(q_vector, skb, staterr, rx_desc); 475 476 next_desc: 477 rx_desc->wb.upper.status_error = 0; 478 479 /* return some buffers to hardware, one at a time is too slow */ 480 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 481 ixgbevf_alloc_rx_buffers(adapter, rx_ring, 482 cleaned_count); 483 cleaned_count = 0; 484 } 485 486 /* use prefetched values */ 487 rx_desc = next_rxd; 488 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 489 490 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 491 } 492 493 rx_ring->next_to_clean = i; 494 cleaned_count = IXGBE_DESC_UNUSED(rx_ring); 495 496 if (cleaned_count) 497 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count); 498 499 u64_stats_update_begin(&rx_ring->syncp); 500 rx_ring->total_packets += total_rx_packets; 501 rx_ring->total_bytes += total_rx_bytes; 502 u64_stats_update_end(&rx_ring->syncp); 503 q_vector->rx.total_packets += total_rx_packets; 504 q_vector->rx.total_bytes += total_rx_bytes; 505 506 return !!budget; 507 } 508 509 /** 510 * ixgbevf_poll - NAPI polling calback 511 * @napi: napi struct with our devices info in it 512 * @budget: amount of work driver is allowed to do this pass, in packets 513 * 514 * This function will clean more than one or more rings associated with a 515 * q_vector. 516 **/ 517 static int ixgbevf_poll(struct napi_struct *napi, int budget) 518 { 519 struct ixgbevf_q_vector *q_vector = 520 container_of(napi, struct ixgbevf_q_vector, napi); 521 struct ixgbevf_adapter *adapter = q_vector->adapter; 522 struct ixgbevf_ring *ring; 523 int per_ring_budget; 524 bool clean_complete = true; 525 526 ixgbevf_for_each_ring(ring, q_vector->tx) 527 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring); 528 529 /* attempt to distribute budget to each queue fairly, but don't allow 530 * the budget to go below 1 because we'll exit polling */ 531 if (q_vector->rx.count > 1) 532 per_ring_budget = max(budget/q_vector->rx.count, 1); 533 else 534 per_ring_budget = budget; 535 536 ixgbevf_for_each_ring(ring, q_vector->rx) 537 clean_complete &= ixgbevf_clean_rx_irq(q_vector, ring, 538 per_ring_budget); 539 540 /* If all work not completed, return budget and keep polling */ 541 if (!clean_complete) 542 return budget; 543 /* all work done, exit the polling mode */ 544 napi_complete(napi); 545 if (adapter->rx_itr_setting & 1) 546 ixgbevf_set_itr(q_vector); 547 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 548 ixgbevf_irq_enable_queues(adapter, 549 1 << q_vector->v_idx); 550 551 return 0; 552 } 553 554 /** 555 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 556 * @q_vector: structure containing interrupt and ring information 557 */ 558 static void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) 559 { 560 struct ixgbevf_adapter *adapter = q_vector->adapter; 561 struct ixgbe_hw *hw = &adapter->hw; 562 int v_idx = q_vector->v_idx; 563 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 564 565 /* 566 * set the WDIS bit to not clear the timer bits and cause an 567 * immediate assertion of the interrupt 568 */ 569 itr_reg |= IXGBE_EITR_CNT_WDIS; 570 571 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 572 } 573 574 /** 575 * ixgbevf_configure_msix - Configure MSI-X hardware 576 * @adapter: board private structure 577 * 578 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 579 * interrupts. 580 **/ 581 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 582 { 583 struct ixgbevf_q_vector *q_vector; 584 int q_vectors, v_idx; 585 586 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 587 adapter->eims_enable_mask = 0; 588 589 /* 590 * Populate the IVAR table and set the ITR values to the 591 * corresponding register. 592 */ 593 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 594 struct ixgbevf_ring *ring; 595 q_vector = adapter->q_vector[v_idx]; 596 597 ixgbevf_for_each_ring(ring, q_vector->rx) 598 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx); 599 600 ixgbevf_for_each_ring(ring, q_vector->tx) 601 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx); 602 603 if (q_vector->tx.ring && !q_vector->rx.ring) { 604 /* tx only vector */ 605 if (adapter->tx_itr_setting == 1) 606 q_vector->itr = IXGBE_10K_ITR; 607 else 608 q_vector->itr = adapter->tx_itr_setting; 609 } else { 610 /* rx or rx/tx vector */ 611 if (adapter->rx_itr_setting == 1) 612 q_vector->itr = IXGBE_20K_ITR; 613 else 614 q_vector->itr = adapter->rx_itr_setting; 615 } 616 617 /* add q_vector eims value to global eims_enable_mask */ 618 adapter->eims_enable_mask |= 1 << v_idx; 619 620 ixgbevf_write_eitr(q_vector); 621 } 622 623 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 624 /* setup eims_other and add value to global eims_enable_mask */ 625 adapter->eims_other = 1 << v_idx; 626 adapter->eims_enable_mask |= adapter->eims_other; 627 } 628 629 enum latency_range { 630 lowest_latency = 0, 631 low_latency = 1, 632 bulk_latency = 2, 633 latency_invalid = 255 634 }; 635 636 /** 637 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 638 * @q_vector: structure containing interrupt and ring information 639 * @ring_container: structure containing ring performance data 640 * 641 * Stores a new ITR value based on packets and byte 642 * counts during the last interrupt. The advantage of per interrupt 643 * computation is faster updates and more accurate ITR for the current 644 * traffic pattern. Constants in this function were computed 645 * based on theoretical maximum wire speed and thresholds were set based 646 * on testing data as well as attempting to minimize response time 647 * while increasing bulk throughput. 648 **/ 649 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector, 650 struct ixgbevf_ring_container *ring_container) 651 { 652 int bytes = ring_container->total_bytes; 653 int packets = ring_container->total_packets; 654 u32 timepassed_us; 655 u64 bytes_perint; 656 u8 itr_setting = ring_container->itr; 657 658 if (packets == 0) 659 return; 660 661 /* simple throttlerate management 662 * 0-20MB/s lowest (100000 ints/s) 663 * 20-100MB/s low (20000 ints/s) 664 * 100-1249MB/s bulk (8000 ints/s) 665 */ 666 /* what was last interrupt timeslice? */ 667 timepassed_us = q_vector->itr >> 2; 668 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 669 670 switch (itr_setting) { 671 case lowest_latency: 672 if (bytes_perint > 10) 673 itr_setting = low_latency; 674 break; 675 case low_latency: 676 if (bytes_perint > 20) 677 itr_setting = bulk_latency; 678 else if (bytes_perint <= 10) 679 itr_setting = lowest_latency; 680 break; 681 case bulk_latency: 682 if (bytes_perint <= 20) 683 itr_setting = low_latency; 684 break; 685 } 686 687 /* clear work counters since we have the values we need */ 688 ring_container->total_bytes = 0; 689 ring_container->total_packets = 0; 690 691 /* write updated itr to ring container */ 692 ring_container->itr = itr_setting; 693 } 694 695 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector) 696 { 697 u32 new_itr = q_vector->itr; 698 u8 current_itr; 699 700 ixgbevf_update_itr(q_vector, &q_vector->tx); 701 ixgbevf_update_itr(q_vector, &q_vector->rx); 702 703 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 704 705 switch (current_itr) { 706 /* counts and packets in update_itr are dependent on these numbers */ 707 case lowest_latency: 708 new_itr = IXGBE_100K_ITR; 709 break; 710 case low_latency: 711 new_itr = IXGBE_20K_ITR; 712 break; 713 case bulk_latency: 714 default: 715 new_itr = IXGBE_8K_ITR; 716 break; 717 } 718 719 if (new_itr != q_vector->itr) { 720 /* do an exponential smoothing */ 721 new_itr = (10 * new_itr * q_vector->itr) / 722 ((9 * new_itr) + q_vector->itr); 723 724 /* save the algorithm value here */ 725 q_vector->itr = new_itr; 726 727 ixgbevf_write_eitr(q_vector); 728 } 729 } 730 731 static irqreturn_t ixgbevf_msix_other(int irq, void *data) 732 { 733 struct ixgbevf_adapter *adapter = data; 734 struct ixgbe_hw *hw = &adapter->hw; 735 736 hw->mac.get_link_status = 1; 737 738 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 739 mod_timer(&adapter->watchdog_timer, jiffies); 740 741 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); 742 743 return IRQ_HANDLED; 744 } 745 746 747 /** 748 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) 749 * @irq: unused 750 * @data: pointer to our q_vector struct for this interrupt vector 751 **/ 752 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data) 753 { 754 struct ixgbevf_q_vector *q_vector = data; 755 756 /* EIAM disabled interrupts (on this vector) for us */ 757 if (q_vector->rx.ring || q_vector->tx.ring) 758 napi_schedule(&q_vector->napi); 759 760 return IRQ_HANDLED; 761 } 762 763 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, 764 int r_idx) 765 { 766 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 767 768 a->rx_ring[r_idx].next = q_vector->rx.ring; 769 q_vector->rx.ring = &a->rx_ring[r_idx]; 770 q_vector->rx.count++; 771 } 772 773 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, 774 int t_idx) 775 { 776 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 777 778 a->tx_ring[t_idx].next = q_vector->tx.ring; 779 q_vector->tx.ring = &a->tx_ring[t_idx]; 780 q_vector->tx.count++; 781 } 782 783 /** 784 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors 785 * @adapter: board private structure to initialize 786 * 787 * This function maps descriptor rings to the queue-specific vectors 788 * we were allotted through the MSI-X enabling code. Ideally, we'd have 789 * one vector per ring/queue, but on a constrained vector budget, we 790 * group the rings as "efficiently" as possible. You would add new 791 * mapping configurations in here. 792 **/ 793 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter) 794 { 795 int q_vectors; 796 int v_start = 0; 797 int rxr_idx = 0, txr_idx = 0; 798 int rxr_remaining = adapter->num_rx_queues; 799 int txr_remaining = adapter->num_tx_queues; 800 int i, j; 801 int rqpv, tqpv; 802 int err = 0; 803 804 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 805 806 /* 807 * The ideal configuration... 808 * We have enough vectors to map one per queue. 809 */ 810 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) { 811 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) 812 map_vector_to_rxq(adapter, v_start, rxr_idx); 813 814 for (; txr_idx < txr_remaining; v_start++, txr_idx++) 815 map_vector_to_txq(adapter, v_start, txr_idx); 816 goto out; 817 } 818 819 /* 820 * If we don't have enough vectors for a 1-to-1 821 * mapping, we'll have to group them so there are 822 * multiple queues per vector. 823 */ 824 /* Re-adjusting *qpv takes care of the remainder. */ 825 for (i = v_start; i < q_vectors; i++) { 826 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i); 827 for (j = 0; j < rqpv; j++) { 828 map_vector_to_rxq(adapter, i, rxr_idx); 829 rxr_idx++; 830 rxr_remaining--; 831 } 832 } 833 for (i = v_start; i < q_vectors; i++) { 834 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i); 835 for (j = 0; j < tqpv; j++) { 836 map_vector_to_txq(adapter, i, txr_idx); 837 txr_idx++; 838 txr_remaining--; 839 } 840 } 841 842 out: 843 return err; 844 } 845 846 /** 847 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 848 * @adapter: board private structure 849 * 850 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 851 * interrupts from the kernel. 852 **/ 853 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 854 { 855 struct net_device *netdev = adapter->netdev; 856 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 857 int vector, err; 858 int ri = 0, ti = 0; 859 860 for (vector = 0; vector < q_vectors; vector++) { 861 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector]; 862 struct msix_entry *entry = &adapter->msix_entries[vector]; 863 864 if (q_vector->tx.ring && q_vector->rx.ring) { 865 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 866 "%s-%s-%d", netdev->name, "TxRx", ri++); 867 ti++; 868 } else if (q_vector->rx.ring) { 869 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 870 "%s-%s-%d", netdev->name, "rx", ri++); 871 } else if (q_vector->tx.ring) { 872 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 873 "%s-%s-%d", netdev->name, "tx", ti++); 874 } else { 875 /* skip this unused q_vector */ 876 continue; 877 } 878 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0, 879 q_vector->name, q_vector); 880 if (err) { 881 hw_dbg(&adapter->hw, 882 "request_irq failed for MSIX interrupt " 883 "Error: %d\n", err); 884 goto free_queue_irqs; 885 } 886 } 887 888 err = request_irq(adapter->msix_entries[vector].vector, 889 &ixgbevf_msix_other, 0, netdev->name, adapter); 890 if (err) { 891 hw_dbg(&adapter->hw, 892 "request_irq for msix_other failed: %d\n", err); 893 goto free_queue_irqs; 894 } 895 896 return 0; 897 898 free_queue_irqs: 899 while (vector) { 900 vector--; 901 free_irq(adapter->msix_entries[vector].vector, 902 adapter->q_vector[vector]); 903 } 904 pci_disable_msix(adapter->pdev); 905 kfree(adapter->msix_entries); 906 adapter->msix_entries = NULL; 907 return err; 908 } 909 910 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter) 911 { 912 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 913 914 for (i = 0; i < q_vectors; i++) { 915 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i]; 916 q_vector->rx.ring = NULL; 917 q_vector->tx.ring = NULL; 918 q_vector->rx.count = 0; 919 q_vector->tx.count = 0; 920 } 921 } 922 923 /** 924 * ixgbevf_request_irq - initialize interrupts 925 * @adapter: board private structure 926 * 927 * Attempts to configure interrupts using the best available 928 * capabilities of the hardware and kernel. 929 **/ 930 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 931 { 932 int err = 0; 933 934 err = ixgbevf_request_msix_irqs(adapter); 935 936 if (err) 937 hw_dbg(&adapter->hw, 938 "request_irq failed, Error %d\n", err); 939 940 return err; 941 } 942 943 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 944 { 945 int i, q_vectors; 946 947 q_vectors = adapter->num_msix_vectors; 948 i = q_vectors - 1; 949 950 free_irq(adapter->msix_entries[i].vector, adapter); 951 i--; 952 953 for (; i >= 0; i--) { 954 /* free only the irqs that were actually requested */ 955 if (!adapter->q_vector[i]->rx.ring && 956 !adapter->q_vector[i]->tx.ring) 957 continue; 958 959 free_irq(adapter->msix_entries[i].vector, 960 adapter->q_vector[i]); 961 } 962 963 ixgbevf_reset_q_vectors(adapter); 964 } 965 966 /** 967 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 968 * @adapter: board private structure 969 **/ 970 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 971 { 972 struct ixgbe_hw *hw = &adapter->hw; 973 int i; 974 975 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0); 976 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 977 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0); 978 979 IXGBE_WRITE_FLUSH(hw); 980 981 for (i = 0; i < adapter->num_msix_vectors; i++) 982 synchronize_irq(adapter->msix_entries[i].vector); 983 } 984 985 /** 986 * ixgbevf_irq_enable - Enable default interrupt generation settings 987 * @adapter: board private structure 988 **/ 989 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) 990 { 991 struct ixgbe_hw *hw = &adapter->hw; 992 993 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask); 994 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask); 995 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); 996 } 997 998 /** 999 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1000 * @adapter: board private structure 1001 * 1002 * Configure the Tx unit of the MAC after a reset. 1003 **/ 1004 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1005 { 1006 u64 tdba; 1007 struct ixgbe_hw *hw = &adapter->hw; 1008 u32 i, j, tdlen, txctrl; 1009 1010 /* Setup the HW Tx Head and Tail descriptor pointers */ 1011 for (i = 0; i < adapter->num_tx_queues; i++) { 1012 struct ixgbevf_ring *ring = &adapter->tx_ring[i]; 1013 j = ring->reg_idx; 1014 tdba = ring->dma; 1015 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc); 1016 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1017 (tdba & DMA_BIT_MASK(32))); 1018 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1019 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen); 1020 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0); 1021 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0); 1022 adapter->tx_ring[i].head = IXGBE_VFTDH(j); 1023 adapter->tx_ring[i].tail = IXGBE_VFTDT(j); 1024 /* Disable Tx Head Writeback RO bit, since this hoses 1025 * bookkeeping if things aren't delivered in order. 1026 */ 1027 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1028 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 1029 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1030 } 1031 } 1032 1033 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1034 1035 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) 1036 { 1037 struct ixgbevf_ring *rx_ring; 1038 struct ixgbe_hw *hw = &adapter->hw; 1039 u32 srrctl; 1040 1041 rx_ring = &adapter->rx_ring[index]; 1042 1043 srrctl = IXGBE_SRRCTL_DROP_EN; 1044 1045 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1046 1047 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >> 1048 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1049 1050 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1051 } 1052 1053 static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter) 1054 { 1055 struct ixgbe_hw *hw = &adapter->hw; 1056 struct net_device *netdev = adapter->netdev; 1057 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 1058 int i; 1059 u16 rx_buf_len; 1060 1061 /* notify the PF of our intent to use this size of frame */ 1062 ixgbevf_rlpml_set_vf(hw, max_frame); 1063 1064 /* PF will allow an extra 4 bytes past for vlan tagged frames */ 1065 max_frame += VLAN_HLEN; 1066 1067 /* 1068 * Make best use of allocation by using all but 1K of a 1069 * power of 2 allocation that will be used for skb->head. 1070 */ 1071 if ((hw->mac.type == ixgbe_mac_X540_vf) && 1072 (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)) 1073 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE; 1074 else if (max_frame <= IXGBEVF_RXBUFFER_3K) 1075 rx_buf_len = IXGBEVF_RXBUFFER_3K; 1076 else if (max_frame <= IXGBEVF_RXBUFFER_7K) 1077 rx_buf_len = IXGBEVF_RXBUFFER_7K; 1078 else if (max_frame <= IXGBEVF_RXBUFFER_15K) 1079 rx_buf_len = IXGBEVF_RXBUFFER_15K; 1080 else 1081 rx_buf_len = IXGBEVF_MAX_RXBUFFER; 1082 1083 for (i = 0; i < adapter->num_rx_queues; i++) 1084 adapter->rx_ring[i].rx_buf_len = rx_buf_len; 1085 } 1086 1087 /** 1088 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1089 * @adapter: board private structure 1090 * 1091 * Configure the Rx unit of the MAC after a reset. 1092 **/ 1093 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1094 { 1095 u64 rdba; 1096 struct ixgbe_hw *hw = &adapter->hw; 1097 int i, j; 1098 u32 rdlen; 1099 1100 /* PSRTYPE must be initialized in 82599 */ 1101 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0); 1102 1103 /* set_rx_buffer_len must be called before ring initialization */ 1104 ixgbevf_set_rx_buffer_len(adapter); 1105 1106 rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc); 1107 /* Setup the HW Rx Head and Tail Descriptor Pointers and 1108 * the Base and Length of the Rx Descriptor Ring */ 1109 for (i = 0; i < adapter->num_rx_queues; i++) { 1110 rdba = adapter->rx_ring[i].dma; 1111 j = adapter->rx_ring[i].reg_idx; 1112 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1113 (rdba & DMA_BIT_MASK(32))); 1114 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1115 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen); 1116 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0); 1117 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0); 1118 adapter->rx_ring[i].head = IXGBE_VFRDH(j); 1119 adapter->rx_ring[i].tail = IXGBE_VFRDT(j); 1120 1121 ixgbevf_configure_srrctl(adapter, j); 1122 } 1123 } 1124 1125 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid) 1126 { 1127 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1128 struct ixgbe_hw *hw = &adapter->hw; 1129 int err; 1130 1131 if (!hw->mac.ops.set_vfta) 1132 return -EOPNOTSUPP; 1133 1134 spin_lock(&adapter->mbx_lock); 1135 1136 /* add VID to filter table */ 1137 err = hw->mac.ops.set_vfta(hw, vid, 0, true); 1138 1139 spin_unlock(&adapter->mbx_lock); 1140 1141 /* translate error return types so error makes sense */ 1142 if (err == IXGBE_ERR_MBX) 1143 return -EIO; 1144 1145 if (err == IXGBE_ERR_INVALID_ARGUMENT) 1146 return -EACCES; 1147 1148 set_bit(vid, adapter->active_vlans); 1149 1150 return err; 1151 } 1152 1153 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) 1154 { 1155 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1156 struct ixgbe_hw *hw = &adapter->hw; 1157 int err = -EOPNOTSUPP; 1158 1159 spin_lock(&adapter->mbx_lock); 1160 1161 /* remove VID from filter table */ 1162 if (hw->mac.ops.set_vfta) 1163 err = hw->mac.ops.set_vfta(hw, vid, 0, false); 1164 1165 spin_unlock(&adapter->mbx_lock); 1166 1167 clear_bit(vid, adapter->active_vlans); 1168 1169 return err; 1170 } 1171 1172 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 1173 { 1174 u16 vid; 1175 1176 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1177 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid); 1178 } 1179 1180 static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 1181 { 1182 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1183 struct ixgbe_hw *hw = &adapter->hw; 1184 int count = 0; 1185 1186 if ((netdev_uc_count(netdev)) > 10) { 1187 pr_err("Too many unicast filters - No Space\n"); 1188 return -ENOSPC; 1189 } 1190 1191 if (!netdev_uc_empty(netdev)) { 1192 struct netdev_hw_addr *ha; 1193 netdev_for_each_uc_addr(ha, netdev) { 1194 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 1195 udelay(200); 1196 } 1197 } else { 1198 /* 1199 * If the list is empty then send message to PF driver to 1200 * clear all macvlans on this VF. 1201 */ 1202 hw->mac.ops.set_uc_addr(hw, 0, NULL); 1203 } 1204 1205 return count; 1206 } 1207 1208 /** 1209 * ixgbevf_set_rx_mode - Multicast set 1210 * @netdev: network interface device structure 1211 * 1212 * The set_rx_method entry point is called whenever the multicast address 1213 * list or the network interface flags are updated. This routine is 1214 * responsible for configuring the hardware for proper multicast mode. 1215 **/ 1216 static void ixgbevf_set_rx_mode(struct net_device *netdev) 1217 { 1218 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1219 struct ixgbe_hw *hw = &adapter->hw; 1220 1221 spin_lock(&adapter->mbx_lock); 1222 1223 /* reprogram multicast list */ 1224 if (hw->mac.ops.update_mc_addr_list) 1225 hw->mac.ops.update_mc_addr_list(hw, netdev); 1226 1227 ixgbevf_write_uc_addr_list(netdev); 1228 1229 spin_unlock(&adapter->mbx_lock); 1230 } 1231 1232 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 1233 { 1234 int q_idx; 1235 struct ixgbevf_q_vector *q_vector; 1236 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1237 1238 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1239 q_vector = adapter->q_vector[q_idx]; 1240 napi_enable(&q_vector->napi); 1241 } 1242 } 1243 1244 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 1245 { 1246 int q_idx; 1247 struct ixgbevf_q_vector *q_vector; 1248 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1249 1250 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1251 q_vector = adapter->q_vector[q_idx]; 1252 napi_disable(&q_vector->napi); 1253 } 1254 } 1255 1256 static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 1257 { 1258 struct net_device *netdev = adapter->netdev; 1259 int i; 1260 1261 ixgbevf_set_rx_mode(netdev); 1262 1263 ixgbevf_restore_vlan(adapter); 1264 1265 ixgbevf_configure_tx(adapter); 1266 ixgbevf_configure_rx(adapter); 1267 for (i = 0; i < adapter->num_rx_queues; i++) { 1268 struct ixgbevf_ring *ring = &adapter->rx_ring[i]; 1269 ixgbevf_alloc_rx_buffers(adapter, ring, 1270 IXGBE_DESC_UNUSED(ring)); 1271 } 1272 } 1273 1274 #define IXGBE_MAX_RX_DESC_POLL 10 1275 static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1276 int rxr) 1277 { 1278 struct ixgbe_hw *hw = &adapter->hw; 1279 int j = adapter->rx_ring[rxr].reg_idx; 1280 int k; 1281 1282 for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) { 1283 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE) 1284 break; 1285 else 1286 msleep(1); 1287 } 1288 if (k >= IXGBE_MAX_RX_DESC_POLL) { 1289 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d " 1290 "not set within the polling period\n", rxr); 1291 } 1292 1293 ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr], 1294 (adapter->rx_ring[rxr].count - 1)); 1295 } 1296 1297 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 1298 { 1299 /* Only save pre-reset stats if there are some */ 1300 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 1301 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 1302 adapter->stats.base_vfgprc; 1303 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 1304 adapter->stats.base_vfgptc; 1305 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 1306 adapter->stats.base_vfgorc; 1307 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 1308 adapter->stats.base_vfgotc; 1309 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 1310 adapter->stats.base_vfmprc; 1311 } 1312 } 1313 1314 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 1315 { 1316 struct ixgbe_hw *hw = &adapter->hw; 1317 1318 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 1319 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 1320 adapter->stats.last_vfgorc |= 1321 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 1322 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 1323 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 1324 adapter->stats.last_vfgotc |= 1325 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 1326 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 1327 1328 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 1329 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 1330 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 1331 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 1332 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 1333 } 1334 1335 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) 1336 { 1337 struct ixgbe_hw *hw = &adapter->hw; 1338 int api[] = { ixgbe_mbox_api_10, 1339 ixgbe_mbox_api_unknown }; 1340 int err = 0, idx = 0; 1341 1342 spin_lock(&adapter->mbx_lock); 1343 1344 while (api[idx] != ixgbe_mbox_api_unknown) { 1345 err = ixgbevf_negotiate_api_version(hw, api[idx]); 1346 if (!err) 1347 break; 1348 idx++; 1349 } 1350 1351 spin_unlock(&adapter->mbx_lock); 1352 } 1353 1354 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 1355 { 1356 struct net_device *netdev = adapter->netdev; 1357 struct ixgbe_hw *hw = &adapter->hw; 1358 int i, j = 0; 1359 int num_rx_rings = adapter->num_rx_queues; 1360 u32 txdctl, rxdctl; 1361 1362 for (i = 0; i < adapter->num_tx_queues; i++) { 1363 j = adapter->tx_ring[i].reg_idx; 1364 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1365 /* enable WTHRESH=8 descriptors, to encourage burst writeback */ 1366 txdctl |= (8 << 16); 1367 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1368 } 1369 1370 for (i = 0; i < adapter->num_tx_queues; i++) { 1371 j = adapter->tx_ring[i].reg_idx; 1372 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1373 txdctl |= IXGBE_TXDCTL_ENABLE; 1374 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1375 } 1376 1377 for (i = 0; i < num_rx_rings; i++) { 1378 j = adapter->rx_ring[i].reg_idx; 1379 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1380 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1381 if (hw->mac.type == ixgbe_mac_X540_vf) { 1382 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK; 1383 rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) | 1384 IXGBE_RXDCTL_RLPML_EN); 1385 } 1386 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1387 ixgbevf_rx_desc_queue_enable(adapter, i); 1388 } 1389 1390 ixgbevf_configure_msix(adapter); 1391 1392 spin_lock(&adapter->mbx_lock); 1393 1394 if (hw->mac.ops.set_rar) { 1395 if (is_valid_ether_addr(hw->mac.addr)) 1396 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 1397 else 1398 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 1399 } 1400 1401 spin_unlock(&adapter->mbx_lock); 1402 1403 clear_bit(__IXGBEVF_DOWN, &adapter->state); 1404 ixgbevf_napi_enable_all(adapter); 1405 1406 /* enable transmits */ 1407 netif_tx_start_all_queues(netdev); 1408 1409 ixgbevf_save_reset_stats(adapter); 1410 ixgbevf_init_last_counter_stats(adapter); 1411 1412 hw->mac.get_link_status = 1; 1413 mod_timer(&adapter->watchdog_timer, jiffies); 1414 } 1415 1416 void ixgbevf_up(struct ixgbevf_adapter *adapter) 1417 { 1418 struct ixgbe_hw *hw = &adapter->hw; 1419 1420 ixgbevf_negotiate_api(adapter); 1421 1422 ixgbevf_configure(adapter); 1423 1424 ixgbevf_up_complete(adapter); 1425 1426 /* clear any pending interrupts, may auto mask */ 1427 IXGBE_READ_REG(hw, IXGBE_VTEICR); 1428 1429 ixgbevf_irq_enable(adapter); 1430 } 1431 1432 /** 1433 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 1434 * @adapter: board private structure 1435 * @rx_ring: ring to free buffers from 1436 **/ 1437 static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter, 1438 struct ixgbevf_ring *rx_ring) 1439 { 1440 struct pci_dev *pdev = adapter->pdev; 1441 unsigned long size; 1442 unsigned int i; 1443 1444 if (!rx_ring->rx_buffer_info) 1445 return; 1446 1447 /* Free all the Rx ring sk_buffs */ 1448 for (i = 0; i < rx_ring->count; i++) { 1449 struct ixgbevf_rx_buffer *rx_buffer_info; 1450 1451 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 1452 if (rx_buffer_info->dma) { 1453 dma_unmap_single(&pdev->dev, rx_buffer_info->dma, 1454 rx_ring->rx_buf_len, 1455 DMA_FROM_DEVICE); 1456 rx_buffer_info->dma = 0; 1457 } 1458 if (rx_buffer_info->skb) { 1459 struct sk_buff *skb = rx_buffer_info->skb; 1460 rx_buffer_info->skb = NULL; 1461 do { 1462 struct sk_buff *this = skb; 1463 skb = IXGBE_CB(skb)->prev; 1464 dev_kfree_skb(this); 1465 } while (skb); 1466 } 1467 } 1468 1469 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 1470 memset(rx_ring->rx_buffer_info, 0, size); 1471 1472 /* Zero out the descriptor ring */ 1473 memset(rx_ring->desc, 0, rx_ring->size); 1474 1475 rx_ring->next_to_clean = 0; 1476 rx_ring->next_to_use = 0; 1477 1478 if (rx_ring->head) 1479 writel(0, adapter->hw.hw_addr + rx_ring->head); 1480 if (rx_ring->tail) 1481 writel(0, adapter->hw.hw_addr + rx_ring->tail); 1482 } 1483 1484 /** 1485 * ixgbevf_clean_tx_ring - Free Tx Buffers 1486 * @adapter: board private structure 1487 * @tx_ring: ring to be cleaned 1488 **/ 1489 static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter, 1490 struct ixgbevf_ring *tx_ring) 1491 { 1492 struct ixgbevf_tx_buffer *tx_buffer_info; 1493 unsigned long size; 1494 unsigned int i; 1495 1496 if (!tx_ring->tx_buffer_info) 1497 return; 1498 1499 /* Free all the Tx ring sk_buffs */ 1500 1501 for (i = 0; i < tx_ring->count; i++) { 1502 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 1503 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 1504 } 1505 1506 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 1507 memset(tx_ring->tx_buffer_info, 0, size); 1508 1509 memset(tx_ring->desc, 0, tx_ring->size); 1510 1511 tx_ring->next_to_use = 0; 1512 tx_ring->next_to_clean = 0; 1513 1514 if (tx_ring->head) 1515 writel(0, adapter->hw.hw_addr + tx_ring->head); 1516 if (tx_ring->tail) 1517 writel(0, adapter->hw.hw_addr + tx_ring->tail); 1518 } 1519 1520 /** 1521 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 1522 * @adapter: board private structure 1523 **/ 1524 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 1525 { 1526 int i; 1527 1528 for (i = 0; i < adapter->num_rx_queues; i++) 1529 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]); 1530 } 1531 1532 /** 1533 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 1534 * @adapter: board private structure 1535 **/ 1536 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 1537 { 1538 int i; 1539 1540 for (i = 0; i < adapter->num_tx_queues; i++) 1541 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]); 1542 } 1543 1544 void ixgbevf_down(struct ixgbevf_adapter *adapter) 1545 { 1546 struct net_device *netdev = adapter->netdev; 1547 struct ixgbe_hw *hw = &adapter->hw; 1548 u32 txdctl; 1549 int i, j; 1550 1551 /* signal that we are down to the interrupt handler */ 1552 set_bit(__IXGBEVF_DOWN, &adapter->state); 1553 /* disable receives */ 1554 1555 netif_tx_disable(netdev); 1556 1557 msleep(10); 1558 1559 netif_tx_stop_all_queues(netdev); 1560 1561 ixgbevf_irq_disable(adapter); 1562 1563 ixgbevf_napi_disable_all(adapter); 1564 1565 del_timer_sync(&adapter->watchdog_timer); 1566 /* can't call flush scheduled work here because it can deadlock 1567 * if linkwatch_event tries to acquire the rtnl_lock which we are 1568 * holding */ 1569 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK) 1570 msleep(1); 1571 1572 /* disable transmits in the hardware now that interrupts are off */ 1573 for (i = 0; i < adapter->num_tx_queues; i++) { 1574 j = adapter->tx_ring[i].reg_idx; 1575 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1576 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), 1577 (txdctl & ~IXGBE_TXDCTL_ENABLE)); 1578 } 1579 1580 netif_carrier_off(netdev); 1581 1582 if (!pci_channel_offline(adapter->pdev)) 1583 ixgbevf_reset(adapter); 1584 1585 ixgbevf_clean_all_tx_rings(adapter); 1586 ixgbevf_clean_all_rx_rings(adapter); 1587 } 1588 1589 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 1590 { 1591 WARN_ON(in_interrupt()); 1592 1593 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 1594 msleep(1); 1595 1596 /* 1597 * Check if PF is up before re-init. If not then skip until 1598 * later when the PF is up and ready to service requests from 1599 * the VF via mailbox. If the VF is up and running then the 1600 * watchdog task will continue to schedule reset tasks until 1601 * the PF is up and running. 1602 */ 1603 ixgbevf_down(adapter); 1604 ixgbevf_up(adapter); 1605 1606 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 1607 } 1608 1609 void ixgbevf_reset(struct ixgbevf_adapter *adapter) 1610 { 1611 struct ixgbe_hw *hw = &adapter->hw; 1612 struct net_device *netdev = adapter->netdev; 1613 1614 spin_lock(&adapter->mbx_lock); 1615 1616 if (hw->mac.ops.reset_hw(hw)) 1617 hw_dbg(hw, "PF still resetting\n"); 1618 else 1619 hw->mac.ops.init_hw(hw); 1620 1621 spin_unlock(&adapter->mbx_lock); 1622 1623 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 1624 memcpy(netdev->dev_addr, adapter->hw.mac.addr, 1625 netdev->addr_len); 1626 memcpy(netdev->perm_addr, adapter->hw.mac.addr, 1627 netdev->addr_len); 1628 } 1629 } 1630 1631 static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 1632 int vectors) 1633 { 1634 int err, vector_threshold; 1635 1636 /* We'll want at least 2 (vector_threshold): 1637 * 1) TxQ[0] + RxQ[0] handler 1638 * 2) Other (Link Status Change, etc.) 1639 */ 1640 vector_threshold = MIN_MSIX_COUNT; 1641 1642 /* The more we get, the more we will assign to Tx/Rx Cleanup 1643 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 1644 * Right now, we simply care about how many we'll get; we'll 1645 * set them up later while requesting irq's. 1646 */ 1647 while (vectors >= vector_threshold) { 1648 err = pci_enable_msix(adapter->pdev, adapter->msix_entries, 1649 vectors); 1650 if (!err) /* Success in acquiring all requested vectors. */ 1651 break; 1652 else if (err < 0) 1653 vectors = 0; /* Nasty failure, quit now */ 1654 else /* err == number of vectors we should try again with */ 1655 vectors = err; 1656 } 1657 1658 if (vectors < vector_threshold) { 1659 /* Can't allocate enough MSI-X interrupts? Oh well. 1660 * This just means we'll go with either a single MSI 1661 * vector or fall back to legacy interrupts. 1662 */ 1663 hw_dbg(&adapter->hw, 1664 "Unable to allocate MSI-X interrupts\n"); 1665 kfree(adapter->msix_entries); 1666 adapter->msix_entries = NULL; 1667 } else { 1668 /* 1669 * Adjust for only the vectors we'll use, which is minimum 1670 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 1671 * vectors we were allocated. 1672 */ 1673 adapter->num_msix_vectors = vectors; 1674 } 1675 } 1676 1677 /** 1678 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent 1679 * @adapter: board private structure to initialize 1680 * 1681 * This is the top level queue allocation routine. The order here is very 1682 * important, starting with the "most" number of features turned on at once, 1683 * and ending with the smallest set of features. This way large combinations 1684 * can be allocated if they're turned on, and smaller combinations are the 1685 * fallthrough conditions. 1686 * 1687 **/ 1688 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 1689 { 1690 /* Start with base case */ 1691 adapter->num_rx_queues = 1; 1692 adapter->num_tx_queues = 1; 1693 } 1694 1695 /** 1696 * ixgbevf_alloc_queues - Allocate memory for all rings 1697 * @adapter: board private structure to initialize 1698 * 1699 * We allocate one ring per queue at run-time since we don't know the 1700 * number of queues at compile-time. The polling_netdev array is 1701 * intended for Multiqueue, but should work fine with a single queue. 1702 **/ 1703 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) 1704 { 1705 int i; 1706 1707 adapter->tx_ring = kcalloc(adapter->num_tx_queues, 1708 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1709 if (!adapter->tx_ring) 1710 goto err_tx_ring_allocation; 1711 1712 adapter->rx_ring = kcalloc(adapter->num_rx_queues, 1713 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1714 if (!adapter->rx_ring) 1715 goto err_rx_ring_allocation; 1716 1717 for (i = 0; i < adapter->num_tx_queues; i++) { 1718 adapter->tx_ring[i].count = adapter->tx_ring_count; 1719 adapter->tx_ring[i].queue_index = i; 1720 adapter->tx_ring[i].reg_idx = i; 1721 adapter->tx_ring[i].dev = &adapter->pdev->dev; 1722 adapter->tx_ring[i].netdev = adapter->netdev; 1723 } 1724 1725 for (i = 0; i < adapter->num_rx_queues; i++) { 1726 adapter->rx_ring[i].count = adapter->rx_ring_count; 1727 adapter->rx_ring[i].queue_index = i; 1728 adapter->rx_ring[i].reg_idx = i; 1729 adapter->rx_ring[i].dev = &adapter->pdev->dev; 1730 adapter->rx_ring[i].netdev = adapter->netdev; 1731 } 1732 1733 return 0; 1734 1735 err_rx_ring_allocation: 1736 kfree(adapter->tx_ring); 1737 err_tx_ring_allocation: 1738 return -ENOMEM; 1739 } 1740 1741 /** 1742 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 1743 * @adapter: board private structure to initialize 1744 * 1745 * Attempt to configure the interrupts using the best available 1746 * capabilities of the hardware and the kernel. 1747 **/ 1748 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 1749 { 1750 struct net_device *netdev = adapter->netdev; 1751 int err = 0; 1752 int vector, v_budget; 1753 1754 /* 1755 * It's easy to be greedy for MSI-X vectors, but it really 1756 * doesn't do us much good if we have a lot more vectors 1757 * than CPU's. So let's be conservative and only ask for 1758 * (roughly) the same number of vectors as there are CPU's. 1759 * The default is to use pairs of vectors. 1760 */ 1761 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); 1762 v_budget = min_t(int, v_budget, num_online_cpus()); 1763 v_budget += NON_Q_VECTORS; 1764 1765 /* A failure in MSI-X entry allocation isn't fatal, but it does 1766 * mean we disable MSI-X capabilities of the adapter. */ 1767 adapter->msix_entries = kcalloc(v_budget, 1768 sizeof(struct msix_entry), GFP_KERNEL); 1769 if (!adapter->msix_entries) { 1770 err = -ENOMEM; 1771 goto out; 1772 } 1773 1774 for (vector = 0; vector < v_budget; vector++) 1775 adapter->msix_entries[vector].entry = vector; 1776 1777 ixgbevf_acquire_msix_vectors(adapter, v_budget); 1778 1779 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 1780 if (err) 1781 goto out; 1782 1783 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 1784 1785 out: 1786 return err; 1787 } 1788 1789 /** 1790 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 1791 * @adapter: board private structure to initialize 1792 * 1793 * We allocate one q_vector per queue interrupt. If allocation fails we 1794 * return -ENOMEM. 1795 **/ 1796 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 1797 { 1798 int q_idx, num_q_vectors; 1799 struct ixgbevf_q_vector *q_vector; 1800 1801 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1802 1803 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1804 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL); 1805 if (!q_vector) 1806 goto err_out; 1807 q_vector->adapter = adapter; 1808 q_vector->v_idx = q_idx; 1809 netif_napi_add(adapter->netdev, &q_vector->napi, 1810 ixgbevf_poll, 64); 1811 adapter->q_vector[q_idx] = q_vector; 1812 } 1813 1814 return 0; 1815 1816 err_out: 1817 while (q_idx) { 1818 q_idx--; 1819 q_vector = adapter->q_vector[q_idx]; 1820 netif_napi_del(&q_vector->napi); 1821 kfree(q_vector); 1822 adapter->q_vector[q_idx] = NULL; 1823 } 1824 return -ENOMEM; 1825 } 1826 1827 /** 1828 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 1829 * @adapter: board private structure to initialize 1830 * 1831 * This function frees the memory allocated to the q_vectors. In addition if 1832 * NAPI is enabled it will delete any references to the NAPI struct prior 1833 * to freeing the q_vector. 1834 **/ 1835 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 1836 { 1837 int q_idx, num_q_vectors; 1838 int napi_vectors; 1839 1840 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1841 napi_vectors = adapter->num_rx_queues; 1842 1843 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1844 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; 1845 1846 adapter->q_vector[q_idx] = NULL; 1847 if (q_idx < napi_vectors) 1848 netif_napi_del(&q_vector->napi); 1849 kfree(q_vector); 1850 } 1851 } 1852 1853 /** 1854 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 1855 * @adapter: board private structure 1856 * 1857 **/ 1858 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 1859 { 1860 pci_disable_msix(adapter->pdev); 1861 kfree(adapter->msix_entries); 1862 adapter->msix_entries = NULL; 1863 } 1864 1865 /** 1866 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 1867 * @adapter: board private structure to initialize 1868 * 1869 **/ 1870 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 1871 { 1872 int err; 1873 1874 /* Number of supported queues */ 1875 ixgbevf_set_num_queues(adapter); 1876 1877 err = ixgbevf_set_interrupt_capability(adapter); 1878 if (err) { 1879 hw_dbg(&adapter->hw, 1880 "Unable to setup interrupt capabilities\n"); 1881 goto err_set_interrupt; 1882 } 1883 1884 err = ixgbevf_alloc_q_vectors(adapter); 1885 if (err) { 1886 hw_dbg(&adapter->hw, "Unable to allocate memory for queue " 1887 "vectors\n"); 1888 goto err_alloc_q_vectors; 1889 } 1890 1891 err = ixgbevf_alloc_queues(adapter); 1892 if (err) { 1893 pr_err("Unable to allocate memory for queues\n"); 1894 goto err_alloc_queues; 1895 } 1896 1897 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, " 1898 "Tx Queue count = %u\n", 1899 (adapter->num_rx_queues > 1) ? "Enabled" : 1900 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues); 1901 1902 set_bit(__IXGBEVF_DOWN, &adapter->state); 1903 1904 return 0; 1905 err_alloc_queues: 1906 ixgbevf_free_q_vectors(adapter); 1907 err_alloc_q_vectors: 1908 ixgbevf_reset_interrupt_capability(adapter); 1909 err_set_interrupt: 1910 return err; 1911 } 1912 1913 /** 1914 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings 1915 * @adapter: board private structure to clear interrupt scheme on 1916 * 1917 * We go through and clear interrupt specific resources and reset the structure 1918 * to pre-load conditions 1919 **/ 1920 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) 1921 { 1922 adapter->num_tx_queues = 0; 1923 adapter->num_rx_queues = 0; 1924 1925 ixgbevf_free_q_vectors(adapter); 1926 ixgbevf_reset_interrupt_capability(adapter); 1927 } 1928 1929 /** 1930 * ixgbevf_sw_init - Initialize general software structures 1931 * (struct ixgbevf_adapter) 1932 * @adapter: board private structure to initialize 1933 * 1934 * ixgbevf_sw_init initializes the Adapter private data structure. 1935 * Fields are initialized based on PCI device information and 1936 * OS network device settings (MTU size). 1937 **/ 1938 static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 1939 { 1940 struct ixgbe_hw *hw = &adapter->hw; 1941 struct pci_dev *pdev = adapter->pdev; 1942 int err; 1943 1944 /* PCI config space info */ 1945 1946 hw->vendor_id = pdev->vendor; 1947 hw->device_id = pdev->device; 1948 hw->revision_id = pdev->revision; 1949 hw->subsystem_vendor_id = pdev->subsystem_vendor; 1950 hw->subsystem_device_id = pdev->subsystem_device; 1951 1952 hw->mbx.ops.init_params(hw); 1953 hw->mac.max_tx_queues = MAX_TX_QUEUES; 1954 hw->mac.max_rx_queues = MAX_RX_QUEUES; 1955 err = hw->mac.ops.reset_hw(hw); 1956 if (err) { 1957 dev_info(&pdev->dev, 1958 "PF still in reset state, assigning new address\n"); 1959 eth_hw_addr_random(adapter->netdev); 1960 memcpy(adapter->hw.mac.addr, adapter->netdev->dev_addr, 1961 adapter->netdev->addr_len); 1962 } else { 1963 err = hw->mac.ops.init_hw(hw); 1964 if (err) { 1965 pr_err("init_shared_code failed: %d\n", err); 1966 goto out; 1967 } 1968 memcpy(adapter->netdev->dev_addr, adapter->hw.mac.addr, 1969 adapter->netdev->addr_len); 1970 } 1971 1972 /* lock to protect mailbox accesses */ 1973 spin_lock_init(&adapter->mbx_lock); 1974 1975 /* Enable dynamic interrupt throttling rates */ 1976 adapter->rx_itr_setting = 1; 1977 adapter->tx_itr_setting = 1; 1978 1979 /* set default ring sizes */ 1980 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 1981 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 1982 1983 set_bit(__IXGBEVF_DOWN, &adapter->state); 1984 return 0; 1985 1986 out: 1987 return err; 1988 } 1989 1990 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 1991 { \ 1992 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 1993 if (current_counter < last_counter) \ 1994 counter += 0x100000000LL; \ 1995 last_counter = current_counter; \ 1996 counter &= 0xFFFFFFFF00000000LL; \ 1997 counter |= current_counter; \ 1998 } 1999 2000 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 2001 { \ 2002 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 2003 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 2004 u64 current_counter = (current_counter_msb << 32) | \ 2005 current_counter_lsb; \ 2006 if (current_counter < last_counter) \ 2007 counter += 0x1000000000LL; \ 2008 last_counter = current_counter; \ 2009 counter &= 0xFFFFFFF000000000LL; \ 2010 counter |= current_counter; \ 2011 } 2012 /** 2013 * ixgbevf_update_stats - Update the board statistics counters. 2014 * @adapter: board private structure 2015 **/ 2016 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 2017 { 2018 struct ixgbe_hw *hw = &adapter->hw; 2019 2020 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 2021 adapter->stats.vfgprc); 2022 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 2023 adapter->stats.vfgptc); 2024 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2025 adapter->stats.last_vfgorc, 2026 adapter->stats.vfgorc); 2027 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2028 adapter->stats.last_vfgotc, 2029 adapter->stats.vfgotc); 2030 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 2031 adapter->stats.vfmprc); 2032 } 2033 2034 /** 2035 * ixgbevf_watchdog - Timer Call-back 2036 * @data: pointer to adapter cast into an unsigned long 2037 **/ 2038 static void ixgbevf_watchdog(unsigned long data) 2039 { 2040 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data; 2041 struct ixgbe_hw *hw = &adapter->hw; 2042 u32 eics = 0; 2043 int i; 2044 2045 /* 2046 * Do the watchdog outside of interrupt context due to the lovely 2047 * delays that some of the newer hardware requires 2048 */ 2049 2050 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 2051 goto watchdog_short_circuit; 2052 2053 /* get one bit for every active tx/rx interrupt vector */ 2054 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 2055 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 2056 if (qv->rx.ring || qv->tx.ring) 2057 eics |= 1 << i; 2058 } 2059 2060 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics); 2061 2062 watchdog_short_circuit: 2063 schedule_work(&adapter->watchdog_task); 2064 } 2065 2066 /** 2067 * ixgbevf_tx_timeout - Respond to a Tx Hang 2068 * @netdev: network interface device structure 2069 **/ 2070 static void ixgbevf_tx_timeout(struct net_device *netdev) 2071 { 2072 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2073 2074 /* Do the reset outside of interrupt context */ 2075 schedule_work(&adapter->reset_task); 2076 } 2077 2078 static void ixgbevf_reset_task(struct work_struct *work) 2079 { 2080 struct ixgbevf_adapter *adapter; 2081 adapter = container_of(work, struct ixgbevf_adapter, reset_task); 2082 2083 /* If we're already down or resetting, just bail */ 2084 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2085 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2086 return; 2087 2088 adapter->tx_timeout_count++; 2089 2090 ixgbevf_reinit_locked(adapter); 2091 } 2092 2093 /** 2094 * ixgbevf_watchdog_task - worker thread to bring link up 2095 * @work: pointer to work_struct containing our data 2096 **/ 2097 static void ixgbevf_watchdog_task(struct work_struct *work) 2098 { 2099 struct ixgbevf_adapter *adapter = container_of(work, 2100 struct ixgbevf_adapter, 2101 watchdog_task); 2102 struct net_device *netdev = adapter->netdev; 2103 struct ixgbe_hw *hw = &adapter->hw; 2104 u32 link_speed = adapter->link_speed; 2105 bool link_up = adapter->link_up; 2106 2107 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK; 2108 2109 /* 2110 * Always check the link on the watchdog because we have 2111 * no LSC interrupt 2112 */ 2113 if (hw->mac.ops.check_link) { 2114 s32 need_reset; 2115 2116 spin_lock(&adapter->mbx_lock); 2117 2118 need_reset = hw->mac.ops.check_link(hw, &link_speed, 2119 &link_up, false); 2120 2121 spin_unlock(&adapter->mbx_lock); 2122 2123 if (need_reset) { 2124 adapter->link_up = link_up; 2125 adapter->link_speed = link_speed; 2126 netif_carrier_off(netdev); 2127 netif_tx_stop_all_queues(netdev); 2128 schedule_work(&adapter->reset_task); 2129 goto pf_has_reset; 2130 } 2131 } else { 2132 /* always assume link is up, if no check link 2133 * function */ 2134 link_speed = IXGBE_LINK_SPEED_10GB_FULL; 2135 link_up = true; 2136 } 2137 adapter->link_up = link_up; 2138 adapter->link_speed = link_speed; 2139 2140 if (link_up) { 2141 if (!netif_carrier_ok(netdev)) { 2142 hw_dbg(&adapter->hw, "NIC Link is Up, %u Gbps\n", 2143 (link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 2144 10 : 1); 2145 netif_carrier_on(netdev); 2146 netif_tx_wake_all_queues(netdev); 2147 } 2148 } else { 2149 adapter->link_up = false; 2150 adapter->link_speed = 0; 2151 if (netif_carrier_ok(netdev)) { 2152 hw_dbg(&adapter->hw, "NIC Link is Down\n"); 2153 netif_carrier_off(netdev); 2154 netif_tx_stop_all_queues(netdev); 2155 } 2156 } 2157 2158 ixgbevf_update_stats(adapter); 2159 2160 pf_has_reset: 2161 /* Reset the timer */ 2162 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 2163 mod_timer(&adapter->watchdog_timer, 2164 round_jiffies(jiffies + (2 * HZ))); 2165 2166 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK; 2167 } 2168 2169 /** 2170 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 2171 * @adapter: board private structure 2172 * @tx_ring: Tx descriptor ring for a specific queue 2173 * 2174 * Free all transmit software resources 2175 **/ 2176 void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter, 2177 struct ixgbevf_ring *tx_ring) 2178 { 2179 struct pci_dev *pdev = adapter->pdev; 2180 2181 ixgbevf_clean_tx_ring(adapter, tx_ring); 2182 2183 vfree(tx_ring->tx_buffer_info); 2184 tx_ring->tx_buffer_info = NULL; 2185 2186 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc, 2187 tx_ring->dma); 2188 2189 tx_ring->desc = NULL; 2190 } 2191 2192 /** 2193 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 2194 * @adapter: board private structure 2195 * 2196 * Free all transmit software resources 2197 **/ 2198 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 2199 { 2200 int i; 2201 2202 for (i = 0; i < adapter->num_tx_queues; i++) 2203 if (adapter->tx_ring[i].desc) 2204 ixgbevf_free_tx_resources(adapter, 2205 &adapter->tx_ring[i]); 2206 2207 } 2208 2209 /** 2210 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 2211 * @adapter: board private structure 2212 * @tx_ring: tx descriptor ring (for a specific queue) to setup 2213 * 2214 * Return 0 on success, negative on failure 2215 **/ 2216 int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter, 2217 struct ixgbevf_ring *tx_ring) 2218 { 2219 struct pci_dev *pdev = adapter->pdev; 2220 int size; 2221 2222 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2223 tx_ring->tx_buffer_info = vzalloc(size); 2224 if (!tx_ring->tx_buffer_info) 2225 goto err; 2226 2227 /* round up to nearest 4K */ 2228 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 2229 tx_ring->size = ALIGN(tx_ring->size, 4096); 2230 2231 tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size, 2232 &tx_ring->dma, GFP_KERNEL); 2233 if (!tx_ring->desc) 2234 goto err; 2235 2236 tx_ring->next_to_use = 0; 2237 tx_ring->next_to_clean = 0; 2238 return 0; 2239 2240 err: 2241 vfree(tx_ring->tx_buffer_info); 2242 tx_ring->tx_buffer_info = NULL; 2243 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit " 2244 "descriptor ring\n"); 2245 return -ENOMEM; 2246 } 2247 2248 /** 2249 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 2250 * @adapter: board private structure 2251 * 2252 * If this function returns with an error, then it's possible one or 2253 * more of the rings is populated (while the rest are not). It is the 2254 * callers duty to clean those orphaned rings. 2255 * 2256 * Return 0 on success, negative on failure 2257 **/ 2258 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 2259 { 2260 int i, err = 0; 2261 2262 for (i = 0; i < adapter->num_tx_queues; i++) { 2263 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]); 2264 if (!err) 2265 continue; 2266 hw_dbg(&adapter->hw, 2267 "Allocation for Tx Queue %u failed\n", i); 2268 break; 2269 } 2270 2271 return err; 2272 } 2273 2274 /** 2275 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 2276 * @adapter: board private structure 2277 * @rx_ring: rx descriptor ring (for a specific queue) to setup 2278 * 2279 * Returns 0 on success, negative on failure 2280 **/ 2281 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, 2282 struct ixgbevf_ring *rx_ring) 2283 { 2284 struct pci_dev *pdev = adapter->pdev; 2285 int size; 2286 2287 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 2288 rx_ring->rx_buffer_info = vzalloc(size); 2289 if (!rx_ring->rx_buffer_info) 2290 goto alloc_failed; 2291 2292 /* Round up to nearest 4K */ 2293 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 2294 rx_ring->size = ALIGN(rx_ring->size, 4096); 2295 2296 rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size, 2297 &rx_ring->dma, GFP_KERNEL); 2298 2299 if (!rx_ring->desc) { 2300 hw_dbg(&adapter->hw, 2301 "Unable to allocate memory for " 2302 "the receive descriptor ring\n"); 2303 vfree(rx_ring->rx_buffer_info); 2304 rx_ring->rx_buffer_info = NULL; 2305 goto alloc_failed; 2306 } 2307 2308 rx_ring->next_to_clean = 0; 2309 rx_ring->next_to_use = 0; 2310 2311 return 0; 2312 alloc_failed: 2313 return -ENOMEM; 2314 } 2315 2316 /** 2317 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 2318 * @adapter: board private structure 2319 * 2320 * If this function returns with an error, then it's possible one or 2321 * more of the rings is populated (while the rest are not). It is the 2322 * callers duty to clean those orphaned rings. 2323 * 2324 * Return 0 on success, negative on failure 2325 **/ 2326 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 2327 { 2328 int i, err = 0; 2329 2330 for (i = 0; i < adapter->num_rx_queues; i++) { 2331 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]); 2332 if (!err) 2333 continue; 2334 hw_dbg(&adapter->hw, 2335 "Allocation for Rx Queue %u failed\n", i); 2336 break; 2337 } 2338 return err; 2339 } 2340 2341 /** 2342 * ixgbevf_free_rx_resources - Free Rx Resources 2343 * @adapter: board private structure 2344 * @rx_ring: ring to clean the resources from 2345 * 2346 * Free all receive software resources 2347 **/ 2348 void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter, 2349 struct ixgbevf_ring *rx_ring) 2350 { 2351 struct pci_dev *pdev = adapter->pdev; 2352 2353 ixgbevf_clean_rx_ring(adapter, rx_ring); 2354 2355 vfree(rx_ring->rx_buffer_info); 2356 rx_ring->rx_buffer_info = NULL; 2357 2358 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, 2359 rx_ring->dma); 2360 2361 rx_ring->desc = NULL; 2362 } 2363 2364 /** 2365 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 2366 * @adapter: board private structure 2367 * 2368 * Free all receive software resources 2369 **/ 2370 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 2371 { 2372 int i; 2373 2374 for (i = 0; i < adapter->num_rx_queues; i++) 2375 if (adapter->rx_ring[i].desc) 2376 ixgbevf_free_rx_resources(adapter, 2377 &adapter->rx_ring[i]); 2378 } 2379 2380 /** 2381 * ixgbevf_open - Called when a network interface is made active 2382 * @netdev: network interface device structure 2383 * 2384 * Returns 0 on success, negative value on failure 2385 * 2386 * The open entry point is called when a network interface is made 2387 * active by the system (IFF_UP). At this point all resources needed 2388 * for transmit and receive operations are allocated, the interrupt 2389 * handler is registered with the OS, the watchdog timer is started, 2390 * and the stack is notified that the interface is ready. 2391 **/ 2392 static int ixgbevf_open(struct net_device *netdev) 2393 { 2394 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2395 struct ixgbe_hw *hw = &adapter->hw; 2396 int err; 2397 2398 /* disallow open during test */ 2399 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 2400 return -EBUSY; 2401 2402 if (hw->adapter_stopped) { 2403 ixgbevf_reset(adapter); 2404 /* if adapter is still stopped then PF isn't up and 2405 * the vf can't start. */ 2406 if (hw->adapter_stopped) { 2407 err = IXGBE_ERR_MBX; 2408 pr_err("Unable to start - perhaps the PF Driver isn't " 2409 "up yet\n"); 2410 goto err_setup_reset; 2411 } 2412 } 2413 2414 ixgbevf_negotiate_api(adapter); 2415 2416 /* allocate transmit descriptors */ 2417 err = ixgbevf_setup_all_tx_resources(adapter); 2418 if (err) 2419 goto err_setup_tx; 2420 2421 /* allocate receive descriptors */ 2422 err = ixgbevf_setup_all_rx_resources(adapter); 2423 if (err) 2424 goto err_setup_rx; 2425 2426 ixgbevf_configure(adapter); 2427 2428 /* 2429 * Map the Tx/Rx rings to the vectors we were allotted. 2430 * if request_irq will be called in this function map_rings 2431 * must be called *before* up_complete 2432 */ 2433 ixgbevf_map_rings_to_vectors(adapter); 2434 2435 ixgbevf_up_complete(adapter); 2436 2437 /* clear any pending interrupts, may auto mask */ 2438 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2439 err = ixgbevf_request_irq(adapter); 2440 if (err) 2441 goto err_req_irq; 2442 2443 ixgbevf_irq_enable(adapter); 2444 2445 return 0; 2446 2447 err_req_irq: 2448 ixgbevf_down(adapter); 2449 ixgbevf_free_irq(adapter); 2450 err_setup_rx: 2451 ixgbevf_free_all_rx_resources(adapter); 2452 err_setup_tx: 2453 ixgbevf_free_all_tx_resources(adapter); 2454 ixgbevf_reset(adapter); 2455 2456 err_setup_reset: 2457 2458 return err; 2459 } 2460 2461 /** 2462 * ixgbevf_close - Disables a network interface 2463 * @netdev: network interface device structure 2464 * 2465 * Returns 0, this is not allowed to fail 2466 * 2467 * The close entry point is called when an interface is de-activated 2468 * by the OS. The hardware is still under the drivers control, but 2469 * needs to be disabled. A global MAC reset is issued to stop the 2470 * hardware, and all transmit and receive resources are freed. 2471 **/ 2472 static int ixgbevf_close(struct net_device *netdev) 2473 { 2474 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2475 2476 ixgbevf_down(adapter); 2477 ixgbevf_free_irq(adapter); 2478 2479 ixgbevf_free_all_tx_resources(adapter); 2480 ixgbevf_free_all_rx_resources(adapter); 2481 2482 return 0; 2483 } 2484 2485 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, 2486 u32 vlan_macip_lens, u32 type_tucmd, 2487 u32 mss_l4len_idx) 2488 { 2489 struct ixgbe_adv_tx_context_desc *context_desc; 2490 u16 i = tx_ring->next_to_use; 2491 2492 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i); 2493 2494 i++; 2495 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 2496 2497 /* set bits to identify this as an advanced context descriptor */ 2498 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2499 2500 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 2501 context_desc->seqnum_seed = 0; 2502 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 2503 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 2504 } 2505 2506 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, 2507 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len) 2508 { 2509 u32 vlan_macip_lens, type_tucmd; 2510 u32 mss_l4len_idx, l4len; 2511 2512 if (!skb_is_gso(skb)) 2513 return 0; 2514 2515 if (skb_header_cloned(skb)) { 2516 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2517 if (err) 2518 return err; 2519 } 2520 2521 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 2522 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 2523 2524 if (skb->protocol == htons(ETH_P_IP)) { 2525 struct iphdr *iph = ip_hdr(skb); 2526 iph->tot_len = 0; 2527 iph->check = 0; 2528 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 2529 iph->daddr, 0, 2530 IPPROTO_TCP, 2531 0); 2532 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 2533 } else if (skb_is_gso_v6(skb)) { 2534 ipv6_hdr(skb)->payload_len = 0; 2535 tcp_hdr(skb)->check = 2536 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 2537 &ipv6_hdr(skb)->daddr, 2538 0, IPPROTO_TCP, 0); 2539 } 2540 2541 /* compute header lengths */ 2542 l4len = tcp_hdrlen(skb); 2543 *hdr_len += l4len; 2544 *hdr_len = skb_transport_offset(skb) + l4len; 2545 2546 /* mss_l4len_id: use 1 as index for TSO */ 2547 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; 2548 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 2549 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT; 2550 2551 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 2552 vlan_macip_lens = skb_network_header_len(skb); 2553 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 2554 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 2555 2556 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 2557 type_tucmd, mss_l4len_idx); 2558 2559 return 1; 2560 } 2561 2562 static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, 2563 struct sk_buff *skb, u32 tx_flags) 2564 { 2565 2566 2567 2568 u32 vlan_macip_lens = 0; 2569 u32 mss_l4len_idx = 0; 2570 u32 type_tucmd = 0; 2571 2572 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2573 u8 l4_hdr = 0; 2574 switch (skb->protocol) { 2575 case __constant_htons(ETH_P_IP): 2576 vlan_macip_lens |= skb_network_header_len(skb); 2577 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 2578 l4_hdr = ip_hdr(skb)->protocol; 2579 break; 2580 case __constant_htons(ETH_P_IPV6): 2581 vlan_macip_lens |= skb_network_header_len(skb); 2582 l4_hdr = ipv6_hdr(skb)->nexthdr; 2583 break; 2584 default: 2585 if (unlikely(net_ratelimit())) { 2586 dev_warn(tx_ring->dev, 2587 "partial checksum but proto=%x!\n", 2588 skb->protocol); 2589 } 2590 break; 2591 } 2592 2593 switch (l4_hdr) { 2594 case IPPROTO_TCP: 2595 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2596 mss_l4len_idx = tcp_hdrlen(skb) << 2597 IXGBE_ADVTXD_L4LEN_SHIFT; 2598 break; 2599 case IPPROTO_SCTP: 2600 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 2601 mss_l4len_idx = sizeof(struct sctphdr) << 2602 IXGBE_ADVTXD_L4LEN_SHIFT; 2603 break; 2604 case IPPROTO_UDP: 2605 mss_l4len_idx = sizeof(struct udphdr) << 2606 IXGBE_ADVTXD_L4LEN_SHIFT; 2607 break; 2608 default: 2609 if (unlikely(net_ratelimit())) { 2610 dev_warn(tx_ring->dev, 2611 "partial checksum but l4 proto=%x!\n", 2612 l4_hdr); 2613 } 2614 break; 2615 } 2616 } 2617 2618 /* vlan_macip_lens: MACLEN, VLAN tag */ 2619 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 2620 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 2621 2622 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 2623 type_tucmd, mss_l4len_idx); 2624 2625 return (skb->ip_summed == CHECKSUM_PARTIAL); 2626 } 2627 2628 static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, 2629 struct sk_buff *skb, u32 tx_flags, 2630 unsigned int first) 2631 { 2632 struct ixgbevf_tx_buffer *tx_buffer_info; 2633 unsigned int len; 2634 unsigned int total = skb->len; 2635 unsigned int offset = 0, size; 2636 int count = 0; 2637 unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 2638 unsigned int f; 2639 int i; 2640 2641 i = tx_ring->next_to_use; 2642 2643 len = min(skb_headlen(skb), total); 2644 while (len) { 2645 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2646 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); 2647 2648 tx_buffer_info->length = size; 2649 tx_buffer_info->mapped_as_page = false; 2650 tx_buffer_info->dma = dma_map_single(tx_ring->dev, 2651 skb->data + offset, 2652 size, DMA_TO_DEVICE); 2653 if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma)) 2654 goto dma_error; 2655 tx_buffer_info->next_to_watch = i; 2656 2657 len -= size; 2658 total -= size; 2659 offset += size; 2660 count++; 2661 i++; 2662 if (i == tx_ring->count) 2663 i = 0; 2664 } 2665 2666 for (f = 0; f < nr_frags; f++) { 2667 const struct skb_frag_struct *frag; 2668 2669 frag = &skb_shinfo(skb)->frags[f]; 2670 len = min((unsigned int)skb_frag_size(frag), total); 2671 offset = 0; 2672 2673 while (len) { 2674 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2675 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); 2676 2677 tx_buffer_info->length = size; 2678 tx_buffer_info->dma = 2679 skb_frag_dma_map(tx_ring->dev, frag, 2680 offset, size, DMA_TO_DEVICE); 2681 tx_buffer_info->mapped_as_page = true; 2682 if (dma_mapping_error(tx_ring->dev, 2683 tx_buffer_info->dma)) 2684 goto dma_error; 2685 tx_buffer_info->next_to_watch = i; 2686 2687 len -= size; 2688 total -= size; 2689 offset += size; 2690 count++; 2691 i++; 2692 if (i == tx_ring->count) 2693 i = 0; 2694 } 2695 if (total == 0) 2696 break; 2697 } 2698 2699 if (i == 0) 2700 i = tx_ring->count - 1; 2701 else 2702 i = i - 1; 2703 tx_ring->tx_buffer_info[i].skb = skb; 2704 tx_ring->tx_buffer_info[first].next_to_watch = i; 2705 tx_ring->tx_buffer_info[first].time_stamp = jiffies; 2706 2707 return count; 2708 2709 dma_error: 2710 dev_err(tx_ring->dev, "TX DMA map failed\n"); 2711 2712 /* clear timestamp and dma mappings for failed tx_buffer_info map */ 2713 tx_buffer_info->dma = 0; 2714 tx_buffer_info->next_to_watch = 0; 2715 count--; 2716 2717 /* clear timestamp and dma mappings for remaining portion of packet */ 2718 while (count >= 0) { 2719 count--; 2720 i--; 2721 if (i < 0) 2722 i += tx_ring->count; 2723 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2724 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 2725 } 2726 2727 return count; 2728 } 2729 2730 static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags, 2731 int count, u32 paylen, u8 hdr_len) 2732 { 2733 union ixgbe_adv_tx_desc *tx_desc = NULL; 2734 struct ixgbevf_tx_buffer *tx_buffer_info; 2735 u32 olinfo_status = 0, cmd_type_len = 0; 2736 unsigned int i; 2737 2738 u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS; 2739 2740 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; 2741 2742 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 2743 2744 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 2745 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 2746 2747 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 2748 olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM; 2749 2750 if (tx_flags & IXGBE_TX_FLAGS_TSO) { 2751 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 2752 2753 /* use index 1 context for tso */ 2754 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT); 2755 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 2756 olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM; 2757 2758 } 2759 2760 /* 2761 * Check Context must be set if Tx switch is enabled, which it 2762 * always is for case where virtual functions are running 2763 */ 2764 olinfo_status |= IXGBE_ADVTXD_CC; 2765 2766 olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT); 2767 2768 i = tx_ring->next_to_use; 2769 while (count--) { 2770 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2771 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 2772 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma); 2773 tx_desc->read.cmd_type_len = 2774 cpu_to_le32(cmd_type_len | tx_buffer_info->length); 2775 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 2776 i++; 2777 if (i == tx_ring->count) 2778 i = 0; 2779 } 2780 2781 tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd); 2782 2783 tx_ring->next_to_use = i; 2784 } 2785 2786 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 2787 { 2788 struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); 2789 2790 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 2791 /* Herbert's original patch had: 2792 * smp_mb__after_netif_stop_queue(); 2793 * but since that doesn't exist yet, just open code it. */ 2794 smp_mb(); 2795 2796 /* We need to check again in a case another CPU has just 2797 * made room available. */ 2798 if (likely(IXGBE_DESC_UNUSED(tx_ring) < size)) 2799 return -EBUSY; 2800 2801 /* A reprieve! - use start_queue because it doesn't call schedule */ 2802 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 2803 ++adapter->restart_queue; 2804 return 0; 2805 } 2806 2807 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 2808 { 2809 if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size)) 2810 return 0; 2811 return __ixgbevf_maybe_stop_tx(tx_ring, size); 2812 } 2813 2814 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 2815 { 2816 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2817 struct ixgbevf_ring *tx_ring; 2818 unsigned int first; 2819 unsigned int tx_flags = 0; 2820 u8 hdr_len = 0; 2821 int r_idx = 0, tso; 2822 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 2823 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 2824 unsigned short f; 2825 #endif 2826 2827 tx_ring = &adapter->tx_ring[r_idx]; 2828 2829 /* 2830 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 2831 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 2832 * + 2 desc gap to keep tail from touching head, 2833 * + 1 desc for context descriptor, 2834 * otherwise try next time 2835 */ 2836 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 2837 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 2838 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 2839 #else 2840 count += skb_shinfo(skb)->nr_frags; 2841 #endif 2842 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { 2843 adapter->tx_busy++; 2844 return NETDEV_TX_BUSY; 2845 } 2846 2847 if (vlan_tx_tag_present(skb)) { 2848 tx_flags |= vlan_tx_tag_get(skb); 2849 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 2850 tx_flags |= IXGBE_TX_FLAGS_VLAN; 2851 } 2852 2853 first = tx_ring->next_to_use; 2854 2855 if (skb->protocol == htons(ETH_P_IP)) 2856 tx_flags |= IXGBE_TX_FLAGS_IPV4; 2857 tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len); 2858 if (tso < 0) { 2859 dev_kfree_skb_any(skb); 2860 return NETDEV_TX_OK; 2861 } 2862 2863 if (tso) 2864 tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM; 2865 else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags)) 2866 tx_flags |= IXGBE_TX_FLAGS_CSUM; 2867 2868 ixgbevf_tx_queue(tx_ring, tx_flags, 2869 ixgbevf_tx_map(tx_ring, skb, tx_flags, first), 2870 skb->len, hdr_len); 2871 /* 2872 * Force memory writes to complete before letting h/w 2873 * know there are new descriptors to fetch. (Only 2874 * applicable for weak-ordered memory model archs, 2875 * such as IA-64). 2876 */ 2877 wmb(); 2878 2879 writel(tx_ring->next_to_use, adapter->hw.hw_addr + tx_ring->tail); 2880 2881 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); 2882 2883 return NETDEV_TX_OK; 2884 } 2885 2886 /** 2887 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 2888 * @netdev: network interface device structure 2889 * @p: pointer to an address structure 2890 * 2891 * Returns 0 on success, negative on failure 2892 **/ 2893 static int ixgbevf_set_mac(struct net_device *netdev, void *p) 2894 { 2895 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2896 struct ixgbe_hw *hw = &adapter->hw; 2897 struct sockaddr *addr = p; 2898 2899 if (!is_valid_ether_addr(addr->sa_data)) 2900 return -EADDRNOTAVAIL; 2901 2902 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 2903 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 2904 2905 spin_lock(&adapter->mbx_lock); 2906 2907 if (hw->mac.ops.set_rar) 2908 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 2909 2910 spin_unlock(&adapter->mbx_lock); 2911 2912 return 0; 2913 } 2914 2915 /** 2916 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 2917 * @netdev: network interface device structure 2918 * @new_mtu: new value for maximum frame size 2919 * 2920 * Returns 0 on success, negative on failure 2921 **/ 2922 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 2923 { 2924 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2925 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 2926 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; 2927 2928 if (adapter->hw.mac.type == ixgbe_mac_X540_vf) 2929 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 2930 2931 /* MTU < 68 is an error and causes problems on some kernels */ 2932 if ((new_mtu < 68) || (max_frame > max_possible_frame)) 2933 return -EINVAL; 2934 2935 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n", 2936 netdev->mtu, new_mtu); 2937 /* must set new MTU before calling down or up */ 2938 netdev->mtu = new_mtu; 2939 2940 if (netif_running(netdev)) 2941 ixgbevf_reinit_locked(adapter); 2942 2943 return 0; 2944 } 2945 2946 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) 2947 { 2948 struct net_device *netdev = pci_get_drvdata(pdev); 2949 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2950 #ifdef CONFIG_PM 2951 int retval = 0; 2952 #endif 2953 2954 netif_device_detach(netdev); 2955 2956 if (netif_running(netdev)) { 2957 rtnl_lock(); 2958 ixgbevf_down(adapter); 2959 ixgbevf_free_irq(adapter); 2960 ixgbevf_free_all_tx_resources(adapter); 2961 ixgbevf_free_all_rx_resources(adapter); 2962 rtnl_unlock(); 2963 } 2964 2965 ixgbevf_clear_interrupt_scheme(adapter); 2966 2967 #ifdef CONFIG_PM 2968 retval = pci_save_state(pdev); 2969 if (retval) 2970 return retval; 2971 2972 #endif 2973 pci_disable_device(pdev); 2974 2975 return 0; 2976 } 2977 2978 #ifdef CONFIG_PM 2979 static int ixgbevf_resume(struct pci_dev *pdev) 2980 { 2981 struct ixgbevf_adapter *adapter = pci_get_drvdata(pdev); 2982 struct net_device *netdev = adapter->netdev; 2983 u32 err; 2984 2985 pci_set_power_state(pdev, PCI_D0); 2986 pci_restore_state(pdev); 2987 /* 2988 * pci_restore_state clears dev->state_saved so call 2989 * pci_save_state to restore it. 2990 */ 2991 pci_save_state(pdev); 2992 2993 err = pci_enable_device_mem(pdev); 2994 if (err) { 2995 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 2996 return err; 2997 } 2998 pci_set_master(pdev); 2999 3000 rtnl_lock(); 3001 err = ixgbevf_init_interrupt_scheme(adapter); 3002 rtnl_unlock(); 3003 if (err) { 3004 dev_err(&pdev->dev, "Cannot initialize interrupts\n"); 3005 return err; 3006 } 3007 3008 ixgbevf_reset(adapter); 3009 3010 if (netif_running(netdev)) { 3011 err = ixgbevf_open(netdev); 3012 if (err) 3013 return err; 3014 } 3015 3016 netif_device_attach(netdev); 3017 3018 return err; 3019 } 3020 3021 #endif /* CONFIG_PM */ 3022 static void ixgbevf_shutdown(struct pci_dev *pdev) 3023 { 3024 ixgbevf_suspend(pdev, PMSG_SUSPEND); 3025 } 3026 3027 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, 3028 struct rtnl_link_stats64 *stats) 3029 { 3030 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3031 unsigned int start; 3032 u64 bytes, packets; 3033 const struct ixgbevf_ring *ring; 3034 int i; 3035 3036 ixgbevf_update_stats(adapter); 3037 3038 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3039 3040 for (i = 0; i < adapter->num_rx_queues; i++) { 3041 ring = &adapter->rx_ring[i]; 3042 do { 3043 start = u64_stats_fetch_begin_bh(&ring->syncp); 3044 bytes = ring->total_bytes; 3045 packets = ring->total_packets; 3046 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 3047 stats->rx_bytes += bytes; 3048 stats->rx_packets += packets; 3049 } 3050 3051 for (i = 0; i < adapter->num_tx_queues; i++) { 3052 ring = &adapter->tx_ring[i]; 3053 do { 3054 start = u64_stats_fetch_begin_bh(&ring->syncp); 3055 bytes = ring->total_bytes; 3056 packets = ring->total_packets; 3057 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 3058 stats->tx_bytes += bytes; 3059 stats->tx_packets += packets; 3060 } 3061 3062 return stats; 3063 } 3064 3065 static const struct net_device_ops ixgbevf_netdev_ops = { 3066 .ndo_open = ixgbevf_open, 3067 .ndo_stop = ixgbevf_close, 3068 .ndo_start_xmit = ixgbevf_xmit_frame, 3069 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 3070 .ndo_get_stats64 = ixgbevf_get_stats, 3071 .ndo_validate_addr = eth_validate_addr, 3072 .ndo_set_mac_address = ixgbevf_set_mac, 3073 .ndo_change_mtu = ixgbevf_change_mtu, 3074 .ndo_tx_timeout = ixgbevf_tx_timeout, 3075 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 3076 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 3077 }; 3078 3079 static void ixgbevf_assign_netdev_ops(struct net_device *dev) 3080 { 3081 dev->netdev_ops = &ixgbevf_netdev_ops; 3082 ixgbevf_set_ethtool_ops(dev); 3083 dev->watchdog_timeo = 5 * HZ; 3084 } 3085 3086 /** 3087 * ixgbevf_probe - Device Initialization Routine 3088 * @pdev: PCI device information struct 3089 * @ent: entry in ixgbevf_pci_tbl 3090 * 3091 * Returns 0 on success, negative on failure 3092 * 3093 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 3094 * The OS initialization, configuring of the adapter private structure, 3095 * and a hardware reset occur. 3096 **/ 3097 static int __devinit ixgbevf_probe(struct pci_dev *pdev, 3098 const struct pci_device_id *ent) 3099 { 3100 struct net_device *netdev; 3101 struct ixgbevf_adapter *adapter = NULL; 3102 struct ixgbe_hw *hw = NULL; 3103 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 3104 static int cards_found; 3105 int err, pci_using_dac; 3106 3107 err = pci_enable_device(pdev); 3108 if (err) 3109 return err; 3110 3111 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && 3112 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { 3113 pci_using_dac = 1; 3114 } else { 3115 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 3116 if (err) { 3117 err = dma_set_coherent_mask(&pdev->dev, 3118 DMA_BIT_MASK(32)); 3119 if (err) { 3120 dev_err(&pdev->dev, "No usable DMA " 3121 "configuration, aborting\n"); 3122 goto err_dma; 3123 } 3124 } 3125 pci_using_dac = 0; 3126 } 3127 3128 err = pci_request_regions(pdev, ixgbevf_driver_name); 3129 if (err) { 3130 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 3131 goto err_pci_reg; 3132 } 3133 3134 pci_set_master(pdev); 3135 3136 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 3137 MAX_TX_QUEUES); 3138 if (!netdev) { 3139 err = -ENOMEM; 3140 goto err_alloc_etherdev; 3141 } 3142 3143 SET_NETDEV_DEV(netdev, &pdev->dev); 3144 3145 pci_set_drvdata(pdev, netdev); 3146 adapter = netdev_priv(netdev); 3147 3148 adapter->netdev = netdev; 3149 adapter->pdev = pdev; 3150 hw = &adapter->hw; 3151 hw->back = adapter; 3152 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 3153 3154 /* 3155 * call save state here in standalone driver because it relies on 3156 * adapter struct to exist, and needs to call netdev_priv 3157 */ 3158 pci_save_state(pdev); 3159 3160 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3161 pci_resource_len(pdev, 0)); 3162 if (!hw->hw_addr) { 3163 err = -EIO; 3164 goto err_ioremap; 3165 } 3166 3167 ixgbevf_assign_netdev_ops(netdev); 3168 3169 adapter->bd_number = cards_found; 3170 3171 /* Setup hw api */ 3172 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 3173 hw->mac.type = ii->mac; 3174 3175 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 3176 sizeof(struct ixgbe_mbx_operations)); 3177 3178 /* setup the private structure */ 3179 err = ixgbevf_sw_init(adapter); 3180 if (err) 3181 goto err_sw_init; 3182 3183 /* The HW MAC address was set and/or determined in sw_init */ 3184 memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len); 3185 3186 if (!is_valid_ether_addr(netdev->dev_addr)) { 3187 pr_err("invalid MAC address\n"); 3188 err = -EIO; 3189 goto err_sw_init; 3190 } 3191 3192 netdev->hw_features = NETIF_F_SG | 3193 NETIF_F_IP_CSUM | 3194 NETIF_F_IPV6_CSUM | 3195 NETIF_F_TSO | 3196 NETIF_F_TSO6 | 3197 NETIF_F_RXCSUM; 3198 3199 netdev->features = netdev->hw_features | 3200 NETIF_F_HW_VLAN_TX | 3201 NETIF_F_HW_VLAN_RX | 3202 NETIF_F_HW_VLAN_FILTER; 3203 3204 netdev->vlan_features |= NETIF_F_TSO; 3205 netdev->vlan_features |= NETIF_F_TSO6; 3206 netdev->vlan_features |= NETIF_F_IP_CSUM; 3207 netdev->vlan_features |= NETIF_F_IPV6_CSUM; 3208 netdev->vlan_features |= NETIF_F_SG; 3209 3210 if (pci_using_dac) 3211 netdev->features |= NETIF_F_HIGHDMA; 3212 3213 netdev->priv_flags |= IFF_UNICAST_FLT; 3214 3215 init_timer(&adapter->watchdog_timer); 3216 adapter->watchdog_timer.function = ixgbevf_watchdog; 3217 adapter->watchdog_timer.data = (unsigned long)adapter; 3218 3219 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task); 3220 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task); 3221 3222 err = ixgbevf_init_interrupt_scheme(adapter); 3223 if (err) 3224 goto err_sw_init; 3225 3226 /* pick up the PCI bus settings for reporting later */ 3227 if (hw->mac.ops.get_bus_info) 3228 hw->mac.ops.get_bus_info(hw); 3229 3230 strcpy(netdev->name, "eth%d"); 3231 3232 err = register_netdev(netdev); 3233 if (err) 3234 goto err_register; 3235 3236 netif_carrier_off(netdev); 3237 3238 ixgbevf_init_last_counter_stats(adapter); 3239 3240 /* print the MAC address */ 3241 hw_dbg(hw, "%pM\n", netdev->dev_addr); 3242 3243 hw_dbg(hw, "MAC: %d\n", hw->mac.type); 3244 3245 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n"); 3246 cards_found++; 3247 return 0; 3248 3249 err_register: 3250 ixgbevf_clear_interrupt_scheme(adapter); 3251 err_sw_init: 3252 ixgbevf_reset_interrupt_capability(adapter); 3253 iounmap(hw->hw_addr); 3254 err_ioremap: 3255 free_netdev(netdev); 3256 err_alloc_etherdev: 3257 pci_release_regions(pdev); 3258 err_pci_reg: 3259 err_dma: 3260 pci_disable_device(pdev); 3261 return err; 3262 } 3263 3264 /** 3265 * ixgbevf_remove - Device Removal Routine 3266 * @pdev: PCI device information struct 3267 * 3268 * ixgbevf_remove is called by the PCI subsystem to alert the driver 3269 * that it should release a PCI device. The could be caused by a 3270 * Hot-Plug event, or because the driver is going to be removed from 3271 * memory. 3272 **/ 3273 static void __devexit ixgbevf_remove(struct pci_dev *pdev) 3274 { 3275 struct net_device *netdev = pci_get_drvdata(pdev); 3276 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3277 3278 set_bit(__IXGBEVF_DOWN, &adapter->state); 3279 3280 del_timer_sync(&adapter->watchdog_timer); 3281 3282 cancel_work_sync(&adapter->reset_task); 3283 cancel_work_sync(&adapter->watchdog_task); 3284 3285 if (netdev->reg_state == NETREG_REGISTERED) 3286 unregister_netdev(netdev); 3287 3288 ixgbevf_clear_interrupt_scheme(adapter); 3289 ixgbevf_reset_interrupt_capability(adapter); 3290 3291 iounmap(adapter->hw.hw_addr); 3292 pci_release_regions(pdev); 3293 3294 hw_dbg(&adapter->hw, "Remove complete\n"); 3295 3296 kfree(adapter->tx_ring); 3297 kfree(adapter->rx_ring); 3298 3299 free_netdev(netdev); 3300 3301 pci_disable_device(pdev); 3302 } 3303 3304 /** 3305 * ixgbevf_io_error_detected - called when PCI error is detected 3306 * @pdev: Pointer to PCI device 3307 * @state: The current pci connection state 3308 * 3309 * This function is called after a PCI bus error affecting 3310 * this device has been detected. 3311 */ 3312 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, 3313 pci_channel_state_t state) 3314 { 3315 struct net_device *netdev = pci_get_drvdata(pdev); 3316 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3317 3318 netif_device_detach(netdev); 3319 3320 if (state == pci_channel_io_perm_failure) 3321 return PCI_ERS_RESULT_DISCONNECT; 3322 3323 if (netif_running(netdev)) 3324 ixgbevf_down(adapter); 3325 3326 pci_disable_device(pdev); 3327 3328 /* Request a slot slot reset. */ 3329 return PCI_ERS_RESULT_NEED_RESET; 3330 } 3331 3332 /** 3333 * ixgbevf_io_slot_reset - called after the pci bus has been reset. 3334 * @pdev: Pointer to PCI device 3335 * 3336 * Restart the card from scratch, as if from a cold-boot. Implementation 3337 * resembles the first-half of the ixgbevf_resume routine. 3338 */ 3339 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) 3340 { 3341 struct net_device *netdev = pci_get_drvdata(pdev); 3342 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3343 3344 if (pci_enable_device_mem(pdev)) { 3345 dev_err(&pdev->dev, 3346 "Cannot re-enable PCI device after reset.\n"); 3347 return PCI_ERS_RESULT_DISCONNECT; 3348 } 3349 3350 pci_set_master(pdev); 3351 3352 ixgbevf_reset(adapter); 3353 3354 return PCI_ERS_RESULT_RECOVERED; 3355 } 3356 3357 /** 3358 * ixgbevf_io_resume - called when traffic can start flowing again. 3359 * @pdev: Pointer to PCI device 3360 * 3361 * This callback is called when the error recovery driver tells us that 3362 * its OK to resume normal operation. Implementation resembles the 3363 * second-half of the ixgbevf_resume routine. 3364 */ 3365 static void ixgbevf_io_resume(struct pci_dev *pdev) 3366 { 3367 struct net_device *netdev = pci_get_drvdata(pdev); 3368 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3369 3370 if (netif_running(netdev)) 3371 ixgbevf_up(adapter); 3372 3373 netif_device_attach(netdev); 3374 } 3375 3376 /* PCI Error Recovery (ERS) */ 3377 static const struct pci_error_handlers ixgbevf_err_handler = { 3378 .error_detected = ixgbevf_io_error_detected, 3379 .slot_reset = ixgbevf_io_slot_reset, 3380 .resume = ixgbevf_io_resume, 3381 }; 3382 3383 static struct pci_driver ixgbevf_driver = { 3384 .name = ixgbevf_driver_name, 3385 .id_table = ixgbevf_pci_tbl, 3386 .probe = ixgbevf_probe, 3387 .remove = __devexit_p(ixgbevf_remove), 3388 #ifdef CONFIG_PM 3389 /* Power Management Hooks */ 3390 .suspend = ixgbevf_suspend, 3391 .resume = ixgbevf_resume, 3392 #endif 3393 .shutdown = ixgbevf_shutdown, 3394 .err_handler = &ixgbevf_err_handler 3395 }; 3396 3397 /** 3398 * ixgbevf_init_module - Driver Registration Routine 3399 * 3400 * ixgbevf_init_module is the first routine called when the driver is 3401 * loaded. All it does is register with the PCI subsystem. 3402 **/ 3403 static int __init ixgbevf_init_module(void) 3404 { 3405 int ret; 3406 pr_info("%s - version %s\n", ixgbevf_driver_string, 3407 ixgbevf_driver_version); 3408 3409 pr_info("%s\n", ixgbevf_copyright); 3410 3411 ret = pci_register_driver(&ixgbevf_driver); 3412 return ret; 3413 } 3414 3415 module_init(ixgbevf_init_module); 3416 3417 /** 3418 * ixgbevf_exit_module - Driver Exit Cleanup Routine 3419 * 3420 * ixgbevf_exit_module is called just before the driver is removed 3421 * from memory. 3422 **/ 3423 static void __exit ixgbevf_exit_module(void) 3424 { 3425 pci_unregister_driver(&ixgbevf_driver); 3426 } 3427 3428 #ifdef DEBUG 3429 /** 3430 * ixgbevf_get_hw_dev_name - return device name string 3431 * used by hardware layer to print debugging information 3432 **/ 3433 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 3434 { 3435 struct ixgbevf_adapter *adapter = hw->back; 3436 return adapter->netdev->name; 3437 } 3438 3439 #endif 3440 module_exit(ixgbevf_exit_module); 3441 3442 /* ixgbevf_main.c */ 3443