1 /******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2012 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 26 *******************************************************************************/ 27 28 29 /****************************************************************************** 30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 31 ******************************************************************************/ 32 33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35 #include <linux/types.h> 36 #include <linux/bitops.h> 37 #include <linux/module.h> 38 #include <linux/pci.h> 39 #include <linux/netdevice.h> 40 #include <linux/vmalloc.h> 41 #include <linux/string.h> 42 #include <linux/in.h> 43 #include <linux/ip.h> 44 #include <linux/tcp.h> 45 #include <linux/ipv6.h> 46 #include <linux/slab.h> 47 #include <net/checksum.h> 48 #include <net/ip6_checksum.h> 49 #include <linux/ethtool.h> 50 #include <linux/if.h> 51 #include <linux/if_vlan.h> 52 #include <linux/prefetch.h> 53 54 #include "ixgbevf.h" 55 56 const char ixgbevf_driver_name[] = "ixgbevf"; 57 static const char ixgbevf_driver_string[] = 58 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 59 60 #define DRV_VERSION "2.2.0-k" 61 const char ixgbevf_driver_version[] = DRV_VERSION; 62 static char ixgbevf_copyright[] = 63 "Copyright (c) 2009 - 2012 Intel Corporation."; 64 65 static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 66 [board_82599_vf] = &ixgbevf_82599_vf_info, 67 [board_X540_vf] = &ixgbevf_X540_vf_info, 68 }; 69 70 /* ixgbevf_pci_tbl - PCI Device ID Table 71 * 72 * Wildcard entries (PCI_ANY_ID) should come last 73 * Last entry must be all 0s 74 * 75 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 76 * Class, Class Mask, private data (not used) } 77 */ 78 static struct pci_device_id ixgbevf_pci_tbl[] = { 79 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), 80 board_82599_vf}, 81 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), 82 board_X540_vf}, 83 84 /* required last entry */ 85 {0, } 86 }; 87 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 88 89 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 90 MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver"); 91 MODULE_LICENSE("GPL"); 92 MODULE_VERSION(DRV_VERSION); 93 94 #define DEFAULT_DEBUG_LEVEL_SHIFT 3 95 96 /* forward decls */ 97 static void ixgbevf_set_itr_msix(struct ixgbevf_q_vector *q_vector); 98 static void ixgbevf_write_eitr(struct ixgbevf_adapter *adapter, int v_idx, 99 u32 itr_reg); 100 101 static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw, 102 struct ixgbevf_ring *rx_ring, 103 u32 val) 104 { 105 /* 106 * Force memory writes to complete before letting h/w 107 * know there are new descriptors to fetch. (Only 108 * applicable for weak-ordered memory model archs, 109 * such as IA-64). 110 */ 111 wmb(); 112 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val); 113 } 114 115 /* 116 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 117 * @adapter: pointer to adapter struct 118 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 119 * @queue: queue to map the corresponding interrupt to 120 * @msix_vector: the vector to map to the corresponding queue 121 * 122 */ 123 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 124 u8 queue, u8 msix_vector) 125 { 126 u32 ivar, index; 127 struct ixgbe_hw *hw = &adapter->hw; 128 if (direction == -1) { 129 /* other causes */ 130 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 131 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 132 ivar &= ~0xFF; 133 ivar |= msix_vector; 134 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 135 } else { 136 /* tx or rx causes */ 137 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 138 index = ((16 * (queue & 1)) + (8 * direction)); 139 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 140 ivar &= ~(0xFF << index); 141 ivar |= (msix_vector << index); 142 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 143 } 144 } 145 146 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_adapter *adapter, 147 struct ixgbevf_tx_buffer 148 *tx_buffer_info) 149 { 150 if (tx_buffer_info->dma) { 151 if (tx_buffer_info->mapped_as_page) 152 dma_unmap_page(&adapter->pdev->dev, 153 tx_buffer_info->dma, 154 tx_buffer_info->length, 155 DMA_TO_DEVICE); 156 else 157 dma_unmap_single(&adapter->pdev->dev, 158 tx_buffer_info->dma, 159 tx_buffer_info->length, 160 DMA_TO_DEVICE); 161 tx_buffer_info->dma = 0; 162 } 163 if (tx_buffer_info->skb) { 164 dev_kfree_skb_any(tx_buffer_info->skb); 165 tx_buffer_info->skb = NULL; 166 } 167 tx_buffer_info->time_stamp = 0; 168 /* tx_buffer_info must be completely set up in the transmit path */ 169 } 170 171 #define IXGBE_MAX_TXD_PWR 14 172 #define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR) 173 174 /* Tx Descriptors needed, worst case */ 175 #define TXD_USE_COUNT(S) (((S) >> IXGBE_MAX_TXD_PWR) + \ 176 (((S) & (IXGBE_MAX_DATA_PER_TXD - 1)) ? 1 : 0)) 177 #ifdef MAX_SKB_FRAGS 178 #define DESC_NEEDED (TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) /* skb->data */ + \ 179 MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1) /* for context */ 180 #else 181 #define DESC_NEEDED TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) 182 #endif 183 184 static void ixgbevf_tx_timeout(struct net_device *netdev); 185 186 /** 187 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 188 * @adapter: board private structure 189 * @tx_ring: tx ring to clean 190 **/ 191 static bool ixgbevf_clean_tx_irq(struct ixgbevf_adapter *adapter, 192 struct ixgbevf_ring *tx_ring) 193 { 194 struct net_device *netdev = adapter->netdev; 195 struct ixgbe_hw *hw = &adapter->hw; 196 union ixgbe_adv_tx_desc *tx_desc, *eop_desc; 197 struct ixgbevf_tx_buffer *tx_buffer_info; 198 unsigned int i, eop, count = 0; 199 unsigned int total_bytes = 0, total_packets = 0; 200 201 i = tx_ring->next_to_clean; 202 eop = tx_ring->tx_buffer_info[i].next_to_watch; 203 eop_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop); 204 205 while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) && 206 (count < tx_ring->work_limit)) { 207 bool cleaned = false; 208 rmb(); /* read buffer_info after eop_desc */ 209 /* eop could change between read and DD-check */ 210 if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch)) 211 goto cont_loop; 212 for ( ; !cleaned; count++) { 213 struct sk_buff *skb; 214 tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i); 215 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 216 cleaned = (i == eop); 217 skb = tx_buffer_info->skb; 218 219 if (cleaned && skb) { 220 unsigned int segs, bytecount; 221 222 /* gso_segs is currently only valid for tcp */ 223 segs = skb_shinfo(skb)->gso_segs ?: 1; 224 /* multiply data chunks by size of headers */ 225 bytecount = ((segs - 1) * skb_headlen(skb)) + 226 skb->len; 227 total_packets += segs; 228 total_bytes += bytecount; 229 } 230 231 ixgbevf_unmap_and_free_tx_resource(adapter, 232 tx_buffer_info); 233 234 tx_desc->wb.status = 0; 235 236 i++; 237 if (i == tx_ring->count) 238 i = 0; 239 } 240 241 cont_loop: 242 eop = tx_ring->tx_buffer_info[i].next_to_watch; 243 eop_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop); 244 } 245 246 tx_ring->next_to_clean = i; 247 248 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 249 if (unlikely(count && netif_carrier_ok(netdev) && 250 (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 251 /* Make sure that anybody stopping the queue after this 252 * sees the new next_to_clean. 253 */ 254 smp_mb(); 255 #ifdef HAVE_TX_MQ 256 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) && 257 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 258 netif_wake_subqueue(netdev, tx_ring->queue_index); 259 ++adapter->restart_queue; 260 } 261 #else 262 if (netif_queue_stopped(netdev) && 263 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 264 netif_wake_queue(netdev); 265 ++adapter->restart_queue; 266 } 267 #endif 268 } 269 270 /* re-arm the interrupt */ 271 if ((count >= tx_ring->work_limit) && 272 (!test_bit(__IXGBEVF_DOWN, &adapter->state))) { 273 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, tx_ring->v_idx); 274 } 275 276 u64_stats_update_begin(&tx_ring->syncp); 277 tx_ring->total_bytes += total_bytes; 278 tx_ring->total_packets += total_packets; 279 u64_stats_update_end(&tx_ring->syncp); 280 281 return count < tx_ring->work_limit; 282 } 283 284 /** 285 * ixgbevf_receive_skb - Send a completed packet up the stack 286 * @q_vector: structure containing interrupt and ring information 287 * @skb: packet to send up 288 * @status: hardware indication of status of receive 289 * @rx_ring: rx descriptor ring (for a specific queue) to setup 290 * @rx_desc: rx descriptor 291 **/ 292 static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector, 293 struct sk_buff *skb, u8 status, 294 struct ixgbevf_ring *ring, 295 union ixgbe_adv_rx_desc *rx_desc) 296 { 297 struct ixgbevf_adapter *adapter = q_vector->adapter; 298 bool is_vlan = (status & IXGBE_RXD_STAT_VP); 299 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan); 300 301 if (is_vlan && test_bit(tag, adapter->active_vlans)) 302 __vlan_hwaccel_put_tag(skb, tag); 303 304 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL)) 305 napi_gro_receive(&q_vector->napi, skb); 306 else 307 netif_rx(skb); 308 } 309 310 /** 311 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 312 * @adapter: address of board private structure 313 * @status_err: hardware indication of status of receive 314 * @skb: skb currently being received and modified 315 **/ 316 static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter, 317 u32 status_err, struct sk_buff *skb) 318 { 319 skb_checksum_none_assert(skb); 320 321 /* Rx csum disabled */ 322 if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED)) 323 return; 324 325 /* if IP and error */ 326 if ((status_err & IXGBE_RXD_STAT_IPCS) && 327 (status_err & IXGBE_RXDADV_ERR_IPE)) { 328 adapter->hw_csum_rx_error++; 329 return; 330 } 331 332 if (!(status_err & IXGBE_RXD_STAT_L4CS)) 333 return; 334 335 if (status_err & IXGBE_RXDADV_ERR_TCPE) { 336 adapter->hw_csum_rx_error++; 337 return; 338 } 339 340 /* It must be a TCP or UDP packet with a valid checksum */ 341 skb->ip_summed = CHECKSUM_UNNECESSARY; 342 adapter->hw_csum_rx_good++; 343 } 344 345 /** 346 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 347 * @adapter: address of board private structure 348 **/ 349 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter, 350 struct ixgbevf_ring *rx_ring, 351 int cleaned_count) 352 { 353 struct pci_dev *pdev = adapter->pdev; 354 union ixgbe_adv_rx_desc *rx_desc; 355 struct ixgbevf_rx_buffer *bi; 356 struct sk_buff *skb; 357 unsigned int i; 358 unsigned int bufsz = rx_ring->rx_buf_len + NET_IP_ALIGN; 359 360 i = rx_ring->next_to_use; 361 bi = &rx_ring->rx_buffer_info[i]; 362 363 while (cleaned_count--) { 364 rx_desc = IXGBE_RX_DESC_ADV(*rx_ring, i); 365 366 if (!bi->page_dma && 367 (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)) { 368 if (!bi->page) { 369 bi->page = alloc_page(GFP_ATOMIC | __GFP_COLD); 370 if (!bi->page) { 371 adapter->alloc_rx_page_failed++; 372 goto no_buffers; 373 } 374 bi->page_offset = 0; 375 } else { 376 /* use a half page if we're re-using */ 377 bi->page_offset ^= (PAGE_SIZE / 2); 378 } 379 380 bi->page_dma = dma_map_page(&pdev->dev, bi->page, 381 bi->page_offset, 382 (PAGE_SIZE / 2), 383 DMA_FROM_DEVICE); 384 } 385 386 skb = bi->skb; 387 if (!skb) { 388 skb = netdev_alloc_skb(adapter->netdev, 389 bufsz); 390 391 if (!skb) { 392 adapter->alloc_rx_buff_failed++; 393 goto no_buffers; 394 } 395 396 /* 397 * Make buffer alignment 2 beyond a 16 byte boundary 398 * this will result in a 16 byte aligned IP header after 399 * the 14 byte MAC header is removed 400 */ 401 skb_reserve(skb, NET_IP_ALIGN); 402 403 bi->skb = skb; 404 } 405 if (!bi->dma) { 406 bi->dma = dma_map_single(&pdev->dev, skb->data, 407 rx_ring->rx_buf_len, 408 DMA_FROM_DEVICE); 409 } 410 /* Refresh the desc even if buffer_addrs didn't change because 411 * each write-back erases this info. */ 412 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 413 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma); 414 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma); 415 } else { 416 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); 417 } 418 419 i++; 420 if (i == rx_ring->count) 421 i = 0; 422 bi = &rx_ring->rx_buffer_info[i]; 423 } 424 425 no_buffers: 426 if (rx_ring->next_to_use != i) { 427 rx_ring->next_to_use = i; 428 if (i-- == 0) 429 i = (rx_ring->count - 1); 430 431 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i); 432 } 433 } 434 435 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 436 u64 qmask) 437 { 438 u32 mask; 439 struct ixgbe_hw *hw = &adapter->hw; 440 441 mask = (qmask & 0xFFFFFFFF); 442 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 443 } 444 445 static inline u16 ixgbevf_get_hdr_info(union ixgbe_adv_rx_desc *rx_desc) 446 { 447 return rx_desc->wb.lower.lo_dword.hs_rss.hdr_info; 448 } 449 450 static inline u16 ixgbevf_get_pkt_info(union ixgbe_adv_rx_desc *rx_desc) 451 { 452 return rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; 453 } 454 455 static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 456 struct ixgbevf_ring *rx_ring, 457 int *work_done, int work_to_do) 458 { 459 struct ixgbevf_adapter *adapter = q_vector->adapter; 460 struct pci_dev *pdev = adapter->pdev; 461 union ixgbe_adv_rx_desc *rx_desc, *next_rxd; 462 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer; 463 struct sk_buff *skb; 464 unsigned int i; 465 u32 len, staterr; 466 u16 hdr_info; 467 bool cleaned = false; 468 int cleaned_count = 0; 469 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 470 471 i = rx_ring->next_to_clean; 472 rx_desc = IXGBE_RX_DESC_ADV(*rx_ring, i); 473 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 474 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 475 476 while (staterr & IXGBE_RXD_STAT_DD) { 477 u32 upper_len = 0; 478 if (*work_done >= work_to_do) 479 break; 480 (*work_done)++; 481 482 rmb(); /* read descriptor and rx_buffer_info after status DD */ 483 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 484 hdr_info = le16_to_cpu(ixgbevf_get_hdr_info(rx_desc)); 485 len = (hdr_info & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 486 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 487 if (hdr_info & IXGBE_RXDADV_SPH) 488 adapter->rx_hdr_split++; 489 if (len > IXGBEVF_RX_HDR_SIZE) 490 len = IXGBEVF_RX_HDR_SIZE; 491 upper_len = le16_to_cpu(rx_desc->wb.upper.length); 492 } else { 493 len = le16_to_cpu(rx_desc->wb.upper.length); 494 } 495 cleaned = true; 496 skb = rx_buffer_info->skb; 497 prefetch(skb->data - NET_IP_ALIGN); 498 rx_buffer_info->skb = NULL; 499 500 if (rx_buffer_info->dma) { 501 dma_unmap_single(&pdev->dev, rx_buffer_info->dma, 502 rx_ring->rx_buf_len, 503 DMA_FROM_DEVICE); 504 rx_buffer_info->dma = 0; 505 skb_put(skb, len); 506 } 507 508 if (upper_len) { 509 dma_unmap_page(&pdev->dev, rx_buffer_info->page_dma, 510 PAGE_SIZE / 2, DMA_FROM_DEVICE); 511 rx_buffer_info->page_dma = 0; 512 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 513 rx_buffer_info->page, 514 rx_buffer_info->page_offset, 515 upper_len); 516 517 if ((rx_ring->rx_buf_len > (PAGE_SIZE / 2)) || 518 (page_count(rx_buffer_info->page) != 1)) 519 rx_buffer_info->page = NULL; 520 else 521 get_page(rx_buffer_info->page); 522 523 skb->len += upper_len; 524 skb->data_len += upper_len; 525 skb->truesize += upper_len; 526 } 527 528 i++; 529 if (i == rx_ring->count) 530 i = 0; 531 532 next_rxd = IXGBE_RX_DESC_ADV(*rx_ring, i); 533 prefetch(next_rxd); 534 cleaned_count++; 535 536 next_buffer = &rx_ring->rx_buffer_info[i]; 537 538 if (!(staterr & IXGBE_RXD_STAT_EOP)) { 539 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 540 rx_buffer_info->skb = next_buffer->skb; 541 rx_buffer_info->dma = next_buffer->dma; 542 next_buffer->skb = skb; 543 next_buffer->dma = 0; 544 } else { 545 skb->next = next_buffer->skb; 546 skb->next->prev = skb; 547 } 548 adapter->non_eop_descs++; 549 goto next_desc; 550 } 551 552 /* ERR_MASK will only have valid bits if EOP set */ 553 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) { 554 dev_kfree_skb_irq(skb); 555 goto next_desc; 556 } 557 558 ixgbevf_rx_checksum(adapter, staterr, skb); 559 560 /* probably a little skewed due to removing CRC */ 561 total_rx_bytes += skb->len; 562 total_rx_packets++; 563 564 /* 565 * Work around issue of some types of VM to VM loop back 566 * packets not getting split correctly 567 */ 568 if (staterr & IXGBE_RXD_STAT_LB) { 569 u32 header_fixup_len = skb_headlen(skb); 570 if (header_fixup_len < 14) 571 skb_push(skb, header_fixup_len); 572 } 573 skb->protocol = eth_type_trans(skb, adapter->netdev); 574 575 ixgbevf_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc); 576 577 next_desc: 578 rx_desc->wb.upper.status_error = 0; 579 580 /* return some buffers to hardware, one at a time is too slow */ 581 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 582 ixgbevf_alloc_rx_buffers(adapter, rx_ring, 583 cleaned_count); 584 cleaned_count = 0; 585 } 586 587 /* use prefetched values */ 588 rx_desc = next_rxd; 589 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 590 591 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 592 } 593 594 rx_ring->next_to_clean = i; 595 cleaned_count = IXGBE_DESC_UNUSED(rx_ring); 596 597 if (cleaned_count) 598 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count); 599 600 u64_stats_update_begin(&rx_ring->syncp); 601 rx_ring->total_packets += total_rx_packets; 602 rx_ring->total_bytes += total_rx_bytes; 603 u64_stats_update_end(&rx_ring->syncp); 604 605 return cleaned; 606 } 607 608 /** 609 * ixgbevf_clean_rxonly - msix (aka one shot) rx clean routine 610 * @napi: napi struct with our devices info in it 611 * @budget: amount of work driver is allowed to do this pass, in packets 612 * 613 * This function is optimized for cleaning one queue only on a single 614 * q_vector!!! 615 **/ 616 static int ixgbevf_clean_rxonly(struct napi_struct *napi, int budget) 617 { 618 struct ixgbevf_q_vector *q_vector = 619 container_of(napi, struct ixgbevf_q_vector, napi); 620 struct ixgbevf_adapter *adapter = q_vector->adapter; 621 struct ixgbevf_ring *rx_ring = NULL; 622 int work_done = 0; 623 long r_idx; 624 625 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 626 rx_ring = &(adapter->rx_ring[r_idx]); 627 628 ixgbevf_clean_rx_irq(q_vector, rx_ring, &work_done, budget); 629 630 /* If all Rx work done, exit the polling mode */ 631 if (work_done < budget) { 632 napi_complete(napi); 633 if (adapter->itr_setting & 1) 634 ixgbevf_set_itr_msix(q_vector); 635 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 636 ixgbevf_irq_enable_queues(adapter, rx_ring->v_idx); 637 } 638 639 return work_done; 640 } 641 642 /** 643 * ixgbevf_clean_rxonly_many - msix (aka one shot) rx clean routine 644 * @napi: napi struct with our devices info in it 645 * @budget: amount of work driver is allowed to do this pass, in packets 646 * 647 * This function will clean more than one rx queue associated with a 648 * q_vector. 649 **/ 650 static int ixgbevf_clean_rxonly_many(struct napi_struct *napi, int budget) 651 { 652 struct ixgbevf_q_vector *q_vector = 653 container_of(napi, struct ixgbevf_q_vector, napi); 654 struct ixgbevf_adapter *adapter = q_vector->adapter; 655 struct ixgbevf_ring *rx_ring = NULL; 656 int work_done = 0, i; 657 long r_idx; 658 u64 enable_mask = 0; 659 660 /* attempt to distribute budget to each queue fairly, but don't allow 661 * the budget to go below 1 because we'll exit polling */ 662 budget /= (q_vector->rxr_count ?: 1); 663 budget = max(budget, 1); 664 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 665 for (i = 0; i < q_vector->rxr_count; i++) { 666 rx_ring = &(adapter->rx_ring[r_idx]); 667 ixgbevf_clean_rx_irq(q_vector, rx_ring, &work_done, budget); 668 enable_mask |= rx_ring->v_idx; 669 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, 670 r_idx + 1); 671 } 672 673 #ifndef HAVE_NETDEV_NAPI_LIST 674 if (!netif_running(adapter->netdev)) 675 work_done = 0; 676 677 #endif 678 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 679 rx_ring = &(adapter->rx_ring[r_idx]); 680 681 /* If all Rx work done, exit the polling mode */ 682 if (work_done < budget) { 683 napi_complete(napi); 684 if (adapter->itr_setting & 1) 685 ixgbevf_set_itr_msix(q_vector); 686 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 687 ixgbevf_irq_enable_queues(adapter, enable_mask); 688 } 689 690 return work_done; 691 } 692 693 694 /** 695 * ixgbevf_configure_msix - Configure MSI-X hardware 696 * @adapter: board private structure 697 * 698 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 699 * interrupts. 700 **/ 701 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 702 { 703 struct ixgbevf_q_vector *q_vector; 704 struct ixgbe_hw *hw = &adapter->hw; 705 int i, j, q_vectors, v_idx, r_idx; 706 u32 mask; 707 708 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 709 710 /* 711 * Populate the IVAR table and set the ITR values to the 712 * corresponding register. 713 */ 714 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 715 q_vector = adapter->q_vector[v_idx]; 716 /* XXX for_each_set_bit(...) */ 717 r_idx = find_first_bit(q_vector->rxr_idx, 718 adapter->num_rx_queues); 719 720 for (i = 0; i < q_vector->rxr_count; i++) { 721 j = adapter->rx_ring[r_idx].reg_idx; 722 ixgbevf_set_ivar(adapter, 0, j, v_idx); 723 r_idx = find_next_bit(q_vector->rxr_idx, 724 adapter->num_rx_queues, 725 r_idx + 1); 726 } 727 r_idx = find_first_bit(q_vector->txr_idx, 728 adapter->num_tx_queues); 729 730 for (i = 0; i < q_vector->txr_count; i++) { 731 j = adapter->tx_ring[r_idx].reg_idx; 732 ixgbevf_set_ivar(adapter, 1, j, v_idx); 733 r_idx = find_next_bit(q_vector->txr_idx, 734 adapter->num_tx_queues, 735 r_idx + 1); 736 } 737 738 /* if this is a tx only vector halve the interrupt rate */ 739 if (q_vector->txr_count && !q_vector->rxr_count) 740 q_vector->eitr = (adapter->eitr_param >> 1); 741 else if (q_vector->rxr_count) 742 /* rx only */ 743 q_vector->eitr = adapter->eitr_param; 744 745 ixgbevf_write_eitr(adapter, v_idx, q_vector->eitr); 746 } 747 748 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 749 750 /* set up to autoclear timer, and the vectors */ 751 mask = IXGBE_EIMS_ENABLE_MASK; 752 mask &= ~IXGBE_EIMS_OTHER; 753 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 754 } 755 756 enum latency_range { 757 lowest_latency = 0, 758 low_latency = 1, 759 bulk_latency = 2, 760 latency_invalid = 255 761 }; 762 763 /** 764 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 765 * @adapter: pointer to adapter 766 * @eitr: eitr setting (ints per sec) to give last timeslice 767 * @itr_setting: current throttle rate in ints/second 768 * @packets: the number of packets during this measurement interval 769 * @bytes: the number of bytes during this measurement interval 770 * 771 * Stores a new ITR value based on packets and byte 772 * counts during the last interrupt. The advantage of per interrupt 773 * computation is faster updates and more accurate ITR for the current 774 * traffic pattern. Constants in this function were computed 775 * based on theoretical maximum wire speed and thresholds were set based 776 * on testing data as well as attempting to minimize response time 777 * while increasing bulk throughput. 778 **/ 779 static u8 ixgbevf_update_itr(struct ixgbevf_adapter *adapter, 780 u32 eitr, u8 itr_setting, 781 int packets, int bytes) 782 { 783 unsigned int retval = itr_setting; 784 u32 timepassed_us; 785 u64 bytes_perint; 786 787 if (packets == 0) 788 goto update_itr_done; 789 790 791 /* simple throttlerate management 792 * 0-20MB/s lowest (100000 ints/s) 793 * 20-100MB/s low (20000 ints/s) 794 * 100-1249MB/s bulk (8000 ints/s) 795 */ 796 /* what was last interrupt timeslice? */ 797 timepassed_us = 1000000/eitr; 798 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 799 800 switch (itr_setting) { 801 case lowest_latency: 802 if (bytes_perint > adapter->eitr_low) 803 retval = low_latency; 804 break; 805 case low_latency: 806 if (bytes_perint > adapter->eitr_high) 807 retval = bulk_latency; 808 else if (bytes_perint <= adapter->eitr_low) 809 retval = lowest_latency; 810 break; 811 case bulk_latency: 812 if (bytes_perint <= adapter->eitr_high) 813 retval = low_latency; 814 break; 815 } 816 817 update_itr_done: 818 return retval; 819 } 820 821 /** 822 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 823 * @adapter: pointer to adapter struct 824 * @v_idx: vector index into q_vector array 825 * @itr_reg: new value to be written in *register* format, not ints/s 826 * 827 * This function is made to be called by ethtool and by the driver 828 * when it needs to update VTEITR registers at runtime. Hardware 829 * specific quirks/differences are taken care of here. 830 */ 831 static void ixgbevf_write_eitr(struct ixgbevf_adapter *adapter, int v_idx, 832 u32 itr_reg) 833 { 834 struct ixgbe_hw *hw = &adapter->hw; 835 836 itr_reg = EITR_INTS_PER_SEC_TO_REG(itr_reg); 837 838 /* 839 * set the WDIS bit to not clear the timer bits and cause an 840 * immediate assertion of the interrupt 841 */ 842 itr_reg |= IXGBE_EITR_CNT_WDIS; 843 844 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 845 } 846 847 static void ixgbevf_set_itr_msix(struct ixgbevf_q_vector *q_vector) 848 { 849 struct ixgbevf_adapter *adapter = q_vector->adapter; 850 u32 new_itr; 851 u8 current_itr, ret_itr; 852 int i, r_idx, v_idx = q_vector->v_idx; 853 struct ixgbevf_ring *rx_ring, *tx_ring; 854 855 r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); 856 for (i = 0; i < q_vector->txr_count; i++) { 857 tx_ring = &(adapter->tx_ring[r_idx]); 858 ret_itr = ixgbevf_update_itr(adapter, q_vector->eitr, 859 q_vector->tx_itr, 860 tx_ring->total_packets, 861 tx_ring->total_bytes); 862 /* if the result for this queue would decrease interrupt 863 * rate for this vector then use that result */ 864 q_vector->tx_itr = ((q_vector->tx_itr > ret_itr) ? 865 q_vector->tx_itr - 1 : ret_itr); 866 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, 867 r_idx + 1); 868 } 869 870 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 871 for (i = 0; i < q_vector->rxr_count; i++) { 872 rx_ring = &(adapter->rx_ring[r_idx]); 873 ret_itr = ixgbevf_update_itr(adapter, q_vector->eitr, 874 q_vector->rx_itr, 875 rx_ring->total_packets, 876 rx_ring->total_bytes); 877 /* if the result for this queue would decrease interrupt 878 * rate for this vector then use that result */ 879 q_vector->rx_itr = ((q_vector->rx_itr > ret_itr) ? 880 q_vector->rx_itr - 1 : ret_itr); 881 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, 882 r_idx + 1); 883 } 884 885 current_itr = max(q_vector->rx_itr, q_vector->tx_itr); 886 887 switch (current_itr) { 888 /* counts and packets in update_itr are dependent on these numbers */ 889 case lowest_latency: 890 new_itr = 100000; 891 break; 892 case low_latency: 893 new_itr = 20000; /* aka hwitr = ~200 */ 894 break; 895 case bulk_latency: 896 default: 897 new_itr = 8000; 898 break; 899 } 900 901 if (new_itr != q_vector->eitr) { 902 u32 itr_reg; 903 904 /* save the algorithm value here, not the smoothed one */ 905 q_vector->eitr = new_itr; 906 /* do an exponential smoothing */ 907 new_itr = ((q_vector->eitr * 90)/100) + ((new_itr * 10)/100); 908 itr_reg = EITR_INTS_PER_SEC_TO_REG(new_itr); 909 ixgbevf_write_eitr(adapter, v_idx, itr_reg); 910 } 911 } 912 913 static irqreturn_t ixgbevf_msix_mbx(int irq, void *data) 914 { 915 struct net_device *netdev = data; 916 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 917 struct ixgbe_hw *hw = &adapter->hw; 918 u32 eicr; 919 u32 msg; 920 bool got_ack = false; 921 922 eicr = IXGBE_READ_REG(hw, IXGBE_VTEICS); 923 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, eicr); 924 925 if (!hw->mbx.ops.check_for_ack(hw)) 926 got_ack = true; 927 928 if (!hw->mbx.ops.check_for_msg(hw)) { 929 hw->mbx.ops.read(hw, &msg, 1); 930 931 if ((msg & IXGBE_MBVFICR_VFREQ_MASK) == IXGBE_PF_CONTROL_MSG) 932 mod_timer(&adapter->watchdog_timer, 933 round_jiffies(jiffies + 1)); 934 935 if (msg & IXGBE_VT_MSGTYPE_NACK) 936 pr_warn("Last Request of type %2.2x to PF Nacked\n", 937 msg & 0xFF); 938 /* 939 * Restore the PFSTS bit in case someone is polling for a 940 * return message from the PF 941 */ 942 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFSTS; 943 } 944 945 /* 946 * checking for the ack clears the PFACK bit. Place 947 * it back in the v2p_mailbox cache so that anyone 948 * polling for an ack will not miss it 949 */ 950 if (got_ack) 951 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFACK; 952 953 return IRQ_HANDLED; 954 } 955 956 static irqreturn_t ixgbevf_msix_clean_tx(int irq, void *data) 957 { 958 struct ixgbevf_q_vector *q_vector = data; 959 struct ixgbevf_adapter *adapter = q_vector->adapter; 960 struct ixgbevf_ring *tx_ring; 961 int i, r_idx; 962 963 if (!q_vector->txr_count) 964 return IRQ_HANDLED; 965 966 r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); 967 for (i = 0; i < q_vector->txr_count; i++) { 968 tx_ring = &(adapter->tx_ring[r_idx]); 969 tx_ring->total_bytes = 0; 970 tx_ring->total_packets = 0; 971 ixgbevf_clean_tx_irq(adapter, tx_ring); 972 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, 973 r_idx + 1); 974 } 975 976 if (adapter->itr_setting & 1) 977 ixgbevf_set_itr_msix(q_vector); 978 979 return IRQ_HANDLED; 980 } 981 982 /** 983 * ixgbevf_msix_clean_rx - single unshared vector rx clean (all queues) 984 * @irq: unused 985 * @data: pointer to our q_vector struct for this interrupt vector 986 **/ 987 static irqreturn_t ixgbevf_msix_clean_rx(int irq, void *data) 988 { 989 struct ixgbevf_q_vector *q_vector = data; 990 struct ixgbevf_adapter *adapter = q_vector->adapter; 991 struct ixgbe_hw *hw = &adapter->hw; 992 struct ixgbevf_ring *rx_ring; 993 int r_idx; 994 int i; 995 996 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 997 for (i = 0; i < q_vector->rxr_count; i++) { 998 rx_ring = &(adapter->rx_ring[r_idx]); 999 rx_ring->total_bytes = 0; 1000 rx_ring->total_packets = 0; 1001 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, 1002 r_idx + 1); 1003 } 1004 1005 if (!q_vector->rxr_count) 1006 return IRQ_HANDLED; 1007 1008 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 1009 rx_ring = &(adapter->rx_ring[r_idx]); 1010 /* disable interrupts on this vector only */ 1011 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, rx_ring->v_idx); 1012 napi_schedule(&q_vector->napi); 1013 1014 1015 return IRQ_HANDLED; 1016 } 1017 1018 static irqreturn_t ixgbevf_msix_clean_many(int irq, void *data) 1019 { 1020 ixgbevf_msix_clean_rx(irq, data); 1021 ixgbevf_msix_clean_tx(irq, data); 1022 1023 return IRQ_HANDLED; 1024 } 1025 1026 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, 1027 int r_idx) 1028 { 1029 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1030 1031 set_bit(r_idx, q_vector->rxr_idx); 1032 q_vector->rxr_count++; 1033 a->rx_ring[r_idx].v_idx = 1 << v_idx; 1034 } 1035 1036 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, 1037 int t_idx) 1038 { 1039 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1040 1041 set_bit(t_idx, q_vector->txr_idx); 1042 q_vector->txr_count++; 1043 a->tx_ring[t_idx].v_idx = 1 << v_idx; 1044 } 1045 1046 /** 1047 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors 1048 * @adapter: board private structure to initialize 1049 * 1050 * This function maps descriptor rings to the queue-specific vectors 1051 * we were allotted through the MSI-X enabling code. Ideally, we'd have 1052 * one vector per ring/queue, but on a constrained vector budget, we 1053 * group the rings as "efficiently" as possible. You would add new 1054 * mapping configurations in here. 1055 **/ 1056 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter) 1057 { 1058 int q_vectors; 1059 int v_start = 0; 1060 int rxr_idx = 0, txr_idx = 0; 1061 int rxr_remaining = adapter->num_rx_queues; 1062 int txr_remaining = adapter->num_tx_queues; 1063 int i, j; 1064 int rqpv, tqpv; 1065 int err = 0; 1066 1067 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1068 1069 /* 1070 * The ideal configuration... 1071 * We have enough vectors to map one per queue. 1072 */ 1073 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) { 1074 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) 1075 map_vector_to_rxq(adapter, v_start, rxr_idx); 1076 1077 for (; txr_idx < txr_remaining; v_start++, txr_idx++) 1078 map_vector_to_txq(adapter, v_start, txr_idx); 1079 goto out; 1080 } 1081 1082 /* 1083 * If we don't have enough vectors for a 1-to-1 1084 * mapping, we'll have to group them so there are 1085 * multiple queues per vector. 1086 */ 1087 /* Re-adjusting *qpv takes care of the remainder. */ 1088 for (i = v_start; i < q_vectors; i++) { 1089 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i); 1090 for (j = 0; j < rqpv; j++) { 1091 map_vector_to_rxq(adapter, i, rxr_idx); 1092 rxr_idx++; 1093 rxr_remaining--; 1094 } 1095 } 1096 for (i = v_start; i < q_vectors; i++) { 1097 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i); 1098 for (j = 0; j < tqpv; j++) { 1099 map_vector_to_txq(adapter, i, txr_idx); 1100 txr_idx++; 1101 txr_remaining--; 1102 } 1103 } 1104 1105 out: 1106 return err; 1107 } 1108 1109 /** 1110 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 1111 * @adapter: board private structure 1112 * 1113 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 1114 * interrupts from the kernel. 1115 **/ 1116 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 1117 { 1118 struct net_device *netdev = adapter->netdev; 1119 irqreturn_t (*handler)(int, void *); 1120 int i, vector, q_vectors, err; 1121 int ri = 0, ti = 0; 1122 1123 /* Decrement for Other and TCP Timer vectors */ 1124 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1125 1126 #define SET_HANDLER(_v) (((_v)->rxr_count && (_v)->txr_count) \ 1127 ? &ixgbevf_msix_clean_many : \ 1128 (_v)->rxr_count ? &ixgbevf_msix_clean_rx : \ 1129 (_v)->txr_count ? &ixgbevf_msix_clean_tx : \ 1130 NULL) 1131 for (vector = 0; vector < q_vectors; vector++) { 1132 handler = SET_HANDLER(adapter->q_vector[vector]); 1133 1134 if (handler == &ixgbevf_msix_clean_rx) { 1135 sprintf(adapter->name[vector], "%s-%s-%d", 1136 netdev->name, "rx", ri++); 1137 } else if (handler == &ixgbevf_msix_clean_tx) { 1138 sprintf(adapter->name[vector], "%s-%s-%d", 1139 netdev->name, "tx", ti++); 1140 } else if (handler == &ixgbevf_msix_clean_many) { 1141 sprintf(adapter->name[vector], "%s-%s-%d", 1142 netdev->name, "TxRx", vector); 1143 } else { 1144 /* skip this unused q_vector */ 1145 continue; 1146 } 1147 err = request_irq(adapter->msix_entries[vector].vector, 1148 handler, 0, adapter->name[vector], 1149 adapter->q_vector[vector]); 1150 if (err) { 1151 hw_dbg(&adapter->hw, 1152 "request_irq failed for MSIX interrupt " 1153 "Error: %d\n", err); 1154 goto free_queue_irqs; 1155 } 1156 } 1157 1158 sprintf(adapter->name[vector], "%s:mbx", netdev->name); 1159 err = request_irq(adapter->msix_entries[vector].vector, 1160 &ixgbevf_msix_mbx, 0, adapter->name[vector], netdev); 1161 if (err) { 1162 hw_dbg(&adapter->hw, 1163 "request_irq for msix_mbx failed: %d\n", err); 1164 goto free_queue_irqs; 1165 } 1166 1167 return 0; 1168 1169 free_queue_irqs: 1170 for (i = vector - 1; i >= 0; i--) 1171 free_irq(adapter->msix_entries[--vector].vector, 1172 &(adapter->q_vector[i])); 1173 pci_disable_msix(adapter->pdev); 1174 kfree(adapter->msix_entries); 1175 adapter->msix_entries = NULL; 1176 return err; 1177 } 1178 1179 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter) 1180 { 1181 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1182 1183 for (i = 0; i < q_vectors; i++) { 1184 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i]; 1185 bitmap_zero(q_vector->rxr_idx, MAX_RX_QUEUES); 1186 bitmap_zero(q_vector->txr_idx, MAX_TX_QUEUES); 1187 q_vector->rxr_count = 0; 1188 q_vector->txr_count = 0; 1189 q_vector->eitr = adapter->eitr_param; 1190 } 1191 } 1192 1193 /** 1194 * ixgbevf_request_irq - initialize interrupts 1195 * @adapter: board private structure 1196 * 1197 * Attempts to configure interrupts using the best available 1198 * capabilities of the hardware and kernel. 1199 **/ 1200 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 1201 { 1202 int err = 0; 1203 1204 err = ixgbevf_request_msix_irqs(adapter); 1205 1206 if (err) 1207 hw_dbg(&adapter->hw, 1208 "request_irq failed, Error %d\n", err); 1209 1210 return err; 1211 } 1212 1213 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 1214 { 1215 struct net_device *netdev = adapter->netdev; 1216 int i, q_vectors; 1217 1218 q_vectors = adapter->num_msix_vectors; 1219 1220 i = q_vectors - 1; 1221 1222 free_irq(adapter->msix_entries[i].vector, netdev); 1223 i--; 1224 1225 for (; i >= 0; i--) { 1226 free_irq(adapter->msix_entries[i].vector, 1227 adapter->q_vector[i]); 1228 } 1229 1230 ixgbevf_reset_q_vectors(adapter); 1231 } 1232 1233 /** 1234 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1235 * @adapter: board private structure 1236 **/ 1237 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1238 { 1239 int i; 1240 struct ixgbe_hw *hw = &adapter->hw; 1241 1242 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1243 1244 IXGBE_WRITE_FLUSH(hw); 1245 1246 for (i = 0; i < adapter->num_msix_vectors; i++) 1247 synchronize_irq(adapter->msix_entries[i].vector); 1248 } 1249 1250 /** 1251 * ixgbevf_irq_enable - Enable default interrupt generation settings 1252 * @adapter: board private structure 1253 **/ 1254 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter, 1255 bool queues, bool flush) 1256 { 1257 struct ixgbe_hw *hw = &adapter->hw; 1258 u32 mask; 1259 u64 qmask; 1260 1261 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 1262 qmask = ~0; 1263 1264 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 1265 1266 if (queues) 1267 ixgbevf_irq_enable_queues(adapter, qmask); 1268 1269 if (flush) 1270 IXGBE_WRITE_FLUSH(hw); 1271 } 1272 1273 /** 1274 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1275 * @adapter: board private structure 1276 * 1277 * Configure the Tx unit of the MAC after a reset. 1278 **/ 1279 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1280 { 1281 u64 tdba; 1282 struct ixgbe_hw *hw = &adapter->hw; 1283 u32 i, j, tdlen, txctrl; 1284 1285 /* Setup the HW Tx Head and Tail descriptor pointers */ 1286 for (i = 0; i < adapter->num_tx_queues; i++) { 1287 struct ixgbevf_ring *ring = &adapter->tx_ring[i]; 1288 j = ring->reg_idx; 1289 tdba = ring->dma; 1290 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc); 1291 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1292 (tdba & DMA_BIT_MASK(32))); 1293 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1294 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen); 1295 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0); 1296 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0); 1297 adapter->tx_ring[i].head = IXGBE_VFTDH(j); 1298 adapter->tx_ring[i].tail = IXGBE_VFTDT(j); 1299 /* Disable Tx Head Writeback RO bit, since this hoses 1300 * bookkeeping if things aren't delivered in order. 1301 */ 1302 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1303 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 1304 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1305 } 1306 } 1307 1308 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1309 1310 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) 1311 { 1312 struct ixgbevf_ring *rx_ring; 1313 struct ixgbe_hw *hw = &adapter->hw; 1314 u32 srrctl; 1315 1316 rx_ring = &adapter->rx_ring[index]; 1317 1318 srrctl = IXGBE_SRRCTL_DROP_EN; 1319 1320 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 1321 u16 bufsz = IXGBEVF_RXBUFFER_2048; 1322 /* grow the amount we can receive on large page machines */ 1323 if (bufsz < (PAGE_SIZE / 2)) 1324 bufsz = (PAGE_SIZE / 2); 1325 /* cap the bufsz at our largest descriptor size */ 1326 bufsz = min((u16)IXGBEVF_MAX_RXBUFFER, bufsz); 1327 1328 srrctl |= bufsz >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1329 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 1330 srrctl |= ((IXGBEVF_RX_HDR_SIZE << 1331 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) & 1332 IXGBE_SRRCTL_BSIZEHDR_MASK); 1333 } else { 1334 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1335 1336 if (rx_ring->rx_buf_len == MAXIMUM_ETHERNET_VLAN_SIZE) 1337 srrctl |= IXGBEVF_RXBUFFER_2048 >> 1338 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1339 else 1340 srrctl |= rx_ring->rx_buf_len >> 1341 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1342 } 1343 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1344 } 1345 1346 /** 1347 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1348 * @adapter: board private structure 1349 * 1350 * Configure the Rx unit of the MAC after a reset. 1351 **/ 1352 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1353 { 1354 u64 rdba; 1355 struct ixgbe_hw *hw = &adapter->hw; 1356 struct net_device *netdev = adapter->netdev; 1357 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 1358 int i, j; 1359 u32 rdlen; 1360 int rx_buf_len; 1361 1362 /* Decide whether to use packet split mode or not */ 1363 if (netdev->mtu > ETH_DATA_LEN) { 1364 if (adapter->flags & IXGBE_FLAG_RX_PS_CAPABLE) 1365 adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED; 1366 else 1367 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED; 1368 } else { 1369 if (adapter->flags & IXGBE_FLAG_RX_1BUF_CAPABLE) 1370 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED; 1371 else 1372 adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED; 1373 } 1374 1375 /* Set the RX buffer length according to the mode */ 1376 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 1377 /* PSRTYPE must be initialized in 82599 */ 1378 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 1379 IXGBE_PSRTYPE_UDPHDR | 1380 IXGBE_PSRTYPE_IPV4HDR | 1381 IXGBE_PSRTYPE_IPV6HDR | 1382 IXGBE_PSRTYPE_L2HDR; 1383 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1384 rx_buf_len = IXGBEVF_RX_HDR_SIZE; 1385 } else { 1386 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0); 1387 if (netdev->mtu <= ETH_DATA_LEN) 1388 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE; 1389 else 1390 rx_buf_len = ALIGN(max_frame, 1024); 1391 } 1392 1393 rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc); 1394 /* Setup the HW Rx Head and Tail Descriptor Pointers and 1395 * the Base and Length of the Rx Descriptor Ring */ 1396 for (i = 0; i < adapter->num_rx_queues; i++) { 1397 rdba = adapter->rx_ring[i].dma; 1398 j = adapter->rx_ring[i].reg_idx; 1399 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1400 (rdba & DMA_BIT_MASK(32))); 1401 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1402 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen); 1403 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0); 1404 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0); 1405 adapter->rx_ring[i].head = IXGBE_VFRDH(j); 1406 adapter->rx_ring[i].tail = IXGBE_VFRDT(j); 1407 adapter->rx_ring[i].rx_buf_len = rx_buf_len; 1408 1409 ixgbevf_configure_srrctl(adapter, j); 1410 } 1411 } 1412 1413 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid) 1414 { 1415 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1416 struct ixgbe_hw *hw = &adapter->hw; 1417 1418 /* add VID to filter table */ 1419 if (hw->mac.ops.set_vfta) 1420 hw->mac.ops.set_vfta(hw, vid, 0, true); 1421 set_bit(vid, adapter->active_vlans); 1422 1423 return 0; 1424 } 1425 1426 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) 1427 { 1428 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1429 struct ixgbe_hw *hw = &adapter->hw; 1430 1431 /* remove VID from filter table */ 1432 if (hw->mac.ops.set_vfta) 1433 hw->mac.ops.set_vfta(hw, vid, 0, false); 1434 clear_bit(vid, adapter->active_vlans); 1435 1436 return 0; 1437 } 1438 1439 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 1440 { 1441 u16 vid; 1442 1443 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1444 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid); 1445 } 1446 1447 static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 1448 { 1449 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1450 struct ixgbe_hw *hw = &adapter->hw; 1451 int count = 0; 1452 1453 if ((netdev_uc_count(netdev)) > 10) { 1454 pr_err("Too many unicast filters - No Space\n"); 1455 return -ENOSPC; 1456 } 1457 1458 if (!netdev_uc_empty(netdev)) { 1459 struct netdev_hw_addr *ha; 1460 netdev_for_each_uc_addr(ha, netdev) { 1461 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 1462 udelay(200); 1463 } 1464 } else { 1465 /* 1466 * If the list is empty then send message to PF driver to 1467 * clear all macvlans on this VF. 1468 */ 1469 hw->mac.ops.set_uc_addr(hw, 0, NULL); 1470 } 1471 1472 return count; 1473 } 1474 1475 /** 1476 * ixgbevf_set_rx_mode - Multicast set 1477 * @netdev: network interface device structure 1478 * 1479 * The set_rx_method entry point is called whenever the multicast address 1480 * list or the network interface flags are updated. This routine is 1481 * responsible for configuring the hardware for proper multicast mode. 1482 **/ 1483 static void ixgbevf_set_rx_mode(struct net_device *netdev) 1484 { 1485 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1486 struct ixgbe_hw *hw = &adapter->hw; 1487 1488 /* reprogram multicast list */ 1489 if (hw->mac.ops.update_mc_addr_list) 1490 hw->mac.ops.update_mc_addr_list(hw, netdev); 1491 1492 ixgbevf_write_uc_addr_list(netdev); 1493 } 1494 1495 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 1496 { 1497 int q_idx; 1498 struct ixgbevf_q_vector *q_vector; 1499 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1500 1501 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1502 struct napi_struct *napi; 1503 q_vector = adapter->q_vector[q_idx]; 1504 if (!q_vector->rxr_count) 1505 continue; 1506 napi = &q_vector->napi; 1507 if (q_vector->rxr_count > 1) 1508 napi->poll = &ixgbevf_clean_rxonly_many; 1509 1510 napi_enable(napi); 1511 } 1512 } 1513 1514 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 1515 { 1516 int q_idx; 1517 struct ixgbevf_q_vector *q_vector; 1518 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1519 1520 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1521 q_vector = adapter->q_vector[q_idx]; 1522 if (!q_vector->rxr_count) 1523 continue; 1524 napi_disable(&q_vector->napi); 1525 } 1526 } 1527 1528 static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 1529 { 1530 struct net_device *netdev = adapter->netdev; 1531 int i; 1532 1533 ixgbevf_set_rx_mode(netdev); 1534 1535 ixgbevf_restore_vlan(adapter); 1536 1537 ixgbevf_configure_tx(adapter); 1538 ixgbevf_configure_rx(adapter); 1539 for (i = 0; i < adapter->num_rx_queues; i++) { 1540 struct ixgbevf_ring *ring = &adapter->rx_ring[i]; 1541 ixgbevf_alloc_rx_buffers(adapter, ring, ring->count); 1542 ring->next_to_use = ring->count - 1; 1543 writel(ring->next_to_use, adapter->hw.hw_addr + ring->tail); 1544 } 1545 } 1546 1547 #define IXGBE_MAX_RX_DESC_POLL 10 1548 static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1549 int rxr) 1550 { 1551 struct ixgbe_hw *hw = &adapter->hw; 1552 int j = adapter->rx_ring[rxr].reg_idx; 1553 int k; 1554 1555 for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) { 1556 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE) 1557 break; 1558 else 1559 msleep(1); 1560 } 1561 if (k >= IXGBE_MAX_RX_DESC_POLL) { 1562 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d " 1563 "not set within the polling period\n", rxr); 1564 } 1565 1566 ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr], 1567 (adapter->rx_ring[rxr].count - 1)); 1568 } 1569 1570 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 1571 { 1572 /* Only save pre-reset stats if there are some */ 1573 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 1574 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 1575 adapter->stats.base_vfgprc; 1576 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 1577 adapter->stats.base_vfgptc; 1578 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 1579 adapter->stats.base_vfgorc; 1580 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 1581 adapter->stats.base_vfgotc; 1582 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 1583 adapter->stats.base_vfmprc; 1584 } 1585 } 1586 1587 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 1588 { 1589 struct ixgbe_hw *hw = &adapter->hw; 1590 1591 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 1592 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 1593 adapter->stats.last_vfgorc |= 1594 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 1595 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 1596 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 1597 adapter->stats.last_vfgotc |= 1598 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 1599 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 1600 1601 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 1602 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 1603 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 1604 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 1605 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 1606 } 1607 1608 static int ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 1609 { 1610 struct net_device *netdev = adapter->netdev; 1611 struct ixgbe_hw *hw = &adapter->hw; 1612 int i, j = 0; 1613 int num_rx_rings = adapter->num_rx_queues; 1614 u32 txdctl, rxdctl; 1615 1616 for (i = 0; i < adapter->num_tx_queues; i++) { 1617 j = adapter->tx_ring[i].reg_idx; 1618 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1619 /* enable WTHRESH=8 descriptors, to encourage burst writeback */ 1620 txdctl |= (8 << 16); 1621 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1622 } 1623 1624 for (i = 0; i < adapter->num_tx_queues; i++) { 1625 j = adapter->tx_ring[i].reg_idx; 1626 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1627 txdctl |= IXGBE_TXDCTL_ENABLE; 1628 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1629 } 1630 1631 for (i = 0; i < num_rx_rings; i++) { 1632 j = adapter->rx_ring[i].reg_idx; 1633 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1634 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1635 if (hw->mac.type == ixgbe_mac_X540_vf) { 1636 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK; 1637 rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) | 1638 IXGBE_RXDCTL_RLPML_EN); 1639 } 1640 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1641 ixgbevf_rx_desc_queue_enable(adapter, i); 1642 } 1643 1644 ixgbevf_configure_msix(adapter); 1645 1646 if (hw->mac.ops.set_rar) { 1647 if (is_valid_ether_addr(hw->mac.addr)) 1648 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 1649 else 1650 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 1651 } 1652 1653 clear_bit(__IXGBEVF_DOWN, &adapter->state); 1654 ixgbevf_napi_enable_all(adapter); 1655 1656 /* enable transmits */ 1657 netif_tx_start_all_queues(netdev); 1658 1659 ixgbevf_save_reset_stats(adapter); 1660 ixgbevf_init_last_counter_stats(adapter); 1661 1662 /* bring the link up in the watchdog, this could race with our first 1663 * link up interrupt but shouldn't be a problem */ 1664 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 1665 adapter->link_check_timeout = jiffies; 1666 mod_timer(&adapter->watchdog_timer, jiffies); 1667 return 0; 1668 } 1669 1670 int ixgbevf_up(struct ixgbevf_adapter *adapter) 1671 { 1672 int err; 1673 struct ixgbe_hw *hw = &adapter->hw; 1674 1675 ixgbevf_configure(adapter); 1676 1677 err = ixgbevf_up_complete(adapter); 1678 1679 /* clear any pending interrupts, may auto mask */ 1680 IXGBE_READ_REG(hw, IXGBE_VTEICR); 1681 1682 ixgbevf_irq_enable(adapter, true, true); 1683 1684 return err; 1685 } 1686 1687 /** 1688 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 1689 * @adapter: board private structure 1690 * @rx_ring: ring to free buffers from 1691 **/ 1692 static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter, 1693 struct ixgbevf_ring *rx_ring) 1694 { 1695 struct pci_dev *pdev = adapter->pdev; 1696 unsigned long size; 1697 unsigned int i; 1698 1699 if (!rx_ring->rx_buffer_info) 1700 return; 1701 1702 /* Free all the Rx ring sk_buffs */ 1703 for (i = 0; i < rx_ring->count; i++) { 1704 struct ixgbevf_rx_buffer *rx_buffer_info; 1705 1706 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 1707 if (rx_buffer_info->dma) { 1708 dma_unmap_single(&pdev->dev, rx_buffer_info->dma, 1709 rx_ring->rx_buf_len, 1710 DMA_FROM_DEVICE); 1711 rx_buffer_info->dma = 0; 1712 } 1713 if (rx_buffer_info->skb) { 1714 struct sk_buff *skb = rx_buffer_info->skb; 1715 rx_buffer_info->skb = NULL; 1716 do { 1717 struct sk_buff *this = skb; 1718 skb = skb->prev; 1719 dev_kfree_skb(this); 1720 } while (skb); 1721 } 1722 if (!rx_buffer_info->page) 1723 continue; 1724 dma_unmap_page(&pdev->dev, rx_buffer_info->page_dma, 1725 PAGE_SIZE / 2, DMA_FROM_DEVICE); 1726 rx_buffer_info->page_dma = 0; 1727 put_page(rx_buffer_info->page); 1728 rx_buffer_info->page = NULL; 1729 rx_buffer_info->page_offset = 0; 1730 } 1731 1732 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 1733 memset(rx_ring->rx_buffer_info, 0, size); 1734 1735 /* Zero out the descriptor ring */ 1736 memset(rx_ring->desc, 0, rx_ring->size); 1737 1738 rx_ring->next_to_clean = 0; 1739 rx_ring->next_to_use = 0; 1740 1741 if (rx_ring->head) 1742 writel(0, adapter->hw.hw_addr + rx_ring->head); 1743 if (rx_ring->tail) 1744 writel(0, adapter->hw.hw_addr + rx_ring->tail); 1745 } 1746 1747 /** 1748 * ixgbevf_clean_tx_ring - Free Tx Buffers 1749 * @adapter: board private structure 1750 * @tx_ring: ring to be cleaned 1751 **/ 1752 static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter, 1753 struct ixgbevf_ring *tx_ring) 1754 { 1755 struct ixgbevf_tx_buffer *tx_buffer_info; 1756 unsigned long size; 1757 unsigned int i; 1758 1759 if (!tx_ring->tx_buffer_info) 1760 return; 1761 1762 /* Free all the Tx ring sk_buffs */ 1763 1764 for (i = 0; i < tx_ring->count; i++) { 1765 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 1766 ixgbevf_unmap_and_free_tx_resource(adapter, tx_buffer_info); 1767 } 1768 1769 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 1770 memset(tx_ring->tx_buffer_info, 0, size); 1771 1772 memset(tx_ring->desc, 0, tx_ring->size); 1773 1774 tx_ring->next_to_use = 0; 1775 tx_ring->next_to_clean = 0; 1776 1777 if (tx_ring->head) 1778 writel(0, adapter->hw.hw_addr + tx_ring->head); 1779 if (tx_ring->tail) 1780 writel(0, adapter->hw.hw_addr + tx_ring->tail); 1781 } 1782 1783 /** 1784 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 1785 * @adapter: board private structure 1786 **/ 1787 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 1788 { 1789 int i; 1790 1791 for (i = 0; i < adapter->num_rx_queues; i++) 1792 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]); 1793 } 1794 1795 /** 1796 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 1797 * @adapter: board private structure 1798 **/ 1799 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 1800 { 1801 int i; 1802 1803 for (i = 0; i < adapter->num_tx_queues; i++) 1804 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]); 1805 } 1806 1807 void ixgbevf_down(struct ixgbevf_adapter *adapter) 1808 { 1809 struct net_device *netdev = adapter->netdev; 1810 struct ixgbe_hw *hw = &adapter->hw; 1811 u32 txdctl; 1812 int i, j; 1813 1814 /* signal that we are down to the interrupt handler */ 1815 set_bit(__IXGBEVF_DOWN, &adapter->state); 1816 /* disable receives */ 1817 1818 netif_tx_disable(netdev); 1819 1820 msleep(10); 1821 1822 netif_tx_stop_all_queues(netdev); 1823 1824 ixgbevf_irq_disable(adapter); 1825 1826 ixgbevf_napi_disable_all(adapter); 1827 1828 del_timer_sync(&adapter->watchdog_timer); 1829 /* can't call flush scheduled work here because it can deadlock 1830 * if linkwatch_event tries to acquire the rtnl_lock which we are 1831 * holding */ 1832 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK) 1833 msleep(1); 1834 1835 /* disable transmits in the hardware now that interrupts are off */ 1836 for (i = 0; i < adapter->num_tx_queues; i++) { 1837 j = adapter->tx_ring[i].reg_idx; 1838 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1839 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), 1840 (txdctl & ~IXGBE_TXDCTL_ENABLE)); 1841 } 1842 1843 netif_carrier_off(netdev); 1844 1845 if (!pci_channel_offline(adapter->pdev)) 1846 ixgbevf_reset(adapter); 1847 1848 ixgbevf_clean_all_tx_rings(adapter); 1849 ixgbevf_clean_all_rx_rings(adapter); 1850 } 1851 1852 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 1853 { 1854 struct ixgbe_hw *hw = &adapter->hw; 1855 1856 WARN_ON(in_interrupt()); 1857 1858 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 1859 msleep(1); 1860 1861 /* 1862 * Check if PF is up before re-init. If not then skip until 1863 * later when the PF is up and ready to service requests from 1864 * the VF via mailbox. If the VF is up and running then the 1865 * watchdog task will continue to schedule reset tasks until 1866 * the PF is up and running. 1867 */ 1868 if (!hw->mac.ops.reset_hw(hw)) { 1869 ixgbevf_down(adapter); 1870 ixgbevf_up(adapter); 1871 } 1872 1873 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 1874 } 1875 1876 void ixgbevf_reset(struct ixgbevf_adapter *adapter) 1877 { 1878 struct ixgbe_hw *hw = &adapter->hw; 1879 struct net_device *netdev = adapter->netdev; 1880 1881 if (hw->mac.ops.reset_hw(hw)) 1882 hw_dbg(hw, "PF still resetting\n"); 1883 else 1884 hw->mac.ops.init_hw(hw); 1885 1886 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 1887 memcpy(netdev->dev_addr, adapter->hw.mac.addr, 1888 netdev->addr_len); 1889 memcpy(netdev->perm_addr, adapter->hw.mac.addr, 1890 netdev->addr_len); 1891 } 1892 } 1893 1894 static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 1895 int vectors) 1896 { 1897 int err, vector_threshold; 1898 1899 /* We'll want at least 3 (vector_threshold): 1900 * 1) TxQ[0] Cleanup 1901 * 2) RxQ[0] Cleanup 1902 * 3) Other (Link Status Change, etc.) 1903 */ 1904 vector_threshold = MIN_MSIX_COUNT; 1905 1906 /* The more we get, the more we will assign to Tx/Rx Cleanup 1907 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 1908 * Right now, we simply care about how many we'll get; we'll 1909 * set them up later while requesting irq's. 1910 */ 1911 while (vectors >= vector_threshold) { 1912 err = pci_enable_msix(adapter->pdev, adapter->msix_entries, 1913 vectors); 1914 if (!err) /* Success in acquiring all requested vectors. */ 1915 break; 1916 else if (err < 0) 1917 vectors = 0; /* Nasty failure, quit now */ 1918 else /* err == number of vectors we should try again with */ 1919 vectors = err; 1920 } 1921 1922 if (vectors < vector_threshold) { 1923 /* Can't allocate enough MSI-X interrupts? Oh well. 1924 * This just means we'll go with either a single MSI 1925 * vector or fall back to legacy interrupts. 1926 */ 1927 hw_dbg(&adapter->hw, 1928 "Unable to allocate MSI-X interrupts\n"); 1929 kfree(adapter->msix_entries); 1930 adapter->msix_entries = NULL; 1931 } else { 1932 /* 1933 * Adjust for only the vectors we'll use, which is minimum 1934 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 1935 * vectors we were allocated. 1936 */ 1937 adapter->num_msix_vectors = vectors; 1938 } 1939 } 1940 1941 /* 1942 * ixgbevf_set_num_queues: Allocate queues for device, feature dependent 1943 * @adapter: board private structure to initialize 1944 * 1945 * This is the top level queue allocation routine. The order here is very 1946 * important, starting with the "most" number of features turned on at once, 1947 * and ending with the smallest set of features. This way large combinations 1948 * can be allocated if they're turned on, and smaller combinations are the 1949 * fallthrough conditions. 1950 * 1951 **/ 1952 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 1953 { 1954 /* Start with base case */ 1955 adapter->num_rx_queues = 1; 1956 adapter->num_tx_queues = 1; 1957 adapter->num_rx_pools = adapter->num_rx_queues; 1958 adapter->num_rx_queues_per_pool = 1; 1959 } 1960 1961 /** 1962 * ixgbevf_alloc_queues - Allocate memory for all rings 1963 * @adapter: board private structure to initialize 1964 * 1965 * We allocate one ring per queue at run-time since we don't know the 1966 * number of queues at compile-time. The polling_netdev array is 1967 * intended for Multiqueue, but should work fine with a single queue. 1968 **/ 1969 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) 1970 { 1971 int i; 1972 1973 adapter->tx_ring = kcalloc(adapter->num_tx_queues, 1974 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1975 if (!adapter->tx_ring) 1976 goto err_tx_ring_allocation; 1977 1978 adapter->rx_ring = kcalloc(adapter->num_rx_queues, 1979 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1980 if (!adapter->rx_ring) 1981 goto err_rx_ring_allocation; 1982 1983 for (i = 0; i < adapter->num_tx_queues; i++) { 1984 adapter->tx_ring[i].count = adapter->tx_ring_count; 1985 adapter->tx_ring[i].queue_index = i; 1986 adapter->tx_ring[i].reg_idx = i; 1987 } 1988 1989 for (i = 0; i < adapter->num_rx_queues; i++) { 1990 adapter->rx_ring[i].count = adapter->rx_ring_count; 1991 adapter->rx_ring[i].queue_index = i; 1992 adapter->rx_ring[i].reg_idx = i; 1993 } 1994 1995 return 0; 1996 1997 err_rx_ring_allocation: 1998 kfree(adapter->tx_ring); 1999 err_tx_ring_allocation: 2000 return -ENOMEM; 2001 } 2002 2003 /** 2004 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 2005 * @adapter: board private structure to initialize 2006 * 2007 * Attempt to configure the interrupts using the best available 2008 * capabilities of the hardware and the kernel. 2009 **/ 2010 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 2011 { 2012 int err = 0; 2013 int vector, v_budget; 2014 2015 /* 2016 * It's easy to be greedy for MSI-X vectors, but it really 2017 * doesn't do us much good if we have a lot more vectors 2018 * than CPU's. So let's be conservative and only ask for 2019 * (roughly) twice the number of vectors as there are CPU's. 2020 */ 2021 v_budget = min(adapter->num_rx_queues + adapter->num_tx_queues, 2022 (int)(num_online_cpus() * 2)) + NON_Q_VECTORS; 2023 2024 /* A failure in MSI-X entry allocation isn't fatal, but it does 2025 * mean we disable MSI-X capabilities of the adapter. */ 2026 adapter->msix_entries = kcalloc(v_budget, 2027 sizeof(struct msix_entry), GFP_KERNEL); 2028 if (!adapter->msix_entries) { 2029 err = -ENOMEM; 2030 goto out; 2031 } 2032 2033 for (vector = 0; vector < v_budget; vector++) 2034 adapter->msix_entries[vector].entry = vector; 2035 2036 ixgbevf_acquire_msix_vectors(adapter, v_budget); 2037 2038 out: 2039 return err; 2040 } 2041 2042 /** 2043 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 2044 * @adapter: board private structure to initialize 2045 * 2046 * We allocate one q_vector per queue interrupt. If allocation fails we 2047 * return -ENOMEM. 2048 **/ 2049 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 2050 { 2051 int q_idx, num_q_vectors; 2052 struct ixgbevf_q_vector *q_vector; 2053 int napi_vectors; 2054 int (*poll)(struct napi_struct *, int); 2055 2056 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2057 napi_vectors = adapter->num_rx_queues; 2058 poll = &ixgbevf_clean_rxonly; 2059 2060 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2061 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL); 2062 if (!q_vector) 2063 goto err_out; 2064 q_vector->adapter = adapter; 2065 q_vector->v_idx = q_idx; 2066 q_vector->eitr = adapter->eitr_param; 2067 if (q_idx < napi_vectors) 2068 netif_napi_add(adapter->netdev, &q_vector->napi, 2069 (*poll), 64); 2070 adapter->q_vector[q_idx] = q_vector; 2071 } 2072 2073 return 0; 2074 2075 err_out: 2076 while (q_idx) { 2077 q_idx--; 2078 q_vector = adapter->q_vector[q_idx]; 2079 netif_napi_del(&q_vector->napi); 2080 kfree(q_vector); 2081 adapter->q_vector[q_idx] = NULL; 2082 } 2083 return -ENOMEM; 2084 } 2085 2086 /** 2087 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 2088 * @adapter: board private structure to initialize 2089 * 2090 * This function frees the memory allocated to the q_vectors. In addition if 2091 * NAPI is enabled it will delete any references to the NAPI struct prior 2092 * to freeing the q_vector. 2093 **/ 2094 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 2095 { 2096 int q_idx, num_q_vectors; 2097 int napi_vectors; 2098 2099 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2100 napi_vectors = adapter->num_rx_queues; 2101 2102 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2103 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; 2104 2105 adapter->q_vector[q_idx] = NULL; 2106 if (q_idx < napi_vectors) 2107 netif_napi_del(&q_vector->napi); 2108 kfree(q_vector); 2109 } 2110 } 2111 2112 /** 2113 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 2114 * @adapter: board private structure 2115 * 2116 **/ 2117 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 2118 { 2119 pci_disable_msix(adapter->pdev); 2120 kfree(adapter->msix_entries); 2121 adapter->msix_entries = NULL; 2122 } 2123 2124 /** 2125 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 2126 * @adapter: board private structure to initialize 2127 * 2128 **/ 2129 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 2130 { 2131 int err; 2132 2133 /* Number of supported queues */ 2134 ixgbevf_set_num_queues(adapter); 2135 2136 err = ixgbevf_set_interrupt_capability(adapter); 2137 if (err) { 2138 hw_dbg(&adapter->hw, 2139 "Unable to setup interrupt capabilities\n"); 2140 goto err_set_interrupt; 2141 } 2142 2143 err = ixgbevf_alloc_q_vectors(adapter); 2144 if (err) { 2145 hw_dbg(&adapter->hw, "Unable to allocate memory for queue " 2146 "vectors\n"); 2147 goto err_alloc_q_vectors; 2148 } 2149 2150 err = ixgbevf_alloc_queues(adapter); 2151 if (err) { 2152 pr_err("Unable to allocate memory for queues\n"); 2153 goto err_alloc_queues; 2154 } 2155 2156 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, " 2157 "Tx Queue count = %u\n", 2158 (adapter->num_rx_queues > 1) ? "Enabled" : 2159 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues); 2160 2161 set_bit(__IXGBEVF_DOWN, &adapter->state); 2162 2163 return 0; 2164 err_alloc_queues: 2165 ixgbevf_free_q_vectors(adapter); 2166 err_alloc_q_vectors: 2167 ixgbevf_reset_interrupt_capability(adapter); 2168 err_set_interrupt: 2169 return err; 2170 } 2171 2172 /** 2173 * ixgbevf_sw_init - Initialize general software structures 2174 * (struct ixgbevf_adapter) 2175 * @adapter: board private structure to initialize 2176 * 2177 * ixgbevf_sw_init initializes the Adapter private data structure. 2178 * Fields are initialized based on PCI device information and 2179 * OS network device settings (MTU size). 2180 **/ 2181 static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 2182 { 2183 struct ixgbe_hw *hw = &adapter->hw; 2184 struct pci_dev *pdev = adapter->pdev; 2185 int err; 2186 2187 /* PCI config space info */ 2188 2189 hw->vendor_id = pdev->vendor; 2190 hw->device_id = pdev->device; 2191 hw->revision_id = pdev->revision; 2192 hw->subsystem_vendor_id = pdev->subsystem_vendor; 2193 hw->subsystem_device_id = pdev->subsystem_device; 2194 2195 hw->mbx.ops.init_params(hw); 2196 hw->mac.max_tx_queues = MAX_TX_QUEUES; 2197 hw->mac.max_rx_queues = MAX_RX_QUEUES; 2198 err = hw->mac.ops.reset_hw(hw); 2199 if (err) { 2200 dev_info(&pdev->dev, 2201 "PF still in reset state, assigning new address\n"); 2202 eth_hw_addr_random(adapter->netdev); 2203 memcpy(adapter->hw.mac.addr, adapter->netdev->dev_addr, 2204 adapter->netdev->addr_len); 2205 } else { 2206 err = hw->mac.ops.init_hw(hw); 2207 if (err) { 2208 pr_err("init_shared_code failed: %d\n", err); 2209 goto out; 2210 } 2211 memcpy(adapter->netdev->dev_addr, adapter->hw.mac.addr, 2212 adapter->netdev->addr_len); 2213 } 2214 2215 /* Enable dynamic interrupt throttling rates */ 2216 adapter->eitr_param = 20000; 2217 adapter->itr_setting = 1; 2218 2219 /* set defaults for eitr in MegaBytes */ 2220 adapter->eitr_low = 10; 2221 adapter->eitr_high = 20; 2222 2223 /* set default ring sizes */ 2224 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 2225 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 2226 2227 /* enable rx csum by default */ 2228 adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED; 2229 2230 set_bit(__IXGBEVF_DOWN, &adapter->state); 2231 return 0; 2232 2233 out: 2234 return err; 2235 } 2236 2237 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 2238 { \ 2239 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 2240 if (current_counter < last_counter) \ 2241 counter += 0x100000000LL; \ 2242 last_counter = current_counter; \ 2243 counter &= 0xFFFFFFFF00000000LL; \ 2244 counter |= current_counter; \ 2245 } 2246 2247 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 2248 { \ 2249 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 2250 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 2251 u64 current_counter = (current_counter_msb << 32) | \ 2252 current_counter_lsb; \ 2253 if (current_counter < last_counter) \ 2254 counter += 0x1000000000LL; \ 2255 last_counter = current_counter; \ 2256 counter &= 0xFFFFFFF000000000LL; \ 2257 counter |= current_counter; \ 2258 } 2259 /** 2260 * ixgbevf_update_stats - Update the board statistics counters. 2261 * @adapter: board private structure 2262 **/ 2263 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 2264 { 2265 struct ixgbe_hw *hw = &adapter->hw; 2266 2267 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 2268 adapter->stats.vfgprc); 2269 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 2270 adapter->stats.vfgptc); 2271 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2272 adapter->stats.last_vfgorc, 2273 adapter->stats.vfgorc); 2274 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2275 adapter->stats.last_vfgotc, 2276 adapter->stats.vfgotc); 2277 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 2278 adapter->stats.vfmprc); 2279 } 2280 2281 /** 2282 * ixgbevf_watchdog - Timer Call-back 2283 * @data: pointer to adapter cast into an unsigned long 2284 **/ 2285 static void ixgbevf_watchdog(unsigned long data) 2286 { 2287 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data; 2288 struct ixgbe_hw *hw = &adapter->hw; 2289 u64 eics = 0; 2290 int i; 2291 2292 /* 2293 * Do the watchdog outside of interrupt context due to the lovely 2294 * delays that some of the newer hardware requires 2295 */ 2296 2297 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 2298 goto watchdog_short_circuit; 2299 2300 /* get one bit for every active tx/rx interrupt vector */ 2301 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 2302 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 2303 if (qv->rxr_count || qv->txr_count) 2304 eics |= (1 << i); 2305 } 2306 2307 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, (u32)eics); 2308 2309 watchdog_short_circuit: 2310 schedule_work(&adapter->watchdog_task); 2311 } 2312 2313 /** 2314 * ixgbevf_tx_timeout - Respond to a Tx Hang 2315 * @netdev: network interface device structure 2316 **/ 2317 static void ixgbevf_tx_timeout(struct net_device *netdev) 2318 { 2319 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2320 2321 /* Do the reset outside of interrupt context */ 2322 schedule_work(&adapter->reset_task); 2323 } 2324 2325 static void ixgbevf_reset_task(struct work_struct *work) 2326 { 2327 struct ixgbevf_adapter *adapter; 2328 adapter = container_of(work, struct ixgbevf_adapter, reset_task); 2329 2330 /* If we're already down or resetting, just bail */ 2331 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2332 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2333 return; 2334 2335 adapter->tx_timeout_count++; 2336 2337 ixgbevf_reinit_locked(adapter); 2338 } 2339 2340 /** 2341 * ixgbevf_watchdog_task - worker thread to bring link up 2342 * @work: pointer to work_struct containing our data 2343 **/ 2344 static void ixgbevf_watchdog_task(struct work_struct *work) 2345 { 2346 struct ixgbevf_adapter *adapter = container_of(work, 2347 struct ixgbevf_adapter, 2348 watchdog_task); 2349 struct net_device *netdev = adapter->netdev; 2350 struct ixgbe_hw *hw = &adapter->hw; 2351 u32 link_speed = adapter->link_speed; 2352 bool link_up = adapter->link_up; 2353 2354 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK; 2355 2356 /* 2357 * Always check the link on the watchdog because we have 2358 * no LSC interrupt 2359 */ 2360 if (hw->mac.ops.check_link) { 2361 if ((hw->mac.ops.check_link(hw, &link_speed, 2362 &link_up, false)) != 0) { 2363 adapter->link_up = link_up; 2364 adapter->link_speed = link_speed; 2365 netif_carrier_off(netdev); 2366 netif_tx_stop_all_queues(netdev); 2367 schedule_work(&adapter->reset_task); 2368 goto pf_has_reset; 2369 } 2370 } else { 2371 /* always assume link is up, if no check link 2372 * function */ 2373 link_speed = IXGBE_LINK_SPEED_10GB_FULL; 2374 link_up = true; 2375 } 2376 adapter->link_up = link_up; 2377 adapter->link_speed = link_speed; 2378 2379 if (link_up) { 2380 if (!netif_carrier_ok(netdev)) { 2381 hw_dbg(&adapter->hw, "NIC Link is Up, %u Gbps\n", 2382 (link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 2383 10 : 1); 2384 netif_carrier_on(netdev); 2385 netif_tx_wake_all_queues(netdev); 2386 } 2387 } else { 2388 adapter->link_up = false; 2389 adapter->link_speed = 0; 2390 if (netif_carrier_ok(netdev)) { 2391 hw_dbg(&adapter->hw, "NIC Link is Down\n"); 2392 netif_carrier_off(netdev); 2393 netif_tx_stop_all_queues(netdev); 2394 } 2395 } 2396 2397 ixgbevf_update_stats(adapter); 2398 2399 pf_has_reset: 2400 /* Reset the timer */ 2401 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 2402 mod_timer(&adapter->watchdog_timer, 2403 round_jiffies(jiffies + (2 * HZ))); 2404 2405 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK; 2406 } 2407 2408 /** 2409 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 2410 * @adapter: board private structure 2411 * @tx_ring: Tx descriptor ring for a specific queue 2412 * 2413 * Free all transmit software resources 2414 **/ 2415 void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter, 2416 struct ixgbevf_ring *tx_ring) 2417 { 2418 struct pci_dev *pdev = adapter->pdev; 2419 2420 ixgbevf_clean_tx_ring(adapter, tx_ring); 2421 2422 vfree(tx_ring->tx_buffer_info); 2423 tx_ring->tx_buffer_info = NULL; 2424 2425 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc, 2426 tx_ring->dma); 2427 2428 tx_ring->desc = NULL; 2429 } 2430 2431 /** 2432 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 2433 * @adapter: board private structure 2434 * 2435 * Free all transmit software resources 2436 **/ 2437 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 2438 { 2439 int i; 2440 2441 for (i = 0; i < adapter->num_tx_queues; i++) 2442 if (adapter->tx_ring[i].desc) 2443 ixgbevf_free_tx_resources(adapter, 2444 &adapter->tx_ring[i]); 2445 2446 } 2447 2448 /** 2449 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 2450 * @adapter: board private structure 2451 * @tx_ring: tx descriptor ring (for a specific queue) to setup 2452 * 2453 * Return 0 on success, negative on failure 2454 **/ 2455 int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter, 2456 struct ixgbevf_ring *tx_ring) 2457 { 2458 struct pci_dev *pdev = adapter->pdev; 2459 int size; 2460 2461 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2462 tx_ring->tx_buffer_info = vzalloc(size); 2463 if (!tx_ring->tx_buffer_info) 2464 goto err; 2465 2466 /* round up to nearest 4K */ 2467 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 2468 tx_ring->size = ALIGN(tx_ring->size, 4096); 2469 2470 tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size, 2471 &tx_ring->dma, GFP_KERNEL); 2472 if (!tx_ring->desc) 2473 goto err; 2474 2475 tx_ring->next_to_use = 0; 2476 tx_ring->next_to_clean = 0; 2477 tx_ring->work_limit = tx_ring->count; 2478 return 0; 2479 2480 err: 2481 vfree(tx_ring->tx_buffer_info); 2482 tx_ring->tx_buffer_info = NULL; 2483 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit " 2484 "descriptor ring\n"); 2485 return -ENOMEM; 2486 } 2487 2488 /** 2489 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 2490 * @adapter: board private structure 2491 * 2492 * If this function returns with an error, then it's possible one or 2493 * more of the rings is populated (while the rest are not). It is the 2494 * callers duty to clean those orphaned rings. 2495 * 2496 * Return 0 on success, negative on failure 2497 **/ 2498 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 2499 { 2500 int i, err = 0; 2501 2502 for (i = 0; i < adapter->num_tx_queues; i++) { 2503 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]); 2504 if (!err) 2505 continue; 2506 hw_dbg(&adapter->hw, 2507 "Allocation for Tx Queue %u failed\n", i); 2508 break; 2509 } 2510 2511 return err; 2512 } 2513 2514 /** 2515 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 2516 * @adapter: board private structure 2517 * @rx_ring: rx descriptor ring (for a specific queue) to setup 2518 * 2519 * Returns 0 on success, negative on failure 2520 **/ 2521 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, 2522 struct ixgbevf_ring *rx_ring) 2523 { 2524 struct pci_dev *pdev = adapter->pdev; 2525 int size; 2526 2527 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 2528 rx_ring->rx_buffer_info = vzalloc(size); 2529 if (!rx_ring->rx_buffer_info) 2530 goto alloc_failed; 2531 2532 /* Round up to nearest 4K */ 2533 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 2534 rx_ring->size = ALIGN(rx_ring->size, 4096); 2535 2536 rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size, 2537 &rx_ring->dma, GFP_KERNEL); 2538 2539 if (!rx_ring->desc) { 2540 hw_dbg(&adapter->hw, 2541 "Unable to allocate memory for " 2542 "the receive descriptor ring\n"); 2543 vfree(rx_ring->rx_buffer_info); 2544 rx_ring->rx_buffer_info = NULL; 2545 goto alloc_failed; 2546 } 2547 2548 rx_ring->next_to_clean = 0; 2549 rx_ring->next_to_use = 0; 2550 2551 return 0; 2552 alloc_failed: 2553 return -ENOMEM; 2554 } 2555 2556 /** 2557 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 2558 * @adapter: board private structure 2559 * 2560 * If this function returns with an error, then it's possible one or 2561 * more of the rings is populated (while the rest are not). It is the 2562 * callers duty to clean those orphaned rings. 2563 * 2564 * Return 0 on success, negative on failure 2565 **/ 2566 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 2567 { 2568 int i, err = 0; 2569 2570 for (i = 0; i < adapter->num_rx_queues; i++) { 2571 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]); 2572 if (!err) 2573 continue; 2574 hw_dbg(&adapter->hw, 2575 "Allocation for Rx Queue %u failed\n", i); 2576 break; 2577 } 2578 return err; 2579 } 2580 2581 /** 2582 * ixgbevf_free_rx_resources - Free Rx Resources 2583 * @adapter: board private structure 2584 * @rx_ring: ring to clean the resources from 2585 * 2586 * Free all receive software resources 2587 **/ 2588 void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter, 2589 struct ixgbevf_ring *rx_ring) 2590 { 2591 struct pci_dev *pdev = adapter->pdev; 2592 2593 ixgbevf_clean_rx_ring(adapter, rx_ring); 2594 2595 vfree(rx_ring->rx_buffer_info); 2596 rx_ring->rx_buffer_info = NULL; 2597 2598 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, 2599 rx_ring->dma); 2600 2601 rx_ring->desc = NULL; 2602 } 2603 2604 /** 2605 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 2606 * @adapter: board private structure 2607 * 2608 * Free all receive software resources 2609 **/ 2610 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 2611 { 2612 int i; 2613 2614 for (i = 0; i < adapter->num_rx_queues; i++) 2615 if (adapter->rx_ring[i].desc) 2616 ixgbevf_free_rx_resources(adapter, 2617 &adapter->rx_ring[i]); 2618 } 2619 2620 /** 2621 * ixgbevf_open - Called when a network interface is made active 2622 * @netdev: network interface device structure 2623 * 2624 * Returns 0 on success, negative value on failure 2625 * 2626 * The open entry point is called when a network interface is made 2627 * active by the system (IFF_UP). At this point all resources needed 2628 * for transmit and receive operations are allocated, the interrupt 2629 * handler is registered with the OS, the watchdog timer is started, 2630 * and the stack is notified that the interface is ready. 2631 **/ 2632 static int ixgbevf_open(struct net_device *netdev) 2633 { 2634 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2635 struct ixgbe_hw *hw = &adapter->hw; 2636 int err; 2637 2638 /* disallow open during test */ 2639 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 2640 return -EBUSY; 2641 2642 if (hw->adapter_stopped) { 2643 ixgbevf_reset(adapter); 2644 /* if adapter is still stopped then PF isn't up and 2645 * the vf can't start. */ 2646 if (hw->adapter_stopped) { 2647 err = IXGBE_ERR_MBX; 2648 pr_err("Unable to start - perhaps the PF Driver isn't " 2649 "up yet\n"); 2650 goto err_setup_reset; 2651 } 2652 } 2653 2654 /* allocate transmit descriptors */ 2655 err = ixgbevf_setup_all_tx_resources(adapter); 2656 if (err) 2657 goto err_setup_tx; 2658 2659 /* allocate receive descriptors */ 2660 err = ixgbevf_setup_all_rx_resources(adapter); 2661 if (err) 2662 goto err_setup_rx; 2663 2664 ixgbevf_configure(adapter); 2665 2666 /* 2667 * Map the Tx/Rx rings to the vectors we were allotted. 2668 * if request_irq will be called in this function map_rings 2669 * must be called *before* up_complete 2670 */ 2671 ixgbevf_map_rings_to_vectors(adapter); 2672 2673 err = ixgbevf_up_complete(adapter); 2674 if (err) 2675 goto err_up; 2676 2677 /* clear any pending interrupts, may auto mask */ 2678 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2679 err = ixgbevf_request_irq(adapter); 2680 if (err) 2681 goto err_req_irq; 2682 2683 ixgbevf_irq_enable(adapter, true, true); 2684 2685 return 0; 2686 2687 err_req_irq: 2688 ixgbevf_down(adapter); 2689 err_up: 2690 ixgbevf_free_irq(adapter); 2691 err_setup_rx: 2692 ixgbevf_free_all_rx_resources(adapter); 2693 err_setup_tx: 2694 ixgbevf_free_all_tx_resources(adapter); 2695 ixgbevf_reset(adapter); 2696 2697 err_setup_reset: 2698 2699 return err; 2700 } 2701 2702 /** 2703 * ixgbevf_close - Disables a network interface 2704 * @netdev: network interface device structure 2705 * 2706 * Returns 0, this is not allowed to fail 2707 * 2708 * The close entry point is called when an interface is de-activated 2709 * by the OS. The hardware is still under the drivers control, but 2710 * needs to be disabled. A global MAC reset is issued to stop the 2711 * hardware, and all transmit and receive resources are freed. 2712 **/ 2713 static int ixgbevf_close(struct net_device *netdev) 2714 { 2715 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2716 2717 ixgbevf_down(adapter); 2718 ixgbevf_free_irq(adapter); 2719 2720 ixgbevf_free_all_tx_resources(adapter); 2721 ixgbevf_free_all_rx_resources(adapter); 2722 2723 return 0; 2724 } 2725 2726 static int ixgbevf_tso(struct ixgbevf_adapter *adapter, 2727 struct ixgbevf_ring *tx_ring, 2728 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len) 2729 { 2730 struct ixgbe_adv_tx_context_desc *context_desc; 2731 unsigned int i; 2732 int err; 2733 struct ixgbevf_tx_buffer *tx_buffer_info; 2734 u32 vlan_macip_lens = 0, type_tucmd_mlhl; 2735 u32 mss_l4len_idx, l4len; 2736 2737 if (skb_is_gso(skb)) { 2738 if (skb_header_cloned(skb)) { 2739 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2740 if (err) 2741 return err; 2742 } 2743 l4len = tcp_hdrlen(skb); 2744 *hdr_len += l4len; 2745 2746 if (skb->protocol == htons(ETH_P_IP)) { 2747 struct iphdr *iph = ip_hdr(skb); 2748 iph->tot_len = 0; 2749 iph->check = 0; 2750 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 2751 iph->daddr, 0, 2752 IPPROTO_TCP, 2753 0); 2754 adapter->hw_tso_ctxt++; 2755 } else if (skb_is_gso_v6(skb)) { 2756 ipv6_hdr(skb)->payload_len = 0; 2757 tcp_hdr(skb)->check = 2758 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 2759 &ipv6_hdr(skb)->daddr, 2760 0, IPPROTO_TCP, 0); 2761 adapter->hw_tso6_ctxt++; 2762 } 2763 2764 i = tx_ring->next_to_use; 2765 2766 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2767 context_desc = IXGBE_TX_CTXTDESC_ADV(*tx_ring, i); 2768 2769 /* VLAN MACLEN IPLEN */ 2770 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 2771 vlan_macip_lens |= 2772 (tx_flags & IXGBE_TX_FLAGS_VLAN_MASK); 2773 vlan_macip_lens |= ((skb_network_offset(skb)) << 2774 IXGBE_ADVTXD_MACLEN_SHIFT); 2775 *hdr_len += skb_network_offset(skb); 2776 vlan_macip_lens |= 2777 (skb_transport_header(skb) - skb_network_header(skb)); 2778 *hdr_len += 2779 (skb_transport_header(skb) - skb_network_header(skb)); 2780 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 2781 context_desc->seqnum_seed = 0; 2782 2783 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 2784 type_tucmd_mlhl = (IXGBE_TXD_CMD_DEXT | 2785 IXGBE_ADVTXD_DTYP_CTXT); 2786 2787 if (skb->protocol == htons(ETH_P_IP)) 2788 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2789 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2790 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl); 2791 2792 /* MSS L4LEN IDX */ 2793 mss_l4len_idx = 2794 (skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT); 2795 mss_l4len_idx |= (l4len << IXGBE_ADVTXD_L4LEN_SHIFT); 2796 /* use index 1 for TSO */ 2797 mss_l4len_idx |= (1 << IXGBE_ADVTXD_IDX_SHIFT); 2798 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 2799 2800 tx_buffer_info->time_stamp = jiffies; 2801 tx_buffer_info->next_to_watch = i; 2802 2803 i++; 2804 if (i == tx_ring->count) 2805 i = 0; 2806 tx_ring->next_to_use = i; 2807 2808 return true; 2809 } 2810 2811 return false; 2812 } 2813 2814 static bool ixgbevf_tx_csum(struct ixgbevf_adapter *adapter, 2815 struct ixgbevf_ring *tx_ring, 2816 struct sk_buff *skb, u32 tx_flags) 2817 { 2818 struct ixgbe_adv_tx_context_desc *context_desc; 2819 unsigned int i; 2820 struct ixgbevf_tx_buffer *tx_buffer_info; 2821 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2822 2823 if (skb->ip_summed == CHECKSUM_PARTIAL || 2824 (tx_flags & IXGBE_TX_FLAGS_VLAN)) { 2825 i = tx_ring->next_to_use; 2826 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2827 context_desc = IXGBE_TX_CTXTDESC_ADV(*tx_ring, i); 2828 2829 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 2830 vlan_macip_lens |= (tx_flags & 2831 IXGBE_TX_FLAGS_VLAN_MASK); 2832 vlan_macip_lens |= (skb_network_offset(skb) << 2833 IXGBE_ADVTXD_MACLEN_SHIFT); 2834 if (skb->ip_summed == CHECKSUM_PARTIAL) 2835 vlan_macip_lens |= (skb_transport_header(skb) - 2836 skb_network_header(skb)); 2837 2838 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 2839 context_desc->seqnum_seed = 0; 2840 2841 type_tucmd_mlhl |= (IXGBE_TXD_CMD_DEXT | 2842 IXGBE_ADVTXD_DTYP_CTXT); 2843 2844 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2845 switch (skb->protocol) { 2846 case __constant_htons(ETH_P_IP): 2847 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2848 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 2849 type_tucmd_mlhl |= 2850 IXGBE_ADVTXD_TUCMD_L4T_TCP; 2851 break; 2852 case __constant_htons(ETH_P_IPV6): 2853 /* XXX what about other V6 headers?? */ 2854 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 2855 type_tucmd_mlhl |= 2856 IXGBE_ADVTXD_TUCMD_L4T_TCP; 2857 break; 2858 default: 2859 if (unlikely(net_ratelimit())) { 2860 pr_warn("partial checksum but " 2861 "proto=%x!\n", skb->protocol); 2862 } 2863 break; 2864 } 2865 } 2866 2867 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl); 2868 /* use index zero for tx checksum offload */ 2869 context_desc->mss_l4len_idx = 0; 2870 2871 tx_buffer_info->time_stamp = jiffies; 2872 tx_buffer_info->next_to_watch = i; 2873 2874 adapter->hw_csum_tx_good++; 2875 i++; 2876 if (i == tx_ring->count) 2877 i = 0; 2878 tx_ring->next_to_use = i; 2879 2880 return true; 2881 } 2882 2883 return false; 2884 } 2885 2886 static int ixgbevf_tx_map(struct ixgbevf_adapter *adapter, 2887 struct ixgbevf_ring *tx_ring, 2888 struct sk_buff *skb, u32 tx_flags, 2889 unsigned int first) 2890 { 2891 struct pci_dev *pdev = adapter->pdev; 2892 struct ixgbevf_tx_buffer *tx_buffer_info; 2893 unsigned int len; 2894 unsigned int total = skb->len; 2895 unsigned int offset = 0, size; 2896 int count = 0; 2897 unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 2898 unsigned int f; 2899 int i; 2900 2901 i = tx_ring->next_to_use; 2902 2903 len = min(skb_headlen(skb), total); 2904 while (len) { 2905 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2906 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); 2907 2908 tx_buffer_info->length = size; 2909 tx_buffer_info->mapped_as_page = false; 2910 tx_buffer_info->dma = dma_map_single(&adapter->pdev->dev, 2911 skb->data + offset, 2912 size, DMA_TO_DEVICE); 2913 if (dma_mapping_error(&pdev->dev, tx_buffer_info->dma)) 2914 goto dma_error; 2915 tx_buffer_info->time_stamp = jiffies; 2916 tx_buffer_info->next_to_watch = i; 2917 2918 len -= size; 2919 total -= size; 2920 offset += size; 2921 count++; 2922 i++; 2923 if (i == tx_ring->count) 2924 i = 0; 2925 } 2926 2927 for (f = 0; f < nr_frags; f++) { 2928 const struct skb_frag_struct *frag; 2929 2930 frag = &skb_shinfo(skb)->frags[f]; 2931 len = min((unsigned int)skb_frag_size(frag), total); 2932 offset = 0; 2933 2934 while (len) { 2935 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2936 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); 2937 2938 tx_buffer_info->length = size; 2939 tx_buffer_info->dma = 2940 skb_frag_dma_map(&adapter->pdev->dev, frag, 2941 offset, size, DMA_TO_DEVICE); 2942 tx_buffer_info->mapped_as_page = true; 2943 if (dma_mapping_error(&pdev->dev, tx_buffer_info->dma)) 2944 goto dma_error; 2945 tx_buffer_info->time_stamp = jiffies; 2946 tx_buffer_info->next_to_watch = i; 2947 2948 len -= size; 2949 total -= size; 2950 offset += size; 2951 count++; 2952 i++; 2953 if (i == tx_ring->count) 2954 i = 0; 2955 } 2956 if (total == 0) 2957 break; 2958 } 2959 2960 if (i == 0) 2961 i = tx_ring->count - 1; 2962 else 2963 i = i - 1; 2964 tx_ring->tx_buffer_info[i].skb = skb; 2965 tx_ring->tx_buffer_info[first].next_to_watch = i; 2966 2967 return count; 2968 2969 dma_error: 2970 dev_err(&pdev->dev, "TX DMA map failed\n"); 2971 2972 /* clear timestamp and dma mappings for failed tx_buffer_info map */ 2973 tx_buffer_info->dma = 0; 2974 tx_buffer_info->time_stamp = 0; 2975 tx_buffer_info->next_to_watch = 0; 2976 count--; 2977 2978 /* clear timestamp and dma mappings for remaining portion of packet */ 2979 while (count >= 0) { 2980 count--; 2981 i--; 2982 if (i < 0) 2983 i += tx_ring->count; 2984 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2985 ixgbevf_unmap_and_free_tx_resource(adapter, tx_buffer_info); 2986 } 2987 2988 return count; 2989 } 2990 2991 static void ixgbevf_tx_queue(struct ixgbevf_adapter *adapter, 2992 struct ixgbevf_ring *tx_ring, int tx_flags, 2993 int count, u32 paylen, u8 hdr_len) 2994 { 2995 union ixgbe_adv_tx_desc *tx_desc = NULL; 2996 struct ixgbevf_tx_buffer *tx_buffer_info; 2997 u32 olinfo_status = 0, cmd_type_len = 0; 2998 unsigned int i; 2999 3000 u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS; 3001 3002 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; 3003 3004 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 3005 3006 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 3007 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 3008 3009 if (tx_flags & IXGBE_TX_FLAGS_TSO) { 3010 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 3011 3012 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 3013 IXGBE_ADVTXD_POPTS_SHIFT; 3014 3015 /* use index 1 context for tso */ 3016 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT); 3017 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 3018 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 3019 IXGBE_ADVTXD_POPTS_SHIFT; 3020 3021 } else if (tx_flags & IXGBE_TX_FLAGS_CSUM) 3022 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 3023 IXGBE_ADVTXD_POPTS_SHIFT; 3024 3025 olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT); 3026 3027 i = tx_ring->next_to_use; 3028 while (count--) { 3029 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 3030 tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i); 3031 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma); 3032 tx_desc->read.cmd_type_len = 3033 cpu_to_le32(cmd_type_len | tx_buffer_info->length); 3034 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 3035 i++; 3036 if (i == tx_ring->count) 3037 i = 0; 3038 } 3039 3040 tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd); 3041 3042 /* 3043 * Force memory writes to complete before letting h/w 3044 * know there are new descriptors to fetch. (Only 3045 * applicable for weak-ordered memory model archs, 3046 * such as IA-64). 3047 */ 3048 wmb(); 3049 3050 tx_ring->next_to_use = i; 3051 writel(i, adapter->hw.hw_addr + tx_ring->tail); 3052 } 3053 3054 static int __ixgbevf_maybe_stop_tx(struct net_device *netdev, 3055 struct ixgbevf_ring *tx_ring, int size) 3056 { 3057 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3058 3059 netif_stop_subqueue(netdev, tx_ring->queue_index); 3060 /* Herbert's original patch had: 3061 * smp_mb__after_netif_stop_queue(); 3062 * but since that doesn't exist yet, just open code it. */ 3063 smp_mb(); 3064 3065 /* We need to check again in a case another CPU has just 3066 * made room available. */ 3067 if (likely(IXGBE_DESC_UNUSED(tx_ring) < size)) 3068 return -EBUSY; 3069 3070 /* A reprieve! - use start_queue because it doesn't call schedule */ 3071 netif_start_subqueue(netdev, tx_ring->queue_index); 3072 ++adapter->restart_queue; 3073 return 0; 3074 } 3075 3076 static int ixgbevf_maybe_stop_tx(struct net_device *netdev, 3077 struct ixgbevf_ring *tx_ring, int size) 3078 { 3079 if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size)) 3080 return 0; 3081 return __ixgbevf_maybe_stop_tx(netdev, tx_ring, size); 3082 } 3083 3084 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 3085 { 3086 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3087 struct ixgbevf_ring *tx_ring; 3088 unsigned int first; 3089 unsigned int tx_flags = 0; 3090 u8 hdr_len = 0; 3091 int r_idx = 0, tso; 3092 int count = 0; 3093 3094 unsigned int f; 3095 3096 tx_ring = &adapter->tx_ring[r_idx]; 3097 3098 if (vlan_tx_tag_present(skb)) { 3099 tx_flags |= vlan_tx_tag_get(skb); 3100 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 3101 tx_flags |= IXGBE_TX_FLAGS_VLAN; 3102 } 3103 3104 /* four things can cause us to need a context descriptor */ 3105 if (skb_is_gso(skb) || 3106 (skb->ip_summed == CHECKSUM_PARTIAL) || 3107 (tx_flags & IXGBE_TX_FLAGS_VLAN)) 3108 count++; 3109 3110 count += TXD_USE_COUNT(skb_headlen(skb)); 3111 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 3112 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f])); 3113 3114 if (ixgbevf_maybe_stop_tx(netdev, tx_ring, count)) { 3115 adapter->tx_busy++; 3116 return NETDEV_TX_BUSY; 3117 } 3118 3119 first = tx_ring->next_to_use; 3120 3121 if (skb->protocol == htons(ETH_P_IP)) 3122 tx_flags |= IXGBE_TX_FLAGS_IPV4; 3123 tso = ixgbevf_tso(adapter, tx_ring, skb, tx_flags, &hdr_len); 3124 if (tso < 0) { 3125 dev_kfree_skb_any(skb); 3126 return NETDEV_TX_OK; 3127 } 3128 3129 if (tso) 3130 tx_flags |= IXGBE_TX_FLAGS_TSO; 3131 else if (ixgbevf_tx_csum(adapter, tx_ring, skb, tx_flags) && 3132 (skb->ip_summed == CHECKSUM_PARTIAL)) 3133 tx_flags |= IXGBE_TX_FLAGS_CSUM; 3134 3135 ixgbevf_tx_queue(adapter, tx_ring, tx_flags, 3136 ixgbevf_tx_map(adapter, tx_ring, skb, tx_flags, first), 3137 skb->len, hdr_len); 3138 3139 ixgbevf_maybe_stop_tx(netdev, tx_ring, DESC_NEEDED); 3140 3141 return NETDEV_TX_OK; 3142 } 3143 3144 /** 3145 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 3146 * @netdev: network interface device structure 3147 * @p: pointer to an address structure 3148 * 3149 * Returns 0 on success, negative on failure 3150 **/ 3151 static int ixgbevf_set_mac(struct net_device *netdev, void *p) 3152 { 3153 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3154 struct ixgbe_hw *hw = &adapter->hw; 3155 struct sockaddr *addr = p; 3156 3157 if (!is_valid_ether_addr(addr->sa_data)) 3158 return -EADDRNOTAVAIL; 3159 3160 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 3161 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 3162 3163 if (hw->mac.ops.set_rar) 3164 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 3165 3166 return 0; 3167 } 3168 3169 /** 3170 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 3171 * @netdev: network interface device structure 3172 * @new_mtu: new value for maximum frame size 3173 * 3174 * Returns 0 on success, negative on failure 3175 **/ 3176 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 3177 { 3178 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3179 struct ixgbe_hw *hw = &adapter->hw; 3180 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 3181 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; 3182 u32 msg[2]; 3183 3184 if (adapter->hw.mac.type == ixgbe_mac_X540_vf) 3185 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3186 3187 /* MTU < 68 is an error and causes problems on some kernels */ 3188 if ((new_mtu < 68) || (max_frame > max_possible_frame)) 3189 return -EINVAL; 3190 3191 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n", 3192 netdev->mtu, new_mtu); 3193 /* must set new MTU before calling down or up */ 3194 netdev->mtu = new_mtu; 3195 3196 msg[0] = IXGBE_VF_SET_LPE; 3197 msg[1] = max_frame; 3198 hw->mbx.ops.write_posted(hw, msg, 2); 3199 3200 if (netif_running(netdev)) 3201 ixgbevf_reinit_locked(adapter); 3202 3203 return 0; 3204 } 3205 3206 static void ixgbevf_shutdown(struct pci_dev *pdev) 3207 { 3208 struct net_device *netdev = pci_get_drvdata(pdev); 3209 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3210 3211 netif_device_detach(netdev); 3212 3213 if (netif_running(netdev)) { 3214 ixgbevf_down(adapter); 3215 ixgbevf_free_irq(adapter); 3216 ixgbevf_free_all_tx_resources(adapter); 3217 ixgbevf_free_all_rx_resources(adapter); 3218 } 3219 3220 #ifdef CONFIG_PM 3221 pci_save_state(pdev); 3222 #endif 3223 3224 pci_disable_device(pdev); 3225 } 3226 3227 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, 3228 struct rtnl_link_stats64 *stats) 3229 { 3230 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3231 unsigned int start; 3232 u64 bytes, packets; 3233 const struct ixgbevf_ring *ring; 3234 int i; 3235 3236 ixgbevf_update_stats(adapter); 3237 3238 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3239 3240 for (i = 0; i < adapter->num_rx_queues; i++) { 3241 ring = &adapter->rx_ring[i]; 3242 do { 3243 start = u64_stats_fetch_begin_bh(&ring->syncp); 3244 bytes = ring->total_bytes; 3245 packets = ring->total_packets; 3246 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 3247 stats->rx_bytes += bytes; 3248 stats->rx_packets += packets; 3249 } 3250 3251 for (i = 0; i < adapter->num_tx_queues; i++) { 3252 ring = &adapter->tx_ring[i]; 3253 do { 3254 start = u64_stats_fetch_begin_bh(&ring->syncp); 3255 bytes = ring->total_bytes; 3256 packets = ring->total_packets; 3257 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 3258 stats->tx_bytes += bytes; 3259 stats->tx_packets += packets; 3260 } 3261 3262 return stats; 3263 } 3264 3265 static int ixgbevf_set_features(struct net_device *netdev, 3266 netdev_features_t features) 3267 { 3268 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3269 3270 if (features & NETIF_F_RXCSUM) 3271 adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED; 3272 else 3273 adapter->flags &= ~IXGBE_FLAG_RX_CSUM_ENABLED; 3274 3275 return 0; 3276 } 3277 3278 static const struct net_device_ops ixgbe_netdev_ops = { 3279 .ndo_open = ixgbevf_open, 3280 .ndo_stop = ixgbevf_close, 3281 .ndo_start_xmit = ixgbevf_xmit_frame, 3282 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 3283 .ndo_get_stats64 = ixgbevf_get_stats, 3284 .ndo_validate_addr = eth_validate_addr, 3285 .ndo_set_mac_address = ixgbevf_set_mac, 3286 .ndo_change_mtu = ixgbevf_change_mtu, 3287 .ndo_tx_timeout = ixgbevf_tx_timeout, 3288 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 3289 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 3290 .ndo_set_features = ixgbevf_set_features, 3291 }; 3292 3293 static void ixgbevf_assign_netdev_ops(struct net_device *dev) 3294 { 3295 dev->netdev_ops = &ixgbe_netdev_ops; 3296 ixgbevf_set_ethtool_ops(dev); 3297 dev->watchdog_timeo = 5 * HZ; 3298 } 3299 3300 /** 3301 * ixgbevf_probe - Device Initialization Routine 3302 * @pdev: PCI device information struct 3303 * @ent: entry in ixgbevf_pci_tbl 3304 * 3305 * Returns 0 on success, negative on failure 3306 * 3307 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 3308 * The OS initialization, configuring of the adapter private structure, 3309 * and a hardware reset occur. 3310 **/ 3311 static int __devinit ixgbevf_probe(struct pci_dev *pdev, 3312 const struct pci_device_id *ent) 3313 { 3314 struct net_device *netdev; 3315 struct ixgbevf_adapter *adapter = NULL; 3316 struct ixgbe_hw *hw = NULL; 3317 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 3318 static int cards_found; 3319 int err, pci_using_dac; 3320 3321 err = pci_enable_device(pdev); 3322 if (err) 3323 return err; 3324 3325 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && 3326 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { 3327 pci_using_dac = 1; 3328 } else { 3329 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 3330 if (err) { 3331 err = dma_set_coherent_mask(&pdev->dev, 3332 DMA_BIT_MASK(32)); 3333 if (err) { 3334 dev_err(&pdev->dev, "No usable DMA " 3335 "configuration, aborting\n"); 3336 goto err_dma; 3337 } 3338 } 3339 pci_using_dac = 0; 3340 } 3341 3342 err = pci_request_regions(pdev, ixgbevf_driver_name); 3343 if (err) { 3344 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 3345 goto err_pci_reg; 3346 } 3347 3348 pci_set_master(pdev); 3349 3350 #ifdef HAVE_TX_MQ 3351 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 3352 MAX_TX_QUEUES); 3353 #else 3354 netdev = alloc_etherdev(sizeof(struct ixgbevf_adapter)); 3355 #endif 3356 if (!netdev) { 3357 err = -ENOMEM; 3358 goto err_alloc_etherdev; 3359 } 3360 3361 SET_NETDEV_DEV(netdev, &pdev->dev); 3362 3363 pci_set_drvdata(pdev, netdev); 3364 adapter = netdev_priv(netdev); 3365 3366 adapter->netdev = netdev; 3367 adapter->pdev = pdev; 3368 hw = &adapter->hw; 3369 hw->back = adapter; 3370 adapter->msg_enable = (1 << DEFAULT_DEBUG_LEVEL_SHIFT) - 1; 3371 3372 /* 3373 * call save state here in standalone driver because it relies on 3374 * adapter struct to exist, and needs to call netdev_priv 3375 */ 3376 pci_save_state(pdev); 3377 3378 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3379 pci_resource_len(pdev, 0)); 3380 if (!hw->hw_addr) { 3381 err = -EIO; 3382 goto err_ioremap; 3383 } 3384 3385 ixgbevf_assign_netdev_ops(netdev); 3386 3387 adapter->bd_number = cards_found; 3388 3389 /* Setup hw api */ 3390 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 3391 hw->mac.type = ii->mac; 3392 3393 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 3394 sizeof(struct ixgbe_mbx_operations)); 3395 3396 adapter->flags &= ~IXGBE_FLAG_RX_PS_CAPABLE; 3397 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED; 3398 adapter->flags |= IXGBE_FLAG_RX_1BUF_CAPABLE; 3399 3400 /* setup the private structure */ 3401 err = ixgbevf_sw_init(adapter); 3402 if (err) 3403 goto err_sw_init; 3404 3405 /* The HW MAC address was set and/or determined in sw_init */ 3406 memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len); 3407 3408 if (!is_valid_ether_addr(netdev->dev_addr)) { 3409 pr_err("invalid MAC address\n"); 3410 err = -EIO; 3411 goto err_sw_init; 3412 } 3413 3414 netdev->hw_features = NETIF_F_SG | 3415 NETIF_F_IP_CSUM | 3416 NETIF_F_IPV6_CSUM | 3417 NETIF_F_TSO | 3418 NETIF_F_TSO6 | 3419 NETIF_F_RXCSUM; 3420 3421 netdev->features = netdev->hw_features | 3422 NETIF_F_HW_VLAN_TX | 3423 NETIF_F_HW_VLAN_RX | 3424 NETIF_F_HW_VLAN_FILTER; 3425 3426 netdev->vlan_features |= NETIF_F_TSO; 3427 netdev->vlan_features |= NETIF_F_TSO6; 3428 netdev->vlan_features |= NETIF_F_IP_CSUM; 3429 netdev->vlan_features |= NETIF_F_IPV6_CSUM; 3430 netdev->vlan_features |= NETIF_F_SG; 3431 3432 if (pci_using_dac) 3433 netdev->features |= NETIF_F_HIGHDMA; 3434 3435 netdev->priv_flags |= IFF_UNICAST_FLT; 3436 3437 init_timer(&adapter->watchdog_timer); 3438 adapter->watchdog_timer.function = ixgbevf_watchdog; 3439 adapter->watchdog_timer.data = (unsigned long)adapter; 3440 3441 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task); 3442 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task); 3443 3444 err = ixgbevf_init_interrupt_scheme(adapter); 3445 if (err) 3446 goto err_sw_init; 3447 3448 /* pick up the PCI bus settings for reporting later */ 3449 if (hw->mac.ops.get_bus_info) 3450 hw->mac.ops.get_bus_info(hw); 3451 3452 strcpy(netdev->name, "eth%d"); 3453 3454 err = register_netdev(netdev); 3455 if (err) 3456 goto err_register; 3457 3458 adapter->netdev_registered = true; 3459 3460 netif_carrier_off(netdev); 3461 3462 ixgbevf_init_last_counter_stats(adapter); 3463 3464 /* print the MAC address */ 3465 hw_dbg(hw, "%pM\n", netdev->dev_addr); 3466 3467 hw_dbg(hw, "MAC: %d\n", hw->mac.type); 3468 3469 hw_dbg(hw, "LRO is disabled\n"); 3470 3471 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n"); 3472 cards_found++; 3473 return 0; 3474 3475 err_register: 3476 err_sw_init: 3477 ixgbevf_reset_interrupt_capability(adapter); 3478 iounmap(hw->hw_addr); 3479 err_ioremap: 3480 free_netdev(netdev); 3481 err_alloc_etherdev: 3482 pci_release_regions(pdev); 3483 err_pci_reg: 3484 err_dma: 3485 pci_disable_device(pdev); 3486 return err; 3487 } 3488 3489 /** 3490 * ixgbevf_remove - Device Removal Routine 3491 * @pdev: PCI device information struct 3492 * 3493 * ixgbevf_remove is called by the PCI subsystem to alert the driver 3494 * that it should release a PCI device. The could be caused by a 3495 * Hot-Plug event, or because the driver is going to be removed from 3496 * memory. 3497 **/ 3498 static void __devexit ixgbevf_remove(struct pci_dev *pdev) 3499 { 3500 struct net_device *netdev = pci_get_drvdata(pdev); 3501 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3502 3503 set_bit(__IXGBEVF_DOWN, &adapter->state); 3504 3505 del_timer_sync(&adapter->watchdog_timer); 3506 3507 cancel_work_sync(&adapter->reset_task); 3508 cancel_work_sync(&adapter->watchdog_task); 3509 3510 if (adapter->netdev_registered) { 3511 unregister_netdev(netdev); 3512 adapter->netdev_registered = false; 3513 } 3514 3515 ixgbevf_reset_interrupt_capability(adapter); 3516 3517 iounmap(adapter->hw.hw_addr); 3518 pci_release_regions(pdev); 3519 3520 hw_dbg(&adapter->hw, "Remove complete\n"); 3521 3522 kfree(adapter->tx_ring); 3523 kfree(adapter->rx_ring); 3524 3525 free_netdev(netdev); 3526 3527 pci_disable_device(pdev); 3528 } 3529 3530 static struct pci_driver ixgbevf_driver = { 3531 .name = ixgbevf_driver_name, 3532 .id_table = ixgbevf_pci_tbl, 3533 .probe = ixgbevf_probe, 3534 .remove = __devexit_p(ixgbevf_remove), 3535 .shutdown = ixgbevf_shutdown, 3536 }; 3537 3538 /** 3539 * ixgbevf_init_module - Driver Registration Routine 3540 * 3541 * ixgbevf_init_module is the first routine called when the driver is 3542 * loaded. All it does is register with the PCI subsystem. 3543 **/ 3544 static int __init ixgbevf_init_module(void) 3545 { 3546 int ret; 3547 pr_info("%s - version %s\n", ixgbevf_driver_string, 3548 ixgbevf_driver_version); 3549 3550 pr_info("%s\n", ixgbevf_copyright); 3551 3552 ret = pci_register_driver(&ixgbevf_driver); 3553 return ret; 3554 } 3555 3556 module_init(ixgbevf_init_module); 3557 3558 /** 3559 * ixgbevf_exit_module - Driver Exit Cleanup Routine 3560 * 3561 * ixgbevf_exit_module is called just before the driver is removed 3562 * from memory. 3563 **/ 3564 static void __exit ixgbevf_exit_module(void) 3565 { 3566 pci_unregister_driver(&ixgbevf_driver); 3567 } 3568 3569 #ifdef DEBUG 3570 /** 3571 * ixgbevf_get_hw_dev_name - return device name string 3572 * used by hardware layer to print debugging information 3573 **/ 3574 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 3575 { 3576 struct ixgbevf_adapter *adapter = hw->back; 3577 return adapter->netdev->name; 3578 } 3579 3580 #endif 3581 module_exit(ixgbevf_exit_module); 3582 3583 /* ixgbevf_main.c */ 3584