1 /* Intel Ethernet Switch Host Interface Driver 2 * Copyright(c) 2013 - 2014 Intel Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * The full GNU General Public License is included in this distribution in 14 * the file called "COPYING". 15 * 16 * Contact Information: 17 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 18 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 19 */ 20 21 #include <linux/types.h> 22 #include <linux/module.h> 23 #include <net/ipv6.h> 24 #include <net/ip.h> 25 #include <net/tcp.h> 26 #include <linux/if_macvlan.h> 27 #include <linux/prefetch.h> 28 29 #include "fm10k.h" 30 31 #define DRV_VERSION "0.12.2-k" 32 const char fm10k_driver_version[] = DRV_VERSION; 33 char fm10k_driver_name[] = "fm10k"; 34 static const char fm10k_driver_string[] = 35 "Intel(R) Ethernet Switch Host Interface Driver"; 36 static const char fm10k_copyright[] = 37 "Copyright (c) 2013 Intel Corporation."; 38 39 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 40 MODULE_DESCRIPTION("Intel(R) Ethernet Switch Host Interface Driver"); 41 MODULE_LICENSE("GPL"); 42 MODULE_VERSION(DRV_VERSION); 43 44 /** 45 * fm10k_init_module - Driver Registration Routine 46 * 47 * fm10k_init_module is the first routine called when the driver is 48 * loaded. All it does is register with the PCI subsystem. 49 **/ 50 static int __init fm10k_init_module(void) 51 { 52 pr_info("%s - version %s\n", fm10k_driver_string, fm10k_driver_version); 53 pr_info("%s\n", fm10k_copyright); 54 55 fm10k_dbg_init(); 56 57 return fm10k_register_pci_driver(); 58 } 59 module_init(fm10k_init_module); 60 61 /** 62 * fm10k_exit_module - Driver Exit Cleanup Routine 63 * 64 * fm10k_exit_module is called just before the driver is removed 65 * from memory. 66 **/ 67 static void __exit fm10k_exit_module(void) 68 { 69 fm10k_unregister_pci_driver(); 70 71 fm10k_dbg_exit(); 72 } 73 module_exit(fm10k_exit_module); 74 75 static bool fm10k_alloc_mapped_page(struct fm10k_ring *rx_ring, 76 struct fm10k_rx_buffer *bi) 77 { 78 struct page *page = bi->page; 79 dma_addr_t dma; 80 81 /* Only page will be NULL if buffer was consumed */ 82 if (likely(page)) 83 return true; 84 85 /* alloc new page for storage */ 86 page = dev_alloc_page(); 87 if (unlikely(!page)) { 88 rx_ring->rx_stats.alloc_failed++; 89 return false; 90 } 91 92 /* map page for use */ 93 dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 94 95 /* if mapping failed free memory back to system since 96 * there isn't much point in holding memory we can't use 97 */ 98 if (dma_mapping_error(rx_ring->dev, dma)) { 99 __free_page(page); 100 101 rx_ring->rx_stats.alloc_failed++; 102 return false; 103 } 104 105 bi->dma = dma; 106 bi->page = page; 107 bi->page_offset = 0; 108 109 return true; 110 } 111 112 /** 113 * fm10k_alloc_rx_buffers - Replace used receive buffers 114 * @rx_ring: ring to place buffers on 115 * @cleaned_count: number of buffers to replace 116 **/ 117 void fm10k_alloc_rx_buffers(struct fm10k_ring *rx_ring, u16 cleaned_count) 118 { 119 union fm10k_rx_desc *rx_desc; 120 struct fm10k_rx_buffer *bi; 121 u16 i = rx_ring->next_to_use; 122 123 /* nothing to do */ 124 if (!cleaned_count) 125 return; 126 127 rx_desc = FM10K_RX_DESC(rx_ring, i); 128 bi = &rx_ring->rx_buffer[i]; 129 i -= rx_ring->count; 130 131 do { 132 if (!fm10k_alloc_mapped_page(rx_ring, bi)) 133 break; 134 135 /* Refresh the desc even if buffer_addrs didn't change 136 * because each write-back erases this info. 137 */ 138 rx_desc->q.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 139 140 rx_desc++; 141 bi++; 142 i++; 143 if (unlikely(!i)) { 144 rx_desc = FM10K_RX_DESC(rx_ring, 0); 145 bi = rx_ring->rx_buffer; 146 i -= rx_ring->count; 147 } 148 149 /* clear the status bits for the next_to_use descriptor */ 150 rx_desc->d.staterr = 0; 151 152 cleaned_count--; 153 } while (cleaned_count); 154 155 i += rx_ring->count; 156 157 if (rx_ring->next_to_use != i) { 158 /* record the next descriptor to use */ 159 rx_ring->next_to_use = i; 160 161 /* update next to alloc since we have filled the ring */ 162 rx_ring->next_to_alloc = i; 163 164 /* Force memory writes to complete before letting h/w 165 * know there are new descriptors to fetch. (Only 166 * applicable for weak-ordered memory model archs, 167 * such as IA-64). 168 */ 169 wmb(); 170 171 /* notify hardware of new descriptors */ 172 writel(i, rx_ring->tail); 173 } 174 } 175 176 /** 177 * fm10k_reuse_rx_page - page flip buffer and store it back on the ring 178 * @rx_ring: rx descriptor ring to store buffers on 179 * @old_buff: donor buffer to have page reused 180 * 181 * Synchronizes page for reuse by the interface 182 **/ 183 static void fm10k_reuse_rx_page(struct fm10k_ring *rx_ring, 184 struct fm10k_rx_buffer *old_buff) 185 { 186 struct fm10k_rx_buffer *new_buff; 187 u16 nta = rx_ring->next_to_alloc; 188 189 new_buff = &rx_ring->rx_buffer[nta]; 190 191 /* update, and store next to alloc */ 192 nta++; 193 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 194 195 /* transfer page from old buffer to new buffer */ 196 *new_buff = *old_buff; 197 198 /* sync the buffer for use by the device */ 199 dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma, 200 old_buff->page_offset, 201 FM10K_RX_BUFSZ, 202 DMA_FROM_DEVICE); 203 } 204 205 static inline bool fm10k_page_is_reserved(struct page *page) 206 { 207 return (page_to_nid(page) != numa_mem_id()) || page->pfmemalloc; 208 } 209 210 static bool fm10k_can_reuse_rx_page(struct fm10k_rx_buffer *rx_buffer, 211 struct page *page, 212 unsigned int truesize) 213 { 214 /* avoid re-using remote pages */ 215 if (unlikely(fm10k_page_is_reserved(page))) 216 return false; 217 218 #if (PAGE_SIZE < 8192) 219 /* if we are only owner of page we can reuse it */ 220 if (unlikely(page_count(page) != 1)) 221 return false; 222 223 /* flip page offset to other buffer */ 224 rx_buffer->page_offset ^= FM10K_RX_BUFSZ; 225 #else 226 /* move offset up to the next cache line */ 227 rx_buffer->page_offset += truesize; 228 229 if (rx_buffer->page_offset > (PAGE_SIZE - FM10K_RX_BUFSZ)) 230 return false; 231 #endif 232 233 /* Even if we own the page, we are not allowed to use atomic_set() 234 * This would break get_page_unless_zero() users. 235 */ 236 atomic_inc(&page->_count); 237 238 return true; 239 } 240 241 /** 242 * fm10k_add_rx_frag - Add contents of Rx buffer to sk_buff 243 * @rx_ring: rx descriptor ring to transact packets on 244 * @rx_buffer: buffer containing page to add 245 * @rx_desc: descriptor containing length of buffer written by hardware 246 * @skb: sk_buff to place the data into 247 * 248 * This function will add the data contained in rx_buffer->page to the skb. 249 * This is done either through a direct copy if the data in the buffer is 250 * less than the skb header size, otherwise it will just attach the page as 251 * a frag to the skb. 252 * 253 * The function will then update the page offset if necessary and return 254 * true if the buffer can be reused by the interface. 255 **/ 256 static bool fm10k_add_rx_frag(struct fm10k_ring *rx_ring, 257 struct fm10k_rx_buffer *rx_buffer, 258 union fm10k_rx_desc *rx_desc, 259 struct sk_buff *skb) 260 { 261 struct page *page = rx_buffer->page; 262 unsigned int size = le16_to_cpu(rx_desc->w.length); 263 #if (PAGE_SIZE < 8192) 264 unsigned int truesize = FM10K_RX_BUFSZ; 265 #else 266 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES); 267 #endif 268 269 if ((size <= FM10K_RX_HDR_LEN) && !skb_is_nonlinear(skb)) { 270 unsigned char *va = page_address(page) + rx_buffer->page_offset; 271 272 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 273 274 /* page is not reserved, we can reuse buffer as-is */ 275 if (likely(!fm10k_page_is_reserved(page))) 276 return true; 277 278 /* this page cannot be reused so discard it */ 279 __free_page(page); 280 return false; 281 } 282 283 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 284 rx_buffer->page_offset, size, truesize); 285 286 return fm10k_can_reuse_rx_page(rx_buffer, page, truesize); 287 } 288 289 static struct sk_buff *fm10k_fetch_rx_buffer(struct fm10k_ring *rx_ring, 290 union fm10k_rx_desc *rx_desc, 291 struct sk_buff *skb) 292 { 293 struct fm10k_rx_buffer *rx_buffer; 294 struct page *page; 295 296 rx_buffer = &rx_ring->rx_buffer[rx_ring->next_to_clean]; 297 page = rx_buffer->page; 298 prefetchw(page); 299 300 if (likely(!skb)) { 301 void *page_addr = page_address(page) + 302 rx_buffer->page_offset; 303 304 /* prefetch first cache line of first page */ 305 prefetch(page_addr); 306 #if L1_CACHE_BYTES < 128 307 prefetch(page_addr + L1_CACHE_BYTES); 308 #endif 309 310 /* allocate a skb to store the frags */ 311 skb = napi_alloc_skb(&rx_ring->q_vector->napi, 312 FM10K_RX_HDR_LEN); 313 if (unlikely(!skb)) { 314 rx_ring->rx_stats.alloc_failed++; 315 return NULL; 316 } 317 318 /* we will be copying header into skb->data in 319 * pskb_may_pull so it is in our interest to prefetch 320 * it now to avoid a possible cache miss 321 */ 322 prefetchw(skb->data); 323 } 324 325 /* we are reusing so sync this buffer for CPU use */ 326 dma_sync_single_range_for_cpu(rx_ring->dev, 327 rx_buffer->dma, 328 rx_buffer->page_offset, 329 FM10K_RX_BUFSZ, 330 DMA_FROM_DEVICE); 331 332 /* pull page into skb */ 333 if (fm10k_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) { 334 /* hand second half of page back to the ring */ 335 fm10k_reuse_rx_page(rx_ring, rx_buffer); 336 } else { 337 /* we are not reusing the buffer so unmap it */ 338 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 339 PAGE_SIZE, DMA_FROM_DEVICE); 340 } 341 342 /* clear contents of rx_buffer */ 343 rx_buffer->page = NULL; 344 345 return skb; 346 } 347 348 static inline void fm10k_rx_checksum(struct fm10k_ring *ring, 349 union fm10k_rx_desc *rx_desc, 350 struct sk_buff *skb) 351 { 352 skb_checksum_none_assert(skb); 353 354 /* Rx checksum disabled via ethtool */ 355 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 356 return; 357 358 /* TCP/UDP checksum error bit is set */ 359 if (fm10k_test_staterr(rx_desc, 360 FM10K_RXD_STATUS_L4E | 361 FM10K_RXD_STATUS_L4E2 | 362 FM10K_RXD_STATUS_IPE | 363 FM10K_RXD_STATUS_IPE2)) { 364 ring->rx_stats.csum_err++; 365 return; 366 } 367 368 /* It must be a TCP or UDP packet with a valid checksum */ 369 if (fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_L4CS2)) 370 skb->encapsulation = true; 371 else if (!fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_L4CS)) 372 return; 373 374 skb->ip_summed = CHECKSUM_UNNECESSARY; 375 } 376 377 #define FM10K_RSS_L4_TYPES_MASK \ 378 ((1ul << FM10K_RSSTYPE_IPV4_TCP) | \ 379 (1ul << FM10K_RSSTYPE_IPV4_UDP) | \ 380 (1ul << FM10K_RSSTYPE_IPV6_TCP) | \ 381 (1ul << FM10K_RSSTYPE_IPV6_UDP)) 382 383 static inline void fm10k_rx_hash(struct fm10k_ring *ring, 384 union fm10k_rx_desc *rx_desc, 385 struct sk_buff *skb) 386 { 387 u16 rss_type; 388 389 if (!(ring->netdev->features & NETIF_F_RXHASH)) 390 return; 391 392 rss_type = le16_to_cpu(rx_desc->w.pkt_info) & FM10K_RXD_RSSTYPE_MASK; 393 if (!rss_type) 394 return; 395 396 skb_set_hash(skb, le32_to_cpu(rx_desc->d.rss), 397 (FM10K_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? 398 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); 399 } 400 401 static void fm10k_rx_hwtstamp(struct fm10k_ring *rx_ring, 402 union fm10k_rx_desc *rx_desc, 403 struct sk_buff *skb) 404 { 405 struct fm10k_intfc *interface = rx_ring->q_vector->interface; 406 407 FM10K_CB(skb)->tstamp = rx_desc->q.timestamp; 408 409 if (unlikely(interface->flags & FM10K_FLAG_RX_TS_ENABLED)) 410 fm10k_systime_to_hwtstamp(interface, skb_hwtstamps(skb), 411 le64_to_cpu(rx_desc->q.timestamp)); 412 } 413 414 static void fm10k_type_trans(struct fm10k_ring *rx_ring, 415 union fm10k_rx_desc *rx_desc, 416 struct sk_buff *skb) 417 { 418 struct net_device *dev = rx_ring->netdev; 419 struct fm10k_l2_accel *l2_accel = rcu_dereference_bh(rx_ring->l2_accel); 420 421 /* check to see if DGLORT belongs to a MACVLAN */ 422 if (l2_accel) { 423 u16 idx = le16_to_cpu(FM10K_CB(skb)->fi.w.dglort) - 1; 424 425 idx -= l2_accel->dglort; 426 if (idx < l2_accel->size && l2_accel->macvlan[idx]) 427 dev = l2_accel->macvlan[idx]; 428 else 429 l2_accel = NULL; 430 } 431 432 skb->protocol = eth_type_trans(skb, dev); 433 434 if (!l2_accel) 435 return; 436 437 /* update MACVLAN statistics */ 438 macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, 1, 439 !!(rx_desc->w.hdr_info & 440 cpu_to_le16(FM10K_RXD_HDR_INFO_XC_MASK))); 441 } 442 443 /** 444 * fm10k_process_skb_fields - Populate skb header fields from Rx descriptor 445 * @rx_ring: rx descriptor ring packet is being transacted on 446 * @rx_desc: pointer to the EOP Rx descriptor 447 * @skb: pointer to current skb being populated 448 * 449 * This function checks the ring, descriptor, and packet information in 450 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 451 * other fields within the skb. 452 **/ 453 static unsigned int fm10k_process_skb_fields(struct fm10k_ring *rx_ring, 454 union fm10k_rx_desc *rx_desc, 455 struct sk_buff *skb) 456 { 457 unsigned int len = skb->len; 458 459 fm10k_rx_hash(rx_ring, rx_desc, skb); 460 461 fm10k_rx_checksum(rx_ring, rx_desc, skb); 462 463 fm10k_rx_hwtstamp(rx_ring, rx_desc, skb); 464 465 FM10K_CB(skb)->fi.w.vlan = rx_desc->w.vlan; 466 467 skb_record_rx_queue(skb, rx_ring->queue_index); 468 469 FM10K_CB(skb)->fi.d.glort = rx_desc->d.glort; 470 471 if (rx_desc->w.vlan) { 472 u16 vid = le16_to_cpu(rx_desc->w.vlan); 473 474 if (vid != rx_ring->vid) 475 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 476 } 477 478 fm10k_type_trans(rx_ring, rx_desc, skb); 479 480 return len; 481 } 482 483 /** 484 * fm10k_is_non_eop - process handling of non-EOP buffers 485 * @rx_ring: Rx ring being processed 486 * @rx_desc: Rx descriptor for current buffer 487 * 488 * This function updates next to clean. If the buffer is an EOP buffer 489 * this function exits returning false, otherwise it will place the 490 * sk_buff in the next buffer to be chained and return true indicating 491 * that this is in fact a non-EOP buffer. 492 **/ 493 static bool fm10k_is_non_eop(struct fm10k_ring *rx_ring, 494 union fm10k_rx_desc *rx_desc) 495 { 496 u32 ntc = rx_ring->next_to_clean + 1; 497 498 /* fetch, update, and store next to clean */ 499 ntc = (ntc < rx_ring->count) ? ntc : 0; 500 rx_ring->next_to_clean = ntc; 501 502 prefetch(FM10K_RX_DESC(rx_ring, ntc)); 503 504 if (likely(fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_EOP))) 505 return false; 506 507 return true; 508 } 509 510 /** 511 * fm10k_pull_tail - fm10k specific version of skb_pull_tail 512 * @rx_ring: rx descriptor ring packet is being transacted on 513 * @rx_desc: pointer to the EOP Rx descriptor 514 * @skb: pointer to current skb being adjusted 515 * 516 * This function is an fm10k specific version of __pskb_pull_tail. The 517 * main difference between this version and the original function is that 518 * this function can make several assumptions about the state of things 519 * that allow for significant optimizations versus the standard function. 520 * As a result we can do things like drop a frag and maintain an accurate 521 * truesize for the skb. 522 */ 523 static void fm10k_pull_tail(struct fm10k_ring *rx_ring, 524 union fm10k_rx_desc *rx_desc, 525 struct sk_buff *skb) 526 { 527 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 528 unsigned char *va; 529 unsigned int pull_len; 530 531 /* it is valid to use page_address instead of kmap since we are 532 * working with pages allocated out of the lomem pool per 533 * alloc_page(GFP_ATOMIC) 534 */ 535 va = skb_frag_address(frag); 536 537 /* we need the header to contain the greater of either ETH_HLEN or 538 * 60 bytes if the skb->len is less than 60 for skb_pad. 539 */ 540 pull_len = eth_get_headlen(va, FM10K_RX_HDR_LEN); 541 542 /* align pull length to size of long to optimize memcpy performance */ 543 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); 544 545 /* update all of the pointers */ 546 skb_frag_size_sub(frag, pull_len); 547 frag->page_offset += pull_len; 548 skb->data_len -= pull_len; 549 skb->tail += pull_len; 550 } 551 552 /** 553 * fm10k_cleanup_headers - Correct corrupted or empty headers 554 * @rx_ring: rx descriptor ring packet is being transacted on 555 * @rx_desc: pointer to the EOP Rx descriptor 556 * @skb: pointer to current skb being fixed 557 * 558 * Address the case where we are pulling data in on pages only 559 * and as such no data is present in the skb header. 560 * 561 * In addition if skb is not at least 60 bytes we need to pad it so that 562 * it is large enough to qualify as a valid Ethernet frame. 563 * 564 * Returns true if an error was encountered and skb was freed. 565 **/ 566 static bool fm10k_cleanup_headers(struct fm10k_ring *rx_ring, 567 union fm10k_rx_desc *rx_desc, 568 struct sk_buff *skb) 569 { 570 if (unlikely((fm10k_test_staterr(rx_desc, 571 FM10K_RXD_STATUS_RXE)))) { 572 dev_kfree_skb_any(skb); 573 rx_ring->rx_stats.errors++; 574 return true; 575 } 576 577 /* place header in linear portion of buffer */ 578 if (skb_is_nonlinear(skb)) 579 fm10k_pull_tail(rx_ring, rx_desc, skb); 580 581 /* if eth_skb_pad returns an error the skb was freed */ 582 if (eth_skb_pad(skb)) 583 return true; 584 585 return false; 586 } 587 588 /** 589 * fm10k_receive_skb - helper function to handle rx indications 590 * @q_vector: structure containing interrupt and ring information 591 * @skb: packet to send up 592 **/ 593 static void fm10k_receive_skb(struct fm10k_q_vector *q_vector, 594 struct sk_buff *skb) 595 { 596 napi_gro_receive(&q_vector->napi, skb); 597 } 598 599 static bool fm10k_clean_rx_irq(struct fm10k_q_vector *q_vector, 600 struct fm10k_ring *rx_ring, 601 int budget) 602 { 603 struct sk_buff *skb = rx_ring->skb; 604 unsigned int total_bytes = 0, total_packets = 0; 605 u16 cleaned_count = fm10k_desc_unused(rx_ring); 606 607 do { 608 union fm10k_rx_desc *rx_desc; 609 610 /* return some buffers to hardware, one at a time is too slow */ 611 if (cleaned_count >= FM10K_RX_BUFFER_WRITE) { 612 fm10k_alloc_rx_buffers(rx_ring, cleaned_count); 613 cleaned_count = 0; 614 } 615 616 rx_desc = FM10K_RX_DESC(rx_ring, rx_ring->next_to_clean); 617 618 if (!rx_desc->d.staterr) 619 break; 620 621 /* This memory barrier is needed to keep us from reading 622 * any other fields out of the rx_desc until we know the 623 * descriptor has been written back 624 */ 625 dma_rmb(); 626 627 /* retrieve a buffer from the ring */ 628 skb = fm10k_fetch_rx_buffer(rx_ring, rx_desc, skb); 629 630 /* exit if we failed to retrieve a buffer */ 631 if (!skb) 632 break; 633 634 cleaned_count++; 635 636 /* fetch next buffer in frame if non-eop */ 637 if (fm10k_is_non_eop(rx_ring, rx_desc)) 638 continue; 639 640 /* verify the packet layout is correct */ 641 if (fm10k_cleanup_headers(rx_ring, rx_desc, skb)) { 642 skb = NULL; 643 continue; 644 } 645 646 /* populate checksum, timestamp, VLAN, and protocol */ 647 total_bytes += fm10k_process_skb_fields(rx_ring, rx_desc, skb); 648 649 fm10k_receive_skb(q_vector, skb); 650 651 /* reset skb pointer */ 652 skb = NULL; 653 654 /* update budget accounting */ 655 total_packets++; 656 } while (likely(total_packets < budget)); 657 658 /* place incomplete frames back on ring for completion */ 659 rx_ring->skb = skb; 660 661 u64_stats_update_begin(&rx_ring->syncp); 662 rx_ring->stats.packets += total_packets; 663 rx_ring->stats.bytes += total_bytes; 664 u64_stats_update_end(&rx_ring->syncp); 665 q_vector->rx.total_packets += total_packets; 666 q_vector->rx.total_bytes += total_bytes; 667 668 return total_packets < budget; 669 } 670 671 #define VXLAN_HLEN (sizeof(struct udphdr) + 8) 672 static struct ethhdr *fm10k_port_is_vxlan(struct sk_buff *skb) 673 { 674 struct fm10k_intfc *interface = netdev_priv(skb->dev); 675 struct fm10k_vxlan_port *vxlan_port; 676 677 /* we can only offload a vxlan if we recognize it as such */ 678 vxlan_port = list_first_entry_or_null(&interface->vxlan_port, 679 struct fm10k_vxlan_port, list); 680 681 if (!vxlan_port) 682 return NULL; 683 if (vxlan_port->port != udp_hdr(skb)->dest) 684 return NULL; 685 686 /* return offset of udp_hdr plus 8 bytes for VXLAN header */ 687 return (struct ethhdr *)(skb_transport_header(skb) + VXLAN_HLEN); 688 } 689 690 #define FM10K_NVGRE_RESERVED0_FLAGS htons(0x9FFF) 691 #define NVGRE_TNI htons(0x2000) 692 struct fm10k_nvgre_hdr { 693 __be16 flags; 694 __be16 proto; 695 __be32 tni; 696 }; 697 698 static struct ethhdr *fm10k_gre_is_nvgre(struct sk_buff *skb) 699 { 700 struct fm10k_nvgre_hdr *nvgre_hdr; 701 int hlen = ip_hdrlen(skb); 702 703 /* currently only IPv4 is supported due to hlen above */ 704 if (vlan_get_protocol(skb) != htons(ETH_P_IP)) 705 return NULL; 706 707 /* our transport header should be NVGRE */ 708 nvgre_hdr = (struct fm10k_nvgre_hdr *)(skb_network_header(skb) + hlen); 709 710 /* verify all reserved flags are 0 */ 711 if (nvgre_hdr->flags & FM10K_NVGRE_RESERVED0_FLAGS) 712 return NULL; 713 714 /* verify protocol is transparent Ethernet bridging */ 715 if (nvgre_hdr->proto != htons(ETH_P_TEB)) 716 return NULL; 717 718 /* report start of ethernet header */ 719 if (nvgre_hdr->flags & NVGRE_TNI) 720 return (struct ethhdr *)(nvgre_hdr + 1); 721 722 return (struct ethhdr *)(&nvgre_hdr->tni); 723 } 724 725 static __be16 fm10k_tx_encap_offload(struct sk_buff *skb) 726 { 727 struct ethhdr *eth_hdr; 728 u8 l4_hdr = 0; 729 730 /* fm10k supports 184 octets of outer+inner headers. Minus 20 for inner L4. */ 731 #define FM10K_MAX_ENCAP_TRANSPORT_OFFSET 164 732 if (skb_inner_transport_header(skb) - skb_mac_header(skb) > 733 FM10K_MAX_ENCAP_TRANSPORT_OFFSET) 734 return 0; 735 736 switch (vlan_get_protocol(skb)) { 737 case htons(ETH_P_IP): 738 l4_hdr = ip_hdr(skb)->protocol; 739 break; 740 case htons(ETH_P_IPV6): 741 l4_hdr = ipv6_hdr(skb)->nexthdr; 742 break; 743 default: 744 return 0; 745 } 746 747 switch (l4_hdr) { 748 case IPPROTO_UDP: 749 eth_hdr = fm10k_port_is_vxlan(skb); 750 break; 751 case IPPROTO_GRE: 752 eth_hdr = fm10k_gre_is_nvgre(skb); 753 break; 754 default: 755 return 0; 756 } 757 758 if (!eth_hdr) 759 return 0; 760 761 switch (eth_hdr->h_proto) { 762 case htons(ETH_P_IP): 763 case htons(ETH_P_IPV6): 764 break; 765 default: 766 return 0; 767 } 768 769 return eth_hdr->h_proto; 770 } 771 772 static int fm10k_tso(struct fm10k_ring *tx_ring, 773 struct fm10k_tx_buffer *first) 774 { 775 struct sk_buff *skb = first->skb; 776 struct fm10k_tx_desc *tx_desc; 777 unsigned char *th; 778 u8 hdrlen; 779 780 if (skb->ip_summed != CHECKSUM_PARTIAL) 781 return 0; 782 783 if (!skb_is_gso(skb)) 784 return 0; 785 786 /* compute header lengths */ 787 if (skb->encapsulation) { 788 if (!fm10k_tx_encap_offload(skb)) 789 goto err_vxlan; 790 th = skb_inner_transport_header(skb); 791 } else { 792 th = skb_transport_header(skb); 793 } 794 795 /* compute offset from SOF to transport header and add header len */ 796 hdrlen = (th - skb->data) + (((struct tcphdr *)th)->doff << 2); 797 798 first->tx_flags |= FM10K_TX_FLAGS_CSUM; 799 800 /* update gso size and bytecount with header size */ 801 first->gso_segs = skb_shinfo(skb)->gso_segs; 802 first->bytecount += (first->gso_segs - 1) * hdrlen; 803 804 /* populate Tx descriptor header size and mss */ 805 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use); 806 tx_desc->hdrlen = hdrlen; 807 tx_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size); 808 809 return 1; 810 err_vxlan: 811 tx_ring->netdev->features &= ~NETIF_F_GSO_UDP_TUNNEL; 812 if (!net_ratelimit()) 813 netdev_err(tx_ring->netdev, 814 "TSO requested for unsupported tunnel, disabling offload\n"); 815 return -1; 816 } 817 818 static void fm10k_tx_csum(struct fm10k_ring *tx_ring, 819 struct fm10k_tx_buffer *first) 820 { 821 struct sk_buff *skb = first->skb; 822 struct fm10k_tx_desc *tx_desc; 823 union { 824 struct iphdr *ipv4; 825 struct ipv6hdr *ipv6; 826 u8 *raw; 827 } network_hdr; 828 __be16 protocol; 829 u8 l4_hdr = 0; 830 831 if (skb->ip_summed != CHECKSUM_PARTIAL) 832 goto no_csum; 833 834 if (skb->encapsulation) { 835 protocol = fm10k_tx_encap_offload(skb); 836 if (!protocol) { 837 if (skb_checksum_help(skb)) { 838 dev_warn(tx_ring->dev, 839 "failed to offload encap csum!\n"); 840 tx_ring->tx_stats.csum_err++; 841 } 842 goto no_csum; 843 } 844 network_hdr.raw = skb_inner_network_header(skb); 845 } else { 846 protocol = vlan_get_protocol(skb); 847 network_hdr.raw = skb_network_header(skb); 848 } 849 850 switch (protocol) { 851 case htons(ETH_P_IP): 852 l4_hdr = network_hdr.ipv4->protocol; 853 break; 854 case htons(ETH_P_IPV6): 855 l4_hdr = network_hdr.ipv6->nexthdr; 856 break; 857 default: 858 if (unlikely(net_ratelimit())) { 859 dev_warn(tx_ring->dev, 860 "partial checksum but ip version=%x!\n", 861 protocol); 862 } 863 tx_ring->tx_stats.csum_err++; 864 goto no_csum; 865 } 866 867 switch (l4_hdr) { 868 case IPPROTO_TCP: 869 case IPPROTO_UDP: 870 break; 871 case IPPROTO_GRE: 872 if (skb->encapsulation) 873 break; 874 default: 875 if (unlikely(net_ratelimit())) { 876 dev_warn(tx_ring->dev, 877 "partial checksum but l4 proto=%x!\n", 878 l4_hdr); 879 } 880 tx_ring->tx_stats.csum_err++; 881 goto no_csum; 882 } 883 884 /* update TX checksum flag */ 885 first->tx_flags |= FM10K_TX_FLAGS_CSUM; 886 887 no_csum: 888 /* populate Tx descriptor header size and mss */ 889 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use); 890 tx_desc->hdrlen = 0; 891 tx_desc->mss = 0; 892 } 893 894 #define FM10K_SET_FLAG(_input, _flag, _result) \ 895 ((_flag <= _result) ? \ 896 ((u32)(_input & _flag) * (_result / _flag)) : \ 897 ((u32)(_input & _flag) / (_flag / _result))) 898 899 static u8 fm10k_tx_desc_flags(struct sk_buff *skb, u32 tx_flags) 900 { 901 /* set type for advanced descriptor with frame checksum insertion */ 902 u32 desc_flags = 0; 903 904 /* set timestamping bits */ 905 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 906 likely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) 907 desc_flags |= FM10K_TXD_FLAG_TIME; 908 909 /* set checksum offload bits */ 910 desc_flags |= FM10K_SET_FLAG(tx_flags, FM10K_TX_FLAGS_CSUM, 911 FM10K_TXD_FLAG_CSUM); 912 913 return desc_flags; 914 } 915 916 static bool fm10k_tx_desc_push(struct fm10k_ring *tx_ring, 917 struct fm10k_tx_desc *tx_desc, u16 i, 918 dma_addr_t dma, unsigned int size, u8 desc_flags) 919 { 920 /* set RS and INT for last frame in a cache line */ 921 if ((++i & (FM10K_TXD_WB_FIFO_SIZE - 1)) == 0) 922 desc_flags |= FM10K_TXD_FLAG_RS | FM10K_TXD_FLAG_INT; 923 924 /* record values to descriptor */ 925 tx_desc->buffer_addr = cpu_to_le64(dma); 926 tx_desc->flags = desc_flags; 927 tx_desc->buflen = cpu_to_le16(size); 928 929 /* return true if we just wrapped the ring */ 930 return i == tx_ring->count; 931 } 932 933 static int __fm10k_maybe_stop_tx(struct fm10k_ring *tx_ring, u16 size) 934 { 935 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 936 937 smp_mb(); 938 939 /* We need to check again in a case another CPU has just 940 * made room available. */ 941 if (likely(fm10k_desc_unused(tx_ring) < size)) 942 return -EBUSY; 943 944 /* A reprieve! - use start_queue because it doesn't call schedule */ 945 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 946 ++tx_ring->tx_stats.restart_queue; 947 return 0; 948 } 949 950 static inline int fm10k_maybe_stop_tx(struct fm10k_ring *tx_ring, u16 size) 951 { 952 if (likely(fm10k_desc_unused(tx_ring) >= size)) 953 return 0; 954 return __fm10k_maybe_stop_tx(tx_ring, size); 955 } 956 957 static void fm10k_tx_map(struct fm10k_ring *tx_ring, 958 struct fm10k_tx_buffer *first) 959 { 960 struct sk_buff *skb = first->skb; 961 struct fm10k_tx_buffer *tx_buffer; 962 struct fm10k_tx_desc *tx_desc; 963 struct skb_frag_struct *frag; 964 unsigned char *data; 965 dma_addr_t dma; 966 unsigned int data_len, size; 967 u32 tx_flags = first->tx_flags; 968 u16 i = tx_ring->next_to_use; 969 u8 flags = fm10k_tx_desc_flags(skb, tx_flags); 970 971 tx_desc = FM10K_TX_DESC(tx_ring, i); 972 973 /* add HW VLAN tag */ 974 if (skb_vlan_tag_present(skb)) 975 tx_desc->vlan = cpu_to_le16(skb_vlan_tag_get(skb)); 976 else 977 tx_desc->vlan = 0; 978 979 size = skb_headlen(skb); 980 data = skb->data; 981 982 dma = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 983 984 data_len = skb->data_len; 985 tx_buffer = first; 986 987 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 988 if (dma_mapping_error(tx_ring->dev, dma)) 989 goto dma_error; 990 991 /* record length, and DMA address */ 992 dma_unmap_len_set(tx_buffer, len, size); 993 dma_unmap_addr_set(tx_buffer, dma, dma); 994 995 while (unlikely(size > FM10K_MAX_DATA_PER_TXD)) { 996 if (fm10k_tx_desc_push(tx_ring, tx_desc++, i++, dma, 997 FM10K_MAX_DATA_PER_TXD, flags)) { 998 tx_desc = FM10K_TX_DESC(tx_ring, 0); 999 i = 0; 1000 } 1001 1002 dma += FM10K_MAX_DATA_PER_TXD; 1003 size -= FM10K_MAX_DATA_PER_TXD; 1004 } 1005 1006 if (likely(!data_len)) 1007 break; 1008 1009 if (fm10k_tx_desc_push(tx_ring, tx_desc++, i++, 1010 dma, size, flags)) { 1011 tx_desc = FM10K_TX_DESC(tx_ring, 0); 1012 i = 0; 1013 } 1014 1015 size = skb_frag_size(frag); 1016 data_len -= size; 1017 1018 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 1019 DMA_TO_DEVICE); 1020 1021 tx_buffer = &tx_ring->tx_buffer[i]; 1022 } 1023 1024 /* write last descriptor with LAST bit set */ 1025 flags |= FM10K_TXD_FLAG_LAST; 1026 1027 if (fm10k_tx_desc_push(tx_ring, tx_desc, i++, dma, size, flags)) 1028 i = 0; 1029 1030 /* record bytecount for BQL */ 1031 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1032 1033 /* record SW timestamp if HW timestamp is not available */ 1034 skb_tx_timestamp(first->skb); 1035 1036 /* Force memory writes to complete before letting h/w know there 1037 * are new descriptors to fetch. (Only applicable for weak-ordered 1038 * memory model archs, such as IA-64). 1039 * 1040 * We also need this memory barrier to make certain all of the 1041 * status bits have been updated before next_to_watch is written. 1042 */ 1043 wmb(); 1044 1045 /* set next_to_watch value indicating a packet is present */ 1046 first->next_to_watch = tx_desc; 1047 1048 tx_ring->next_to_use = i; 1049 1050 /* Make sure there is space in the ring for the next send. */ 1051 fm10k_maybe_stop_tx(tx_ring, DESC_NEEDED); 1052 1053 /* notify HW of packet */ 1054 if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) { 1055 writel(i, tx_ring->tail); 1056 1057 /* we need this if more than one processor can write to our tail 1058 * at a time, it synchronizes IO on IA64/Altix systems 1059 */ 1060 mmiowb(); 1061 } 1062 1063 return; 1064 dma_error: 1065 dev_err(tx_ring->dev, "TX DMA map failed\n"); 1066 1067 /* clear dma mappings for failed tx_buffer map */ 1068 for (;;) { 1069 tx_buffer = &tx_ring->tx_buffer[i]; 1070 fm10k_unmap_and_free_tx_resource(tx_ring, tx_buffer); 1071 if (tx_buffer == first) 1072 break; 1073 if (i == 0) 1074 i = tx_ring->count; 1075 i--; 1076 } 1077 1078 tx_ring->next_to_use = i; 1079 } 1080 1081 netdev_tx_t fm10k_xmit_frame_ring(struct sk_buff *skb, 1082 struct fm10k_ring *tx_ring) 1083 { 1084 struct fm10k_tx_buffer *first; 1085 int tso; 1086 u32 tx_flags = 0; 1087 #if PAGE_SIZE > FM10K_MAX_DATA_PER_TXD 1088 unsigned short f; 1089 #endif 1090 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1091 1092 /* need: 1 descriptor per page * PAGE_SIZE/FM10K_MAX_DATA_PER_TXD, 1093 * + 1 desc for skb_headlen/FM10K_MAX_DATA_PER_TXD, 1094 * + 2 desc gap to keep tail from touching head 1095 * otherwise try next time 1096 */ 1097 #if PAGE_SIZE > FM10K_MAX_DATA_PER_TXD 1098 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1099 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 1100 #else 1101 count += skb_shinfo(skb)->nr_frags; 1102 #endif 1103 if (fm10k_maybe_stop_tx(tx_ring, count + 3)) { 1104 tx_ring->tx_stats.tx_busy++; 1105 return NETDEV_TX_BUSY; 1106 } 1107 1108 /* record the location of the first descriptor for this packet */ 1109 first = &tx_ring->tx_buffer[tx_ring->next_to_use]; 1110 first->skb = skb; 1111 first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); 1112 first->gso_segs = 1; 1113 1114 /* record initial flags and protocol */ 1115 first->tx_flags = tx_flags; 1116 1117 tso = fm10k_tso(tx_ring, first); 1118 if (tso < 0) 1119 goto out_drop; 1120 else if (!tso) 1121 fm10k_tx_csum(tx_ring, first); 1122 1123 fm10k_tx_map(tx_ring, first); 1124 1125 return NETDEV_TX_OK; 1126 1127 out_drop: 1128 dev_kfree_skb_any(first->skb); 1129 first->skb = NULL; 1130 1131 return NETDEV_TX_OK; 1132 } 1133 1134 static u64 fm10k_get_tx_completed(struct fm10k_ring *ring) 1135 { 1136 return ring->stats.packets; 1137 } 1138 1139 static u64 fm10k_get_tx_pending(struct fm10k_ring *ring) 1140 { 1141 /* use SW head and tail until we have real hardware */ 1142 u32 head = ring->next_to_clean; 1143 u32 tail = ring->next_to_use; 1144 1145 return ((head <= tail) ? tail : tail + ring->count) - head; 1146 } 1147 1148 bool fm10k_check_tx_hang(struct fm10k_ring *tx_ring) 1149 { 1150 u32 tx_done = fm10k_get_tx_completed(tx_ring); 1151 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 1152 u32 tx_pending = fm10k_get_tx_pending(tx_ring); 1153 1154 clear_check_for_tx_hang(tx_ring); 1155 1156 /* Check for a hung queue, but be thorough. This verifies 1157 * that a transmit has been completed since the previous 1158 * check AND there is at least one packet pending. By 1159 * requiring this to fail twice we avoid races with 1160 * clearing the ARMED bit and conditions where we 1161 * run the check_tx_hang logic with a transmit completion 1162 * pending but without time to complete it yet. 1163 */ 1164 if (!tx_pending || (tx_done_old != tx_done)) { 1165 /* update completed stats and continue */ 1166 tx_ring->tx_stats.tx_done_old = tx_done; 1167 /* reset the countdown */ 1168 clear_bit(__FM10K_HANG_CHECK_ARMED, &tx_ring->state); 1169 1170 return false; 1171 } 1172 1173 /* make sure it is true for two checks in a row */ 1174 return test_and_set_bit(__FM10K_HANG_CHECK_ARMED, &tx_ring->state); 1175 } 1176 1177 /** 1178 * fm10k_tx_timeout_reset - initiate reset due to Tx timeout 1179 * @interface: driver private struct 1180 **/ 1181 void fm10k_tx_timeout_reset(struct fm10k_intfc *interface) 1182 { 1183 /* Do the reset outside of interrupt context */ 1184 if (!test_bit(__FM10K_DOWN, &interface->state)) { 1185 netdev_err(interface->netdev, "Reset interface\n"); 1186 interface->tx_timeout_count++; 1187 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1188 fm10k_service_event_schedule(interface); 1189 } 1190 } 1191 1192 /** 1193 * fm10k_clean_tx_irq - Reclaim resources after transmit completes 1194 * @q_vector: structure containing interrupt and ring information 1195 * @tx_ring: tx ring to clean 1196 **/ 1197 static bool fm10k_clean_tx_irq(struct fm10k_q_vector *q_vector, 1198 struct fm10k_ring *tx_ring) 1199 { 1200 struct fm10k_intfc *interface = q_vector->interface; 1201 struct fm10k_tx_buffer *tx_buffer; 1202 struct fm10k_tx_desc *tx_desc; 1203 unsigned int total_bytes = 0, total_packets = 0; 1204 unsigned int budget = q_vector->tx.work_limit; 1205 unsigned int i = tx_ring->next_to_clean; 1206 1207 if (test_bit(__FM10K_DOWN, &interface->state)) 1208 return true; 1209 1210 tx_buffer = &tx_ring->tx_buffer[i]; 1211 tx_desc = FM10K_TX_DESC(tx_ring, i); 1212 i -= tx_ring->count; 1213 1214 do { 1215 struct fm10k_tx_desc *eop_desc = tx_buffer->next_to_watch; 1216 1217 /* if next_to_watch is not set then there is no work pending */ 1218 if (!eop_desc) 1219 break; 1220 1221 /* prevent any other reads prior to eop_desc */ 1222 read_barrier_depends(); 1223 1224 /* if DD is not set pending work has not been completed */ 1225 if (!(eop_desc->flags & FM10K_TXD_FLAG_DONE)) 1226 break; 1227 1228 /* clear next_to_watch to prevent false hangs */ 1229 tx_buffer->next_to_watch = NULL; 1230 1231 /* update the statistics for this packet */ 1232 total_bytes += tx_buffer->bytecount; 1233 total_packets += tx_buffer->gso_segs; 1234 1235 /* free the skb */ 1236 dev_consume_skb_any(tx_buffer->skb); 1237 1238 /* unmap skb header data */ 1239 dma_unmap_single(tx_ring->dev, 1240 dma_unmap_addr(tx_buffer, dma), 1241 dma_unmap_len(tx_buffer, len), 1242 DMA_TO_DEVICE); 1243 1244 /* clear tx_buffer data */ 1245 tx_buffer->skb = NULL; 1246 dma_unmap_len_set(tx_buffer, len, 0); 1247 1248 /* unmap remaining buffers */ 1249 while (tx_desc != eop_desc) { 1250 tx_buffer++; 1251 tx_desc++; 1252 i++; 1253 if (unlikely(!i)) { 1254 i -= tx_ring->count; 1255 tx_buffer = tx_ring->tx_buffer; 1256 tx_desc = FM10K_TX_DESC(tx_ring, 0); 1257 } 1258 1259 /* unmap any remaining paged data */ 1260 if (dma_unmap_len(tx_buffer, len)) { 1261 dma_unmap_page(tx_ring->dev, 1262 dma_unmap_addr(tx_buffer, dma), 1263 dma_unmap_len(tx_buffer, len), 1264 DMA_TO_DEVICE); 1265 dma_unmap_len_set(tx_buffer, len, 0); 1266 } 1267 } 1268 1269 /* move us one more past the eop_desc for start of next pkt */ 1270 tx_buffer++; 1271 tx_desc++; 1272 i++; 1273 if (unlikely(!i)) { 1274 i -= tx_ring->count; 1275 tx_buffer = tx_ring->tx_buffer; 1276 tx_desc = FM10K_TX_DESC(tx_ring, 0); 1277 } 1278 1279 /* issue prefetch for next Tx descriptor */ 1280 prefetch(tx_desc); 1281 1282 /* update budget accounting */ 1283 budget--; 1284 } while (likely(budget)); 1285 1286 i += tx_ring->count; 1287 tx_ring->next_to_clean = i; 1288 u64_stats_update_begin(&tx_ring->syncp); 1289 tx_ring->stats.bytes += total_bytes; 1290 tx_ring->stats.packets += total_packets; 1291 u64_stats_update_end(&tx_ring->syncp); 1292 q_vector->tx.total_bytes += total_bytes; 1293 q_vector->tx.total_packets += total_packets; 1294 1295 if (check_for_tx_hang(tx_ring) && fm10k_check_tx_hang(tx_ring)) { 1296 /* schedule immediate reset if we believe we hung */ 1297 struct fm10k_hw *hw = &interface->hw; 1298 1299 netif_err(interface, drv, tx_ring->netdev, 1300 "Detected Tx Unit Hang\n" 1301 " Tx Queue <%d>\n" 1302 " TDH, TDT <%x>, <%x>\n" 1303 " next_to_use <%x>\n" 1304 " next_to_clean <%x>\n", 1305 tx_ring->queue_index, 1306 fm10k_read_reg(hw, FM10K_TDH(tx_ring->reg_idx)), 1307 fm10k_read_reg(hw, FM10K_TDT(tx_ring->reg_idx)), 1308 tx_ring->next_to_use, i); 1309 1310 netif_stop_subqueue(tx_ring->netdev, 1311 tx_ring->queue_index); 1312 1313 netif_info(interface, probe, tx_ring->netdev, 1314 "tx hang %d detected on queue %d, resetting interface\n", 1315 interface->tx_timeout_count + 1, 1316 tx_ring->queue_index); 1317 1318 fm10k_tx_timeout_reset(interface); 1319 1320 /* the netdev is about to reset, no point in enabling stuff */ 1321 return true; 1322 } 1323 1324 /* notify netdev of completed buffers */ 1325 netdev_tx_completed_queue(txring_txq(tx_ring), 1326 total_packets, total_bytes); 1327 1328 #define TX_WAKE_THRESHOLD min_t(u16, FM10K_MIN_TXD - 1, DESC_NEEDED * 2) 1329 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 1330 (fm10k_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 1331 /* Make sure that anybody stopping the queue after this 1332 * sees the new next_to_clean. 1333 */ 1334 smp_mb(); 1335 if (__netif_subqueue_stopped(tx_ring->netdev, 1336 tx_ring->queue_index) && 1337 !test_bit(__FM10K_DOWN, &interface->state)) { 1338 netif_wake_subqueue(tx_ring->netdev, 1339 tx_ring->queue_index); 1340 ++tx_ring->tx_stats.restart_queue; 1341 } 1342 } 1343 1344 return !!budget; 1345 } 1346 1347 /** 1348 * fm10k_update_itr - update the dynamic ITR value based on packet size 1349 * 1350 * Stores a new ITR value based on strictly on packet size. The 1351 * divisors and thresholds used by this function were determined based 1352 * on theoretical maximum wire speed and testing data, in order to 1353 * minimize response time while increasing bulk throughput. 1354 * 1355 * @ring_container: Container for rings to have ITR updated 1356 **/ 1357 static void fm10k_update_itr(struct fm10k_ring_container *ring_container) 1358 { 1359 unsigned int avg_wire_size, packets; 1360 1361 /* Only update ITR if we are using adaptive setting */ 1362 if (!(ring_container->itr & FM10K_ITR_ADAPTIVE)) 1363 goto clear_counts; 1364 1365 packets = ring_container->total_packets; 1366 if (!packets) 1367 goto clear_counts; 1368 1369 avg_wire_size = ring_container->total_bytes / packets; 1370 1371 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 1372 avg_wire_size += 24; 1373 1374 /* Don't starve jumbo frames */ 1375 if (avg_wire_size > 3000) 1376 avg_wire_size = 3000; 1377 1378 /* Give a little boost to mid-size frames */ 1379 if ((avg_wire_size > 300) && (avg_wire_size < 1200)) 1380 avg_wire_size /= 3; 1381 else 1382 avg_wire_size /= 2; 1383 1384 /* write back value and retain adaptive flag */ 1385 ring_container->itr = avg_wire_size | FM10K_ITR_ADAPTIVE; 1386 1387 clear_counts: 1388 ring_container->total_bytes = 0; 1389 ring_container->total_packets = 0; 1390 } 1391 1392 static void fm10k_qv_enable(struct fm10k_q_vector *q_vector) 1393 { 1394 /* Enable auto-mask and clear the current mask */ 1395 u32 itr = FM10K_ITR_ENABLE; 1396 1397 /* Update Tx ITR */ 1398 fm10k_update_itr(&q_vector->tx); 1399 1400 /* Update Rx ITR */ 1401 fm10k_update_itr(&q_vector->rx); 1402 1403 /* Store Tx itr in timer slot 0 */ 1404 itr |= (q_vector->tx.itr & FM10K_ITR_MAX); 1405 1406 /* Shift Rx itr to timer slot 1 */ 1407 itr |= (q_vector->rx.itr & FM10K_ITR_MAX) << FM10K_ITR_INTERVAL1_SHIFT; 1408 1409 /* Write the final value to the ITR register */ 1410 writel(itr, q_vector->itr); 1411 } 1412 1413 static int fm10k_poll(struct napi_struct *napi, int budget) 1414 { 1415 struct fm10k_q_vector *q_vector = 1416 container_of(napi, struct fm10k_q_vector, napi); 1417 struct fm10k_ring *ring; 1418 int per_ring_budget; 1419 bool clean_complete = true; 1420 1421 fm10k_for_each_ring(ring, q_vector->tx) 1422 clean_complete &= fm10k_clean_tx_irq(q_vector, ring); 1423 1424 /* attempt to distribute budget to each queue fairly, but don't 1425 * allow the budget to go below 1 because we'll exit polling 1426 */ 1427 if (q_vector->rx.count > 1) 1428 per_ring_budget = max(budget/q_vector->rx.count, 1); 1429 else 1430 per_ring_budget = budget; 1431 1432 fm10k_for_each_ring(ring, q_vector->rx) 1433 clean_complete &= fm10k_clean_rx_irq(q_vector, ring, 1434 per_ring_budget); 1435 1436 /* If all work not completed, return budget and keep polling */ 1437 if (!clean_complete) 1438 return budget; 1439 1440 /* all work done, exit the polling mode */ 1441 napi_complete(napi); 1442 1443 /* re-enable the q_vector */ 1444 fm10k_qv_enable(q_vector); 1445 1446 return 0; 1447 } 1448 1449 /** 1450 * fm10k_set_qos_queues: Allocate queues for a QOS-enabled device 1451 * @interface: board private structure to initialize 1452 * 1453 * When QoS (Quality of Service) is enabled, allocate queues for 1454 * each traffic class. If multiqueue isn't available,then abort QoS 1455 * initialization. 1456 * 1457 * This function handles all combinations of Qos and RSS. 1458 * 1459 **/ 1460 static bool fm10k_set_qos_queues(struct fm10k_intfc *interface) 1461 { 1462 struct net_device *dev = interface->netdev; 1463 struct fm10k_ring_feature *f; 1464 int rss_i, i; 1465 int pcs; 1466 1467 /* Map queue offset and counts onto allocated tx queues */ 1468 pcs = netdev_get_num_tc(dev); 1469 1470 if (pcs <= 1) 1471 return false; 1472 1473 /* set QoS mask and indices */ 1474 f = &interface->ring_feature[RING_F_QOS]; 1475 f->indices = pcs; 1476 f->mask = (1 << fls(pcs - 1)) - 1; 1477 1478 /* determine the upper limit for our current DCB mode */ 1479 rss_i = interface->hw.mac.max_queues / pcs; 1480 rss_i = 1 << (fls(rss_i) - 1); 1481 1482 /* set RSS mask and indices */ 1483 f = &interface->ring_feature[RING_F_RSS]; 1484 rss_i = min_t(u16, rss_i, f->limit); 1485 f->indices = rss_i; 1486 f->mask = (1 << fls(rss_i - 1)) - 1; 1487 1488 /* configure pause class to queue mapping */ 1489 for (i = 0; i < pcs; i++) 1490 netdev_set_tc_queue(dev, i, rss_i, rss_i * i); 1491 1492 interface->num_rx_queues = rss_i * pcs; 1493 interface->num_tx_queues = rss_i * pcs; 1494 1495 return true; 1496 } 1497 1498 /** 1499 * fm10k_set_rss_queues: Allocate queues for RSS 1500 * @interface: board private structure to initialize 1501 * 1502 * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try 1503 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU. 1504 * 1505 **/ 1506 static bool fm10k_set_rss_queues(struct fm10k_intfc *interface) 1507 { 1508 struct fm10k_ring_feature *f; 1509 u16 rss_i; 1510 1511 f = &interface->ring_feature[RING_F_RSS]; 1512 rss_i = min_t(u16, interface->hw.mac.max_queues, f->limit); 1513 1514 /* record indices and power of 2 mask for RSS */ 1515 f->indices = rss_i; 1516 f->mask = (1 << fls(rss_i - 1)) - 1; 1517 1518 interface->num_rx_queues = rss_i; 1519 interface->num_tx_queues = rss_i; 1520 1521 return true; 1522 } 1523 1524 /** 1525 * fm10k_set_num_queues: Allocate queues for device, feature dependent 1526 * @interface: board private structure to initialize 1527 * 1528 * This is the top level queue allocation routine. The order here is very 1529 * important, starting with the "most" number of features turned on at once, 1530 * and ending with the smallest set of features. This way large combinations 1531 * can be allocated if they're turned on, and smaller combinations are the 1532 * fallthrough conditions. 1533 * 1534 **/ 1535 static void fm10k_set_num_queues(struct fm10k_intfc *interface) 1536 { 1537 /* Start with base case */ 1538 interface->num_rx_queues = 1; 1539 interface->num_tx_queues = 1; 1540 1541 if (fm10k_set_qos_queues(interface)) 1542 return; 1543 1544 fm10k_set_rss_queues(interface); 1545 } 1546 1547 /** 1548 * fm10k_alloc_q_vector - Allocate memory for a single interrupt vector 1549 * @interface: board private structure to initialize 1550 * @v_count: q_vectors allocated on interface, used for ring interleaving 1551 * @v_idx: index of vector in interface struct 1552 * @txr_count: total number of Tx rings to allocate 1553 * @txr_idx: index of first Tx ring to allocate 1554 * @rxr_count: total number of Rx rings to allocate 1555 * @rxr_idx: index of first Rx ring to allocate 1556 * 1557 * We allocate one q_vector. If allocation fails we return -ENOMEM. 1558 **/ 1559 static int fm10k_alloc_q_vector(struct fm10k_intfc *interface, 1560 unsigned int v_count, unsigned int v_idx, 1561 unsigned int txr_count, unsigned int txr_idx, 1562 unsigned int rxr_count, unsigned int rxr_idx) 1563 { 1564 struct fm10k_q_vector *q_vector; 1565 struct fm10k_ring *ring; 1566 int ring_count, size; 1567 1568 ring_count = txr_count + rxr_count; 1569 size = sizeof(struct fm10k_q_vector) + 1570 (sizeof(struct fm10k_ring) * ring_count); 1571 1572 /* allocate q_vector and rings */ 1573 q_vector = kzalloc(size, GFP_KERNEL); 1574 if (!q_vector) 1575 return -ENOMEM; 1576 1577 /* initialize NAPI */ 1578 netif_napi_add(interface->netdev, &q_vector->napi, 1579 fm10k_poll, NAPI_POLL_WEIGHT); 1580 1581 /* tie q_vector and interface together */ 1582 interface->q_vector[v_idx] = q_vector; 1583 q_vector->interface = interface; 1584 q_vector->v_idx = v_idx; 1585 1586 /* initialize pointer to rings */ 1587 ring = q_vector->ring; 1588 1589 /* save Tx ring container info */ 1590 q_vector->tx.ring = ring; 1591 q_vector->tx.work_limit = FM10K_DEFAULT_TX_WORK; 1592 q_vector->tx.itr = interface->tx_itr; 1593 q_vector->tx.count = txr_count; 1594 1595 while (txr_count) { 1596 /* assign generic ring traits */ 1597 ring->dev = &interface->pdev->dev; 1598 ring->netdev = interface->netdev; 1599 1600 /* configure backlink on ring */ 1601 ring->q_vector = q_vector; 1602 1603 /* apply Tx specific ring traits */ 1604 ring->count = interface->tx_ring_count; 1605 ring->queue_index = txr_idx; 1606 1607 /* assign ring to interface */ 1608 interface->tx_ring[txr_idx] = ring; 1609 1610 /* update count and index */ 1611 txr_count--; 1612 txr_idx += v_count; 1613 1614 /* push pointer to next ring */ 1615 ring++; 1616 } 1617 1618 /* save Rx ring container info */ 1619 q_vector->rx.ring = ring; 1620 q_vector->rx.itr = interface->rx_itr; 1621 q_vector->rx.count = rxr_count; 1622 1623 while (rxr_count) { 1624 /* assign generic ring traits */ 1625 ring->dev = &interface->pdev->dev; 1626 ring->netdev = interface->netdev; 1627 rcu_assign_pointer(ring->l2_accel, interface->l2_accel); 1628 1629 /* configure backlink on ring */ 1630 ring->q_vector = q_vector; 1631 1632 /* apply Rx specific ring traits */ 1633 ring->count = interface->rx_ring_count; 1634 ring->queue_index = rxr_idx; 1635 1636 /* assign ring to interface */ 1637 interface->rx_ring[rxr_idx] = ring; 1638 1639 /* update count and index */ 1640 rxr_count--; 1641 rxr_idx += v_count; 1642 1643 /* push pointer to next ring */ 1644 ring++; 1645 } 1646 1647 fm10k_dbg_q_vector_init(q_vector); 1648 1649 return 0; 1650 } 1651 1652 /** 1653 * fm10k_free_q_vector - Free memory allocated for specific interrupt vector 1654 * @interface: board private structure to initialize 1655 * @v_idx: Index of vector to be freed 1656 * 1657 * This function frees the memory allocated to the q_vector. In addition if 1658 * NAPI is enabled it will delete any references to the NAPI struct prior 1659 * to freeing the q_vector. 1660 **/ 1661 static void fm10k_free_q_vector(struct fm10k_intfc *interface, int v_idx) 1662 { 1663 struct fm10k_q_vector *q_vector = interface->q_vector[v_idx]; 1664 struct fm10k_ring *ring; 1665 1666 fm10k_dbg_q_vector_exit(q_vector); 1667 1668 fm10k_for_each_ring(ring, q_vector->tx) 1669 interface->tx_ring[ring->queue_index] = NULL; 1670 1671 fm10k_for_each_ring(ring, q_vector->rx) 1672 interface->rx_ring[ring->queue_index] = NULL; 1673 1674 interface->q_vector[v_idx] = NULL; 1675 netif_napi_del(&q_vector->napi); 1676 kfree_rcu(q_vector, rcu); 1677 } 1678 1679 /** 1680 * fm10k_alloc_q_vectors - Allocate memory for interrupt vectors 1681 * @interface: board private structure to initialize 1682 * 1683 * We allocate one q_vector per queue interrupt. If allocation fails we 1684 * return -ENOMEM. 1685 **/ 1686 static int fm10k_alloc_q_vectors(struct fm10k_intfc *interface) 1687 { 1688 unsigned int q_vectors = interface->num_q_vectors; 1689 unsigned int rxr_remaining = interface->num_rx_queues; 1690 unsigned int txr_remaining = interface->num_tx_queues; 1691 unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0; 1692 int err; 1693 1694 if (q_vectors >= (rxr_remaining + txr_remaining)) { 1695 for (; rxr_remaining; v_idx++) { 1696 err = fm10k_alloc_q_vector(interface, q_vectors, v_idx, 1697 0, 0, 1, rxr_idx); 1698 if (err) 1699 goto err_out; 1700 1701 /* update counts and index */ 1702 rxr_remaining--; 1703 rxr_idx++; 1704 } 1705 } 1706 1707 for (; v_idx < q_vectors; v_idx++) { 1708 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 1709 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 1710 1711 err = fm10k_alloc_q_vector(interface, q_vectors, v_idx, 1712 tqpv, txr_idx, 1713 rqpv, rxr_idx); 1714 1715 if (err) 1716 goto err_out; 1717 1718 /* update counts and index */ 1719 rxr_remaining -= rqpv; 1720 txr_remaining -= tqpv; 1721 rxr_idx++; 1722 txr_idx++; 1723 } 1724 1725 return 0; 1726 1727 err_out: 1728 interface->num_tx_queues = 0; 1729 interface->num_rx_queues = 0; 1730 interface->num_q_vectors = 0; 1731 1732 while (v_idx--) 1733 fm10k_free_q_vector(interface, v_idx); 1734 1735 return -ENOMEM; 1736 } 1737 1738 /** 1739 * fm10k_free_q_vectors - Free memory allocated for interrupt vectors 1740 * @interface: board private structure to initialize 1741 * 1742 * This function frees the memory allocated to the q_vectors. In addition if 1743 * NAPI is enabled it will delete any references to the NAPI struct prior 1744 * to freeing the q_vector. 1745 **/ 1746 static void fm10k_free_q_vectors(struct fm10k_intfc *interface) 1747 { 1748 int v_idx = interface->num_q_vectors; 1749 1750 interface->num_tx_queues = 0; 1751 interface->num_rx_queues = 0; 1752 interface->num_q_vectors = 0; 1753 1754 while (v_idx--) 1755 fm10k_free_q_vector(interface, v_idx); 1756 } 1757 1758 /** 1759 * f10k_reset_msix_capability - reset MSI-X capability 1760 * @interface: board private structure to initialize 1761 * 1762 * Reset the MSI-X capability back to its starting state 1763 **/ 1764 static void fm10k_reset_msix_capability(struct fm10k_intfc *interface) 1765 { 1766 pci_disable_msix(interface->pdev); 1767 kfree(interface->msix_entries); 1768 interface->msix_entries = NULL; 1769 } 1770 1771 /** 1772 * f10k_init_msix_capability - configure MSI-X capability 1773 * @interface: board private structure to initialize 1774 * 1775 * Attempt to configure the interrupts using the best available 1776 * capabilities of the hardware and the kernel. 1777 **/ 1778 static int fm10k_init_msix_capability(struct fm10k_intfc *interface) 1779 { 1780 struct fm10k_hw *hw = &interface->hw; 1781 int v_budget, vector; 1782 1783 /* It's easy to be greedy for MSI-X vectors, but it really 1784 * doesn't do us much good if we have a lot more vectors 1785 * than CPU's. So let's be conservative and only ask for 1786 * (roughly) the same number of vectors as there are CPU's. 1787 * the default is to use pairs of vectors 1788 */ 1789 v_budget = max(interface->num_rx_queues, interface->num_tx_queues); 1790 v_budget = min_t(u16, v_budget, num_online_cpus()); 1791 1792 /* account for vectors not related to queues */ 1793 v_budget += NON_Q_VECTORS(hw); 1794 1795 /* At the same time, hardware can only support a maximum of 1796 * hw.mac->max_msix_vectors vectors. With features 1797 * such as RSS and VMDq, we can easily surpass the number of Rx and Tx 1798 * descriptor queues supported by our device. Thus, we cap it off in 1799 * those rare cases where the cpu count also exceeds our vector limit. 1800 */ 1801 v_budget = min_t(int, v_budget, hw->mac.max_msix_vectors); 1802 1803 /* A failure in MSI-X entry allocation is fatal. */ 1804 interface->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 1805 GFP_KERNEL); 1806 if (!interface->msix_entries) 1807 return -ENOMEM; 1808 1809 /* populate entry values */ 1810 for (vector = 0; vector < v_budget; vector++) 1811 interface->msix_entries[vector].entry = vector; 1812 1813 /* Attempt to enable MSI-X with requested value */ 1814 v_budget = pci_enable_msix_range(interface->pdev, 1815 interface->msix_entries, 1816 MIN_MSIX_COUNT(hw), 1817 v_budget); 1818 if (v_budget < 0) { 1819 kfree(interface->msix_entries); 1820 interface->msix_entries = NULL; 1821 return -ENOMEM; 1822 } 1823 1824 /* record the number of queues available for q_vectors */ 1825 interface->num_q_vectors = v_budget - NON_Q_VECTORS(hw); 1826 1827 return 0; 1828 } 1829 1830 /** 1831 * fm10k_cache_ring_qos - Descriptor ring to register mapping for QoS 1832 * @interface: Interface structure continaining rings and devices 1833 * 1834 * Cache the descriptor ring offsets for Qos 1835 **/ 1836 static bool fm10k_cache_ring_qos(struct fm10k_intfc *interface) 1837 { 1838 struct net_device *dev = interface->netdev; 1839 int pc, offset, rss_i, i, q_idx; 1840 u16 pc_stride = interface->ring_feature[RING_F_QOS].mask + 1; 1841 u8 num_pcs = netdev_get_num_tc(dev); 1842 1843 if (num_pcs <= 1) 1844 return false; 1845 1846 rss_i = interface->ring_feature[RING_F_RSS].indices; 1847 1848 for (pc = 0, offset = 0; pc < num_pcs; pc++, offset += rss_i) { 1849 q_idx = pc; 1850 for (i = 0; i < rss_i; i++) { 1851 interface->tx_ring[offset + i]->reg_idx = q_idx; 1852 interface->tx_ring[offset + i]->qos_pc = pc; 1853 interface->rx_ring[offset + i]->reg_idx = q_idx; 1854 interface->rx_ring[offset + i]->qos_pc = pc; 1855 q_idx += pc_stride; 1856 } 1857 } 1858 1859 return true; 1860 } 1861 1862 /** 1863 * fm10k_cache_ring_rss - Descriptor ring to register mapping for RSS 1864 * @interface: Interface structure continaining rings and devices 1865 * 1866 * Cache the descriptor ring offsets for RSS 1867 **/ 1868 static void fm10k_cache_ring_rss(struct fm10k_intfc *interface) 1869 { 1870 int i; 1871 1872 for (i = 0; i < interface->num_rx_queues; i++) 1873 interface->rx_ring[i]->reg_idx = i; 1874 1875 for (i = 0; i < interface->num_tx_queues; i++) 1876 interface->tx_ring[i]->reg_idx = i; 1877 } 1878 1879 /** 1880 * fm10k_assign_rings - Map rings to network devices 1881 * @interface: Interface structure containing rings and devices 1882 * 1883 * This function is meant to go though and configure both the network 1884 * devices so that they contain rings, and configure the rings so that 1885 * they function with their network devices. 1886 **/ 1887 static void fm10k_assign_rings(struct fm10k_intfc *interface) 1888 { 1889 if (fm10k_cache_ring_qos(interface)) 1890 return; 1891 1892 fm10k_cache_ring_rss(interface); 1893 } 1894 1895 static void fm10k_init_reta(struct fm10k_intfc *interface) 1896 { 1897 u16 i, rss_i = interface->ring_feature[RING_F_RSS].indices; 1898 u32 reta, base; 1899 1900 /* If the netdev is initialized we have to maintain table if possible */ 1901 if (interface->netdev->reg_state) { 1902 for (i = FM10K_RETA_SIZE; i--;) { 1903 reta = interface->reta[i]; 1904 if ((((reta << 24) >> 24) < rss_i) && 1905 (((reta << 16) >> 24) < rss_i) && 1906 (((reta << 8) >> 24) < rss_i) && 1907 (((reta) >> 24) < rss_i)) 1908 continue; 1909 goto repopulate_reta; 1910 } 1911 1912 /* do nothing if all of the elements are in bounds */ 1913 return; 1914 } 1915 1916 repopulate_reta: 1917 /* Populate the redirection table 4 entries at a time. To do this 1918 * we are generating the results for n and n+2 and then interleaving 1919 * those with the results with n+1 and n+3. 1920 */ 1921 for (i = FM10K_RETA_SIZE; i--;) { 1922 /* first pass generates n and n+2 */ 1923 base = ((i * 0x00040004) + 0x00020000) * rss_i; 1924 reta = (base & 0x3F803F80) >> 7; 1925 1926 /* second pass generates n+1 and n+3 */ 1927 base += 0x00010001 * rss_i; 1928 reta |= (base & 0x3F803F80) << 1; 1929 1930 interface->reta[i] = reta; 1931 } 1932 } 1933 1934 /** 1935 * fm10k_init_queueing_scheme - Determine proper queueing scheme 1936 * @interface: board private structure to initialize 1937 * 1938 * We determine which queueing scheme to use based on... 1939 * - Hardware queue count (num_*_queues) 1940 * - defined by miscellaneous hardware support/features (RSS, etc.) 1941 **/ 1942 int fm10k_init_queueing_scheme(struct fm10k_intfc *interface) 1943 { 1944 int err; 1945 1946 /* Number of supported queues */ 1947 fm10k_set_num_queues(interface); 1948 1949 /* Configure MSI-X capability */ 1950 err = fm10k_init_msix_capability(interface); 1951 if (err) { 1952 dev_err(&interface->pdev->dev, 1953 "Unable to initialize MSI-X capability\n"); 1954 return err; 1955 } 1956 1957 /* Allocate memory for queues */ 1958 err = fm10k_alloc_q_vectors(interface); 1959 if (err) 1960 return err; 1961 1962 /* Map rings to devices, and map devices to physical queues */ 1963 fm10k_assign_rings(interface); 1964 1965 /* Initialize RSS redirection table */ 1966 fm10k_init_reta(interface); 1967 1968 return 0; 1969 } 1970 1971 /** 1972 * fm10k_clear_queueing_scheme - Clear the current queueing scheme settings 1973 * @interface: board private structure to clear queueing scheme on 1974 * 1975 * We go through and clear queueing specific resources and reset the structure 1976 * to pre-load conditions 1977 **/ 1978 void fm10k_clear_queueing_scheme(struct fm10k_intfc *interface) 1979 { 1980 fm10k_free_q_vectors(interface); 1981 fm10k_reset_msix_capability(interface); 1982 } 1983