1 /* Intel Ethernet Switch Host Interface Driver 2 * Copyright(c) 2013 - 2014 Intel Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * The full GNU General Public License is included in this distribution in 14 * the file called "COPYING". 15 * 16 * Contact Information: 17 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 18 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 19 */ 20 21 #include <linux/types.h> 22 #include <linux/module.h> 23 #include <net/ipv6.h> 24 #include <net/ip.h> 25 #include <net/tcp.h> 26 #include <linux/if_macvlan.h> 27 #include <linux/prefetch.h> 28 29 #include "fm10k.h" 30 31 #define DRV_VERSION "0.15.2-k" 32 const char fm10k_driver_version[] = DRV_VERSION; 33 char fm10k_driver_name[] = "fm10k"; 34 static const char fm10k_driver_string[] = 35 "Intel(R) Ethernet Switch Host Interface Driver"; 36 static const char fm10k_copyright[] = 37 "Copyright (c) 2013 Intel Corporation."; 38 39 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 40 MODULE_DESCRIPTION("Intel(R) Ethernet Switch Host Interface Driver"); 41 MODULE_LICENSE("GPL"); 42 MODULE_VERSION(DRV_VERSION); 43 44 /* single workqueue for entire fm10k driver */ 45 struct workqueue_struct *fm10k_workqueue = NULL; 46 47 /** 48 * fm10k_init_module - Driver Registration Routine 49 * 50 * fm10k_init_module is the first routine called when the driver is 51 * loaded. All it does is register with the PCI subsystem. 52 **/ 53 static int __init fm10k_init_module(void) 54 { 55 pr_info("%s - version %s\n", fm10k_driver_string, fm10k_driver_version); 56 pr_info("%s\n", fm10k_copyright); 57 58 /* create driver workqueue */ 59 if (!fm10k_workqueue) 60 fm10k_workqueue = create_workqueue("fm10k"); 61 62 fm10k_dbg_init(); 63 64 return fm10k_register_pci_driver(); 65 } 66 module_init(fm10k_init_module); 67 68 /** 69 * fm10k_exit_module - Driver Exit Cleanup Routine 70 * 71 * fm10k_exit_module is called just before the driver is removed 72 * from memory. 73 **/ 74 static void __exit fm10k_exit_module(void) 75 { 76 fm10k_unregister_pci_driver(); 77 78 fm10k_dbg_exit(); 79 80 /* destroy driver workqueue */ 81 flush_workqueue(fm10k_workqueue); 82 destroy_workqueue(fm10k_workqueue); 83 fm10k_workqueue = NULL; 84 } 85 module_exit(fm10k_exit_module); 86 87 static bool fm10k_alloc_mapped_page(struct fm10k_ring *rx_ring, 88 struct fm10k_rx_buffer *bi) 89 { 90 struct page *page = bi->page; 91 dma_addr_t dma; 92 93 /* Only page will be NULL if buffer was consumed */ 94 if (likely(page)) 95 return true; 96 97 /* alloc new page for storage */ 98 page = dev_alloc_page(); 99 if (unlikely(!page)) { 100 rx_ring->rx_stats.alloc_failed++; 101 return false; 102 } 103 104 /* map page for use */ 105 dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 106 107 /* if mapping failed free memory back to system since 108 * there isn't much point in holding memory we can't use 109 */ 110 if (dma_mapping_error(rx_ring->dev, dma)) { 111 __free_page(page); 112 113 rx_ring->rx_stats.alloc_failed++; 114 return false; 115 } 116 117 bi->dma = dma; 118 bi->page = page; 119 bi->page_offset = 0; 120 121 return true; 122 } 123 124 /** 125 * fm10k_alloc_rx_buffers - Replace used receive buffers 126 * @rx_ring: ring to place buffers on 127 * @cleaned_count: number of buffers to replace 128 **/ 129 void fm10k_alloc_rx_buffers(struct fm10k_ring *rx_ring, u16 cleaned_count) 130 { 131 union fm10k_rx_desc *rx_desc; 132 struct fm10k_rx_buffer *bi; 133 u16 i = rx_ring->next_to_use; 134 135 /* nothing to do */ 136 if (!cleaned_count) 137 return; 138 139 rx_desc = FM10K_RX_DESC(rx_ring, i); 140 bi = &rx_ring->rx_buffer[i]; 141 i -= rx_ring->count; 142 143 do { 144 if (!fm10k_alloc_mapped_page(rx_ring, bi)) 145 break; 146 147 /* Refresh the desc even if buffer_addrs didn't change 148 * because each write-back erases this info. 149 */ 150 rx_desc->q.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 151 152 rx_desc++; 153 bi++; 154 i++; 155 if (unlikely(!i)) { 156 rx_desc = FM10K_RX_DESC(rx_ring, 0); 157 bi = rx_ring->rx_buffer; 158 i -= rx_ring->count; 159 } 160 161 /* clear the status bits for the next_to_use descriptor */ 162 rx_desc->d.staterr = 0; 163 164 cleaned_count--; 165 } while (cleaned_count); 166 167 i += rx_ring->count; 168 169 if (rx_ring->next_to_use != i) { 170 /* record the next descriptor to use */ 171 rx_ring->next_to_use = i; 172 173 /* update next to alloc since we have filled the ring */ 174 rx_ring->next_to_alloc = i; 175 176 /* Force memory writes to complete before letting h/w 177 * know there are new descriptors to fetch. (Only 178 * applicable for weak-ordered memory model archs, 179 * such as IA-64). 180 */ 181 wmb(); 182 183 /* notify hardware of new descriptors */ 184 writel(i, rx_ring->tail); 185 } 186 } 187 188 /** 189 * fm10k_reuse_rx_page - page flip buffer and store it back on the ring 190 * @rx_ring: rx descriptor ring to store buffers on 191 * @old_buff: donor buffer to have page reused 192 * 193 * Synchronizes page for reuse by the interface 194 **/ 195 static void fm10k_reuse_rx_page(struct fm10k_ring *rx_ring, 196 struct fm10k_rx_buffer *old_buff) 197 { 198 struct fm10k_rx_buffer *new_buff; 199 u16 nta = rx_ring->next_to_alloc; 200 201 new_buff = &rx_ring->rx_buffer[nta]; 202 203 /* update, and store next to alloc */ 204 nta++; 205 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 206 207 /* transfer page from old buffer to new buffer */ 208 *new_buff = *old_buff; 209 210 /* sync the buffer for use by the device */ 211 dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma, 212 old_buff->page_offset, 213 FM10K_RX_BUFSZ, 214 DMA_FROM_DEVICE); 215 } 216 217 static inline bool fm10k_page_is_reserved(struct page *page) 218 { 219 return (page_to_nid(page) != numa_mem_id()) || page->pfmemalloc; 220 } 221 222 static bool fm10k_can_reuse_rx_page(struct fm10k_rx_buffer *rx_buffer, 223 struct page *page, 224 unsigned int __maybe_unused truesize) 225 { 226 /* avoid re-using remote pages */ 227 if (unlikely(fm10k_page_is_reserved(page))) 228 return false; 229 230 #if (PAGE_SIZE < 8192) 231 /* if we are only owner of page we can reuse it */ 232 if (unlikely(page_count(page) != 1)) 233 return false; 234 235 /* flip page offset to other buffer */ 236 rx_buffer->page_offset ^= FM10K_RX_BUFSZ; 237 #else 238 /* move offset up to the next cache line */ 239 rx_buffer->page_offset += truesize; 240 241 if (rx_buffer->page_offset > (PAGE_SIZE - FM10K_RX_BUFSZ)) 242 return false; 243 #endif 244 245 /* Even if we own the page, we are not allowed to use atomic_set() 246 * This would break get_page_unless_zero() users. 247 */ 248 atomic_inc(&page->_count); 249 250 return true; 251 } 252 253 /** 254 * fm10k_add_rx_frag - Add contents of Rx buffer to sk_buff 255 * @rx_buffer: buffer containing page to add 256 * @rx_desc: descriptor containing length of buffer written by hardware 257 * @skb: sk_buff to place the data into 258 * 259 * This function will add the data contained in rx_buffer->page to the skb. 260 * This is done either through a direct copy if the data in the buffer is 261 * less than the skb header size, otherwise it will just attach the page as 262 * a frag to the skb. 263 * 264 * The function will then update the page offset if necessary and return 265 * true if the buffer can be reused by the interface. 266 **/ 267 static bool fm10k_add_rx_frag(struct fm10k_rx_buffer *rx_buffer, 268 union fm10k_rx_desc *rx_desc, 269 struct sk_buff *skb) 270 { 271 struct page *page = rx_buffer->page; 272 unsigned int size = le16_to_cpu(rx_desc->w.length); 273 #if (PAGE_SIZE < 8192) 274 unsigned int truesize = FM10K_RX_BUFSZ; 275 #else 276 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES); 277 #endif 278 279 if ((size <= FM10K_RX_HDR_LEN) && !skb_is_nonlinear(skb)) { 280 unsigned char *va = page_address(page) + rx_buffer->page_offset; 281 282 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 283 284 /* page is not reserved, we can reuse buffer as-is */ 285 if (likely(!fm10k_page_is_reserved(page))) 286 return true; 287 288 /* this page cannot be reused so discard it */ 289 __free_page(page); 290 return false; 291 } 292 293 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 294 rx_buffer->page_offset, size, truesize); 295 296 return fm10k_can_reuse_rx_page(rx_buffer, page, truesize); 297 } 298 299 static struct sk_buff *fm10k_fetch_rx_buffer(struct fm10k_ring *rx_ring, 300 union fm10k_rx_desc *rx_desc, 301 struct sk_buff *skb) 302 { 303 struct fm10k_rx_buffer *rx_buffer; 304 struct page *page; 305 306 rx_buffer = &rx_ring->rx_buffer[rx_ring->next_to_clean]; 307 page = rx_buffer->page; 308 prefetchw(page); 309 310 if (likely(!skb)) { 311 void *page_addr = page_address(page) + 312 rx_buffer->page_offset; 313 314 /* prefetch first cache line of first page */ 315 prefetch(page_addr); 316 #if L1_CACHE_BYTES < 128 317 prefetch(page_addr + L1_CACHE_BYTES); 318 #endif 319 320 /* allocate a skb to store the frags */ 321 skb = napi_alloc_skb(&rx_ring->q_vector->napi, 322 FM10K_RX_HDR_LEN); 323 if (unlikely(!skb)) { 324 rx_ring->rx_stats.alloc_failed++; 325 return NULL; 326 } 327 328 /* we will be copying header into skb->data in 329 * pskb_may_pull so it is in our interest to prefetch 330 * it now to avoid a possible cache miss 331 */ 332 prefetchw(skb->data); 333 } 334 335 /* we are reusing so sync this buffer for CPU use */ 336 dma_sync_single_range_for_cpu(rx_ring->dev, 337 rx_buffer->dma, 338 rx_buffer->page_offset, 339 FM10K_RX_BUFSZ, 340 DMA_FROM_DEVICE); 341 342 /* pull page into skb */ 343 if (fm10k_add_rx_frag(rx_buffer, rx_desc, skb)) { 344 /* hand second half of page back to the ring */ 345 fm10k_reuse_rx_page(rx_ring, rx_buffer); 346 } else { 347 /* we are not reusing the buffer so unmap it */ 348 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 349 PAGE_SIZE, DMA_FROM_DEVICE); 350 } 351 352 /* clear contents of rx_buffer */ 353 rx_buffer->page = NULL; 354 355 return skb; 356 } 357 358 static inline void fm10k_rx_checksum(struct fm10k_ring *ring, 359 union fm10k_rx_desc *rx_desc, 360 struct sk_buff *skb) 361 { 362 skb_checksum_none_assert(skb); 363 364 /* Rx checksum disabled via ethtool */ 365 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 366 return; 367 368 /* TCP/UDP checksum error bit is set */ 369 if (fm10k_test_staterr(rx_desc, 370 FM10K_RXD_STATUS_L4E | 371 FM10K_RXD_STATUS_L4E2 | 372 FM10K_RXD_STATUS_IPE | 373 FM10K_RXD_STATUS_IPE2)) { 374 ring->rx_stats.csum_err++; 375 return; 376 } 377 378 /* It must be a TCP or UDP packet with a valid checksum */ 379 if (fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_L4CS2)) 380 skb->encapsulation = true; 381 else if (!fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_L4CS)) 382 return; 383 384 skb->ip_summed = CHECKSUM_UNNECESSARY; 385 } 386 387 #define FM10K_RSS_L4_TYPES_MASK \ 388 ((1ul << FM10K_RSSTYPE_IPV4_TCP) | \ 389 (1ul << FM10K_RSSTYPE_IPV4_UDP) | \ 390 (1ul << FM10K_RSSTYPE_IPV6_TCP) | \ 391 (1ul << FM10K_RSSTYPE_IPV6_UDP)) 392 393 static inline void fm10k_rx_hash(struct fm10k_ring *ring, 394 union fm10k_rx_desc *rx_desc, 395 struct sk_buff *skb) 396 { 397 u16 rss_type; 398 399 if (!(ring->netdev->features & NETIF_F_RXHASH)) 400 return; 401 402 rss_type = le16_to_cpu(rx_desc->w.pkt_info) & FM10K_RXD_RSSTYPE_MASK; 403 if (!rss_type) 404 return; 405 406 skb_set_hash(skb, le32_to_cpu(rx_desc->d.rss), 407 (FM10K_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? 408 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); 409 } 410 411 static void fm10k_rx_hwtstamp(struct fm10k_ring *rx_ring, 412 union fm10k_rx_desc *rx_desc, 413 struct sk_buff *skb) 414 { 415 struct fm10k_intfc *interface = rx_ring->q_vector->interface; 416 417 FM10K_CB(skb)->tstamp = rx_desc->q.timestamp; 418 419 if (unlikely(interface->flags & FM10K_FLAG_RX_TS_ENABLED)) 420 fm10k_systime_to_hwtstamp(interface, skb_hwtstamps(skb), 421 le64_to_cpu(rx_desc->q.timestamp)); 422 } 423 424 static void fm10k_type_trans(struct fm10k_ring *rx_ring, 425 union fm10k_rx_desc __maybe_unused *rx_desc, 426 struct sk_buff *skb) 427 { 428 struct net_device *dev = rx_ring->netdev; 429 struct fm10k_l2_accel *l2_accel = rcu_dereference_bh(rx_ring->l2_accel); 430 431 /* check to see if DGLORT belongs to a MACVLAN */ 432 if (l2_accel) { 433 u16 idx = le16_to_cpu(FM10K_CB(skb)->fi.w.dglort) - 1; 434 435 idx -= l2_accel->dglort; 436 if (idx < l2_accel->size && l2_accel->macvlan[idx]) 437 dev = l2_accel->macvlan[idx]; 438 else 439 l2_accel = NULL; 440 } 441 442 skb->protocol = eth_type_trans(skb, dev); 443 444 if (!l2_accel) 445 return; 446 447 /* update MACVLAN statistics */ 448 macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, 1, 449 !!(rx_desc->w.hdr_info & 450 cpu_to_le16(FM10K_RXD_HDR_INFO_XC_MASK))); 451 } 452 453 /** 454 * fm10k_process_skb_fields - Populate skb header fields from Rx descriptor 455 * @rx_ring: rx descriptor ring packet is being transacted on 456 * @rx_desc: pointer to the EOP Rx descriptor 457 * @skb: pointer to current skb being populated 458 * 459 * This function checks the ring, descriptor, and packet information in 460 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 461 * other fields within the skb. 462 **/ 463 static unsigned int fm10k_process_skb_fields(struct fm10k_ring *rx_ring, 464 union fm10k_rx_desc *rx_desc, 465 struct sk_buff *skb) 466 { 467 unsigned int len = skb->len; 468 469 fm10k_rx_hash(rx_ring, rx_desc, skb); 470 471 fm10k_rx_checksum(rx_ring, rx_desc, skb); 472 473 fm10k_rx_hwtstamp(rx_ring, rx_desc, skb); 474 475 FM10K_CB(skb)->fi.w.vlan = rx_desc->w.vlan; 476 477 skb_record_rx_queue(skb, rx_ring->queue_index); 478 479 FM10K_CB(skb)->fi.d.glort = rx_desc->d.glort; 480 481 if (rx_desc->w.vlan) { 482 u16 vid = le16_to_cpu(rx_desc->w.vlan); 483 484 if (vid != rx_ring->vid) 485 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 486 } 487 488 fm10k_type_trans(rx_ring, rx_desc, skb); 489 490 return len; 491 } 492 493 /** 494 * fm10k_is_non_eop - process handling of non-EOP buffers 495 * @rx_ring: Rx ring being processed 496 * @rx_desc: Rx descriptor for current buffer 497 * 498 * This function updates next to clean. If the buffer is an EOP buffer 499 * this function exits returning false, otherwise it will place the 500 * sk_buff in the next buffer to be chained and return true indicating 501 * that this is in fact a non-EOP buffer. 502 **/ 503 static bool fm10k_is_non_eop(struct fm10k_ring *rx_ring, 504 union fm10k_rx_desc *rx_desc) 505 { 506 u32 ntc = rx_ring->next_to_clean + 1; 507 508 /* fetch, update, and store next to clean */ 509 ntc = (ntc < rx_ring->count) ? ntc : 0; 510 rx_ring->next_to_clean = ntc; 511 512 prefetch(FM10K_RX_DESC(rx_ring, ntc)); 513 514 if (likely(fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_EOP))) 515 return false; 516 517 return true; 518 } 519 520 /** 521 * fm10k_pull_tail - fm10k specific version of skb_pull_tail 522 * @skb: pointer to current skb being adjusted 523 * 524 * This function is an fm10k specific version of __pskb_pull_tail. The 525 * main difference between this version and the original function is that 526 * this function can make several assumptions about the state of things 527 * that allow for significant optimizations versus the standard function. 528 * As a result we can do things like drop a frag and maintain an accurate 529 * truesize for the skb. 530 */ 531 static void fm10k_pull_tail(struct sk_buff *skb) 532 { 533 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 534 unsigned char *va; 535 unsigned int pull_len; 536 537 /* it is valid to use page_address instead of kmap since we are 538 * working with pages allocated out of the lomem pool per 539 * alloc_page(GFP_ATOMIC) 540 */ 541 va = skb_frag_address(frag); 542 543 /* we need the header to contain the greater of either ETH_HLEN or 544 * 60 bytes if the skb->len is less than 60 for skb_pad. 545 */ 546 pull_len = eth_get_headlen(va, FM10K_RX_HDR_LEN); 547 548 /* align pull length to size of long to optimize memcpy performance */ 549 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); 550 551 /* update all of the pointers */ 552 skb_frag_size_sub(frag, pull_len); 553 frag->page_offset += pull_len; 554 skb->data_len -= pull_len; 555 skb->tail += pull_len; 556 } 557 558 /** 559 * fm10k_cleanup_headers - Correct corrupted or empty headers 560 * @rx_ring: rx descriptor ring packet is being transacted on 561 * @rx_desc: pointer to the EOP Rx descriptor 562 * @skb: pointer to current skb being fixed 563 * 564 * Address the case where we are pulling data in on pages only 565 * and as such no data is present in the skb header. 566 * 567 * In addition if skb is not at least 60 bytes we need to pad it so that 568 * it is large enough to qualify as a valid Ethernet frame. 569 * 570 * Returns true if an error was encountered and skb was freed. 571 **/ 572 static bool fm10k_cleanup_headers(struct fm10k_ring *rx_ring, 573 union fm10k_rx_desc *rx_desc, 574 struct sk_buff *skb) 575 { 576 if (unlikely((fm10k_test_staterr(rx_desc, 577 FM10K_RXD_STATUS_RXE)))) { 578 dev_kfree_skb_any(skb); 579 rx_ring->rx_stats.errors++; 580 return true; 581 } 582 583 /* place header in linear portion of buffer */ 584 if (skb_is_nonlinear(skb)) 585 fm10k_pull_tail(skb); 586 587 /* if eth_skb_pad returns an error the skb was freed */ 588 if (eth_skb_pad(skb)) 589 return true; 590 591 return false; 592 } 593 594 /** 595 * fm10k_receive_skb - helper function to handle rx indications 596 * @q_vector: structure containing interrupt and ring information 597 * @skb: packet to send up 598 **/ 599 static void fm10k_receive_skb(struct fm10k_q_vector *q_vector, 600 struct sk_buff *skb) 601 { 602 napi_gro_receive(&q_vector->napi, skb); 603 } 604 605 static bool fm10k_clean_rx_irq(struct fm10k_q_vector *q_vector, 606 struct fm10k_ring *rx_ring, 607 int budget) 608 { 609 struct sk_buff *skb = rx_ring->skb; 610 unsigned int total_bytes = 0, total_packets = 0; 611 u16 cleaned_count = fm10k_desc_unused(rx_ring); 612 613 while (likely(total_packets < budget)) { 614 union fm10k_rx_desc *rx_desc; 615 616 /* return some buffers to hardware, one at a time is too slow */ 617 if (cleaned_count >= FM10K_RX_BUFFER_WRITE) { 618 fm10k_alloc_rx_buffers(rx_ring, cleaned_count); 619 cleaned_count = 0; 620 } 621 622 rx_desc = FM10K_RX_DESC(rx_ring, rx_ring->next_to_clean); 623 624 if (!rx_desc->d.staterr) 625 break; 626 627 /* This memory barrier is needed to keep us from reading 628 * any other fields out of the rx_desc until we know the 629 * descriptor has been written back 630 */ 631 dma_rmb(); 632 633 /* retrieve a buffer from the ring */ 634 skb = fm10k_fetch_rx_buffer(rx_ring, rx_desc, skb); 635 636 /* exit if we failed to retrieve a buffer */ 637 if (!skb) 638 break; 639 640 cleaned_count++; 641 642 /* fetch next buffer in frame if non-eop */ 643 if (fm10k_is_non_eop(rx_ring, rx_desc)) 644 continue; 645 646 /* verify the packet layout is correct */ 647 if (fm10k_cleanup_headers(rx_ring, rx_desc, skb)) { 648 skb = NULL; 649 continue; 650 } 651 652 /* populate checksum, timestamp, VLAN, and protocol */ 653 total_bytes += fm10k_process_skb_fields(rx_ring, rx_desc, skb); 654 655 fm10k_receive_skb(q_vector, skb); 656 657 /* reset skb pointer */ 658 skb = NULL; 659 660 /* update budget accounting */ 661 total_packets++; 662 } 663 664 /* place incomplete frames back on ring for completion */ 665 rx_ring->skb = skb; 666 667 u64_stats_update_begin(&rx_ring->syncp); 668 rx_ring->stats.packets += total_packets; 669 rx_ring->stats.bytes += total_bytes; 670 u64_stats_update_end(&rx_ring->syncp); 671 q_vector->rx.total_packets += total_packets; 672 q_vector->rx.total_bytes += total_bytes; 673 674 return total_packets < budget; 675 } 676 677 #define VXLAN_HLEN (sizeof(struct udphdr) + 8) 678 static struct ethhdr *fm10k_port_is_vxlan(struct sk_buff *skb) 679 { 680 struct fm10k_intfc *interface = netdev_priv(skb->dev); 681 struct fm10k_vxlan_port *vxlan_port; 682 683 /* we can only offload a vxlan if we recognize it as such */ 684 vxlan_port = list_first_entry_or_null(&interface->vxlan_port, 685 struct fm10k_vxlan_port, list); 686 687 if (!vxlan_port) 688 return NULL; 689 if (vxlan_port->port != udp_hdr(skb)->dest) 690 return NULL; 691 692 /* return offset of udp_hdr plus 8 bytes for VXLAN header */ 693 return (struct ethhdr *)(skb_transport_header(skb) + VXLAN_HLEN); 694 } 695 696 #define FM10K_NVGRE_RESERVED0_FLAGS htons(0x9FFF) 697 #define NVGRE_TNI htons(0x2000) 698 struct fm10k_nvgre_hdr { 699 __be16 flags; 700 __be16 proto; 701 __be32 tni; 702 }; 703 704 static struct ethhdr *fm10k_gre_is_nvgre(struct sk_buff *skb) 705 { 706 struct fm10k_nvgre_hdr *nvgre_hdr; 707 int hlen = ip_hdrlen(skb); 708 709 /* currently only IPv4 is supported due to hlen above */ 710 if (vlan_get_protocol(skb) != htons(ETH_P_IP)) 711 return NULL; 712 713 /* our transport header should be NVGRE */ 714 nvgre_hdr = (struct fm10k_nvgre_hdr *)(skb_network_header(skb) + hlen); 715 716 /* verify all reserved flags are 0 */ 717 if (nvgre_hdr->flags & FM10K_NVGRE_RESERVED0_FLAGS) 718 return NULL; 719 720 /* report start of ethernet header */ 721 if (nvgre_hdr->flags & NVGRE_TNI) 722 return (struct ethhdr *)(nvgre_hdr + 1); 723 724 return (struct ethhdr *)(&nvgre_hdr->tni); 725 } 726 727 __be16 fm10k_tx_encap_offload(struct sk_buff *skb) 728 { 729 u8 l4_hdr = 0, inner_l4_hdr = 0, inner_l4_hlen; 730 struct ethhdr *eth_hdr; 731 732 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER || 733 skb->inner_protocol != htons(ETH_P_TEB)) 734 return 0; 735 736 switch (vlan_get_protocol(skb)) { 737 case htons(ETH_P_IP): 738 l4_hdr = ip_hdr(skb)->protocol; 739 break; 740 case htons(ETH_P_IPV6): 741 l4_hdr = ipv6_hdr(skb)->nexthdr; 742 break; 743 default: 744 return 0; 745 } 746 747 switch (l4_hdr) { 748 case IPPROTO_UDP: 749 eth_hdr = fm10k_port_is_vxlan(skb); 750 break; 751 case IPPROTO_GRE: 752 eth_hdr = fm10k_gre_is_nvgre(skb); 753 break; 754 default: 755 return 0; 756 } 757 758 if (!eth_hdr) 759 return 0; 760 761 switch (eth_hdr->h_proto) { 762 case htons(ETH_P_IP): 763 inner_l4_hdr = inner_ip_hdr(skb)->protocol; 764 break; 765 case htons(ETH_P_IPV6): 766 inner_l4_hdr = inner_ipv6_hdr(skb)->nexthdr; 767 break; 768 default: 769 return 0; 770 } 771 772 switch (inner_l4_hdr) { 773 case IPPROTO_TCP: 774 inner_l4_hlen = inner_tcp_hdrlen(skb); 775 break; 776 case IPPROTO_UDP: 777 inner_l4_hlen = 8; 778 break; 779 default: 780 return 0; 781 } 782 783 /* The hardware allows tunnel offloads only if the combined inner and 784 * outer header is 184 bytes or less 785 */ 786 if (skb_inner_transport_header(skb) + inner_l4_hlen - 787 skb_mac_header(skb) > FM10K_TUNNEL_HEADER_LENGTH) 788 return 0; 789 790 return eth_hdr->h_proto; 791 } 792 793 static int fm10k_tso(struct fm10k_ring *tx_ring, 794 struct fm10k_tx_buffer *first) 795 { 796 struct sk_buff *skb = first->skb; 797 struct fm10k_tx_desc *tx_desc; 798 unsigned char *th; 799 u8 hdrlen; 800 801 if (skb->ip_summed != CHECKSUM_PARTIAL) 802 return 0; 803 804 if (!skb_is_gso(skb)) 805 return 0; 806 807 /* compute header lengths */ 808 if (skb->encapsulation) { 809 if (!fm10k_tx_encap_offload(skb)) 810 goto err_vxlan; 811 th = skb_inner_transport_header(skb); 812 } else { 813 th = skb_transport_header(skb); 814 } 815 816 /* compute offset from SOF to transport header and add header len */ 817 hdrlen = (th - skb->data) + (((struct tcphdr *)th)->doff << 2); 818 819 first->tx_flags |= FM10K_TX_FLAGS_CSUM; 820 821 /* update gso size and bytecount with header size */ 822 first->gso_segs = skb_shinfo(skb)->gso_segs; 823 first->bytecount += (first->gso_segs - 1) * hdrlen; 824 825 /* populate Tx descriptor header size and mss */ 826 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use); 827 tx_desc->hdrlen = hdrlen; 828 tx_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size); 829 830 return 1; 831 err_vxlan: 832 tx_ring->netdev->features &= ~NETIF_F_GSO_UDP_TUNNEL; 833 if (!net_ratelimit()) 834 netdev_err(tx_ring->netdev, 835 "TSO requested for unsupported tunnel, disabling offload\n"); 836 return -1; 837 } 838 839 static void fm10k_tx_csum(struct fm10k_ring *tx_ring, 840 struct fm10k_tx_buffer *first) 841 { 842 struct sk_buff *skb = first->skb; 843 struct fm10k_tx_desc *tx_desc; 844 union { 845 struct iphdr *ipv4; 846 struct ipv6hdr *ipv6; 847 u8 *raw; 848 } network_hdr; 849 __be16 protocol; 850 u8 l4_hdr = 0; 851 852 if (skb->ip_summed != CHECKSUM_PARTIAL) 853 goto no_csum; 854 855 if (skb->encapsulation) { 856 protocol = fm10k_tx_encap_offload(skb); 857 if (!protocol) { 858 if (skb_checksum_help(skb)) { 859 dev_warn(tx_ring->dev, 860 "failed to offload encap csum!\n"); 861 tx_ring->tx_stats.csum_err++; 862 } 863 goto no_csum; 864 } 865 network_hdr.raw = skb_inner_network_header(skb); 866 } else { 867 protocol = vlan_get_protocol(skb); 868 network_hdr.raw = skb_network_header(skb); 869 } 870 871 switch (protocol) { 872 case htons(ETH_P_IP): 873 l4_hdr = network_hdr.ipv4->protocol; 874 break; 875 case htons(ETH_P_IPV6): 876 l4_hdr = network_hdr.ipv6->nexthdr; 877 break; 878 default: 879 if (unlikely(net_ratelimit())) { 880 dev_warn(tx_ring->dev, 881 "partial checksum but ip version=%x!\n", 882 protocol); 883 } 884 tx_ring->tx_stats.csum_err++; 885 goto no_csum; 886 } 887 888 switch (l4_hdr) { 889 case IPPROTO_TCP: 890 case IPPROTO_UDP: 891 break; 892 case IPPROTO_GRE: 893 if (skb->encapsulation) 894 break; 895 default: 896 if (unlikely(net_ratelimit())) { 897 dev_warn(tx_ring->dev, 898 "partial checksum but l4 proto=%x!\n", 899 l4_hdr); 900 } 901 tx_ring->tx_stats.csum_err++; 902 goto no_csum; 903 } 904 905 /* update TX checksum flag */ 906 first->tx_flags |= FM10K_TX_FLAGS_CSUM; 907 908 no_csum: 909 /* populate Tx descriptor header size and mss */ 910 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use); 911 tx_desc->hdrlen = 0; 912 tx_desc->mss = 0; 913 } 914 915 #define FM10K_SET_FLAG(_input, _flag, _result) \ 916 ((_flag <= _result) ? \ 917 ((u32)(_input & _flag) * (_result / _flag)) : \ 918 ((u32)(_input & _flag) / (_flag / _result))) 919 920 static u8 fm10k_tx_desc_flags(struct sk_buff *skb, u32 tx_flags) 921 { 922 /* set type for advanced descriptor with frame checksum insertion */ 923 u32 desc_flags = 0; 924 925 /* set timestamping bits */ 926 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 927 likely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) 928 desc_flags |= FM10K_TXD_FLAG_TIME; 929 930 /* set checksum offload bits */ 931 desc_flags |= FM10K_SET_FLAG(tx_flags, FM10K_TX_FLAGS_CSUM, 932 FM10K_TXD_FLAG_CSUM); 933 934 return desc_flags; 935 } 936 937 static bool fm10k_tx_desc_push(struct fm10k_ring *tx_ring, 938 struct fm10k_tx_desc *tx_desc, u16 i, 939 dma_addr_t dma, unsigned int size, u8 desc_flags) 940 { 941 /* set RS and INT for last frame in a cache line */ 942 if ((++i & (FM10K_TXD_WB_FIFO_SIZE - 1)) == 0) 943 desc_flags |= FM10K_TXD_FLAG_RS | FM10K_TXD_FLAG_INT; 944 945 /* record values to descriptor */ 946 tx_desc->buffer_addr = cpu_to_le64(dma); 947 tx_desc->flags = desc_flags; 948 tx_desc->buflen = cpu_to_le16(size); 949 950 /* return true if we just wrapped the ring */ 951 return i == tx_ring->count; 952 } 953 954 static int __fm10k_maybe_stop_tx(struct fm10k_ring *tx_ring, u16 size) 955 { 956 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 957 958 /* Memory barrier before checking head and tail */ 959 smp_mb(); 960 961 /* Check again in a case another CPU has just made room available */ 962 if (likely(fm10k_desc_unused(tx_ring) < size)) 963 return -EBUSY; 964 965 /* A reprieve! - use start_queue because it doesn't call schedule */ 966 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 967 ++tx_ring->tx_stats.restart_queue; 968 return 0; 969 } 970 971 static inline int fm10k_maybe_stop_tx(struct fm10k_ring *tx_ring, u16 size) 972 { 973 if (likely(fm10k_desc_unused(tx_ring) >= size)) 974 return 0; 975 return __fm10k_maybe_stop_tx(tx_ring, size); 976 } 977 978 static void fm10k_tx_map(struct fm10k_ring *tx_ring, 979 struct fm10k_tx_buffer *first) 980 { 981 struct sk_buff *skb = first->skb; 982 struct fm10k_tx_buffer *tx_buffer; 983 struct fm10k_tx_desc *tx_desc; 984 struct skb_frag_struct *frag; 985 unsigned char *data; 986 dma_addr_t dma; 987 unsigned int data_len, size; 988 u32 tx_flags = first->tx_flags; 989 u16 i = tx_ring->next_to_use; 990 u8 flags = fm10k_tx_desc_flags(skb, tx_flags); 991 992 tx_desc = FM10K_TX_DESC(tx_ring, i); 993 994 /* add HW VLAN tag */ 995 if (skb_vlan_tag_present(skb)) 996 tx_desc->vlan = cpu_to_le16(skb_vlan_tag_get(skb)); 997 else 998 tx_desc->vlan = 0; 999 1000 size = skb_headlen(skb); 1001 data = skb->data; 1002 1003 dma = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 1004 1005 data_len = skb->data_len; 1006 tx_buffer = first; 1007 1008 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1009 if (dma_mapping_error(tx_ring->dev, dma)) 1010 goto dma_error; 1011 1012 /* record length, and DMA address */ 1013 dma_unmap_len_set(tx_buffer, len, size); 1014 dma_unmap_addr_set(tx_buffer, dma, dma); 1015 1016 while (unlikely(size > FM10K_MAX_DATA_PER_TXD)) { 1017 if (fm10k_tx_desc_push(tx_ring, tx_desc++, i++, dma, 1018 FM10K_MAX_DATA_PER_TXD, flags)) { 1019 tx_desc = FM10K_TX_DESC(tx_ring, 0); 1020 i = 0; 1021 } 1022 1023 dma += FM10K_MAX_DATA_PER_TXD; 1024 size -= FM10K_MAX_DATA_PER_TXD; 1025 } 1026 1027 if (likely(!data_len)) 1028 break; 1029 1030 if (fm10k_tx_desc_push(tx_ring, tx_desc++, i++, 1031 dma, size, flags)) { 1032 tx_desc = FM10K_TX_DESC(tx_ring, 0); 1033 i = 0; 1034 } 1035 1036 size = skb_frag_size(frag); 1037 data_len -= size; 1038 1039 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 1040 DMA_TO_DEVICE); 1041 1042 tx_buffer = &tx_ring->tx_buffer[i]; 1043 } 1044 1045 /* write last descriptor with LAST bit set */ 1046 flags |= FM10K_TXD_FLAG_LAST; 1047 1048 if (fm10k_tx_desc_push(tx_ring, tx_desc, i++, dma, size, flags)) 1049 i = 0; 1050 1051 /* record bytecount for BQL */ 1052 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1053 1054 /* record SW timestamp if HW timestamp is not available */ 1055 skb_tx_timestamp(first->skb); 1056 1057 /* Force memory writes to complete before letting h/w know there 1058 * are new descriptors to fetch. (Only applicable for weak-ordered 1059 * memory model archs, such as IA-64). 1060 * 1061 * We also need this memory barrier to make certain all of the 1062 * status bits have been updated before next_to_watch is written. 1063 */ 1064 wmb(); 1065 1066 /* set next_to_watch value indicating a packet is present */ 1067 first->next_to_watch = tx_desc; 1068 1069 tx_ring->next_to_use = i; 1070 1071 /* Make sure there is space in the ring for the next send. */ 1072 fm10k_maybe_stop_tx(tx_ring, DESC_NEEDED); 1073 1074 /* notify HW of packet */ 1075 if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) { 1076 writel(i, tx_ring->tail); 1077 1078 /* we need this if more than one processor can write to our tail 1079 * at a time, it synchronizes IO on IA64/Altix systems 1080 */ 1081 mmiowb(); 1082 } 1083 1084 return; 1085 dma_error: 1086 dev_err(tx_ring->dev, "TX DMA map failed\n"); 1087 1088 /* clear dma mappings for failed tx_buffer map */ 1089 for (;;) { 1090 tx_buffer = &tx_ring->tx_buffer[i]; 1091 fm10k_unmap_and_free_tx_resource(tx_ring, tx_buffer); 1092 if (tx_buffer == first) 1093 break; 1094 if (i == 0) 1095 i = tx_ring->count; 1096 i--; 1097 } 1098 1099 tx_ring->next_to_use = i; 1100 } 1101 1102 netdev_tx_t fm10k_xmit_frame_ring(struct sk_buff *skb, 1103 struct fm10k_ring *tx_ring) 1104 { 1105 struct fm10k_tx_buffer *first; 1106 int tso; 1107 u32 tx_flags = 0; 1108 #if PAGE_SIZE > FM10K_MAX_DATA_PER_TXD 1109 unsigned short f; 1110 #endif 1111 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1112 1113 /* need: 1 descriptor per page * PAGE_SIZE/FM10K_MAX_DATA_PER_TXD, 1114 * + 1 desc for skb_headlen/FM10K_MAX_DATA_PER_TXD, 1115 * + 2 desc gap to keep tail from touching head 1116 * otherwise try next time 1117 */ 1118 #if PAGE_SIZE > FM10K_MAX_DATA_PER_TXD 1119 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1120 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 1121 #else 1122 count += skb_shinfo(skb)->nr_frags; 1123 #endif 1124 if (fm10k_maybe_stop_tx(tx_ring, count + 3)) { 1125 tx_ring->tx_stats.tx_busy++; 1126 return NETDEV_TX_BUSY; 1127 } 1128 1129 /* record the location of the first descriptor for this packet */ 1130 first = &tx_ring->tx_buffer[tx_ring->next_to_use]; 1131 first->skb = skb; 1132 first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); 1133 first->gso_segs = 1; 1134 1135 /* record initial flags and protocol */ 1136 first->tx_flags = tx_flags; 1137 1138 tso = fm10k_tso(tx_ring, first); 1139 if (tso < 0) 1140 goto out_drop; 1141 else if (!tso) 1142 fm10k_tx_csum(tx_ring, first); 1143 1144 fm10k_tx_map(tx_ring, first); 1145 1146 return NETDEV_TX_OK; 1147 1148 out_drop: 1149 dev_kfree_skb_any(first->skb); 1150 first->skb = NULL; 1151 1152 return NETDEV_TX_OK; 1153 } 1154 1155 static u64 fm10k_get_tx_completed(struct fm10k_ring *ring) 1156 { 1157 return ring->stats.packets; 1158 } 1159 1160 static u64 fm10k_get_tx_pending(struct fm10k_ring *ring) 1161 { 1162 /* use SW head and tail until we have real hardware */ 1163 u32 head = ring->next_to_clean; 1164 u32 tail = ring->next_to_use; 1165 1166 return ((head <= tail) ? tail : tail + ring->count) - head; 1167 } 1168 1169 bool fm10k_check_tx_hang(struct fm10k_ring *tx_ring) 1170 { 1171 u32 tx_done = fm10k_get_tx_completed(tx_ring); 1172 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 1173 u32 tx_pending = fm10k_get_tx_pending(tx_ring); 1174 1175 clear_check_for_tx_hang(tx_ring); 1176 1177 /* Check for a hung queue, but be thorough. This verifies 1178 * that a transmit has been completed since the previous 1179 * check AND there is at least one packet pending. By 1180 * requiring this to fail twice we avoid races with 1181 * clearing the ARMED bit and conditions where we 1182 * run the check_tx_hang logic with a transmit completion 1183 * pending but without time to complete it yet. 1184 */ 1185 if (!tx_pending || (tx_done_old != tx_done)) { 1186 /* update completed stats and continue */ 1187 tx_ring->tx_stats.tx_done_old = tx_done; 1188 /* reset the countdown */ 1189 clear_bit(__FM10K_HANG_CHECK_ARMED, &tx_ring->state); 1190 1191 return false; 1192 } 1193 1194 /* make sure it is true for two checks in a row */ 1195 return test_and_set_bit(__FM10K_HANG_CHECK_ARMED, &tx_ring->state); 1196 } 1197 1198 /** 1199 * fm10k_tx_timeout_reset - initiate reset due to Tx timeout 1200 * @interface: driver private struct 1201 **/ 1202 void fm10k_tx_timeout_reset(struct fm10k_intfc *interface) 1203 { 1204 /* Do the reset outside of interrupt context */ 1205 if (!test_bit(__FM10K_DOWN, &interface->state)) { 1206 interface->tx_timeout_count++; 1207 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1208 fm10k_service_event_schedule(interface); 1209 } 1210 } 1211 1212 /** 1213 * fm10k_clean_tx_irq - Reclaim resources after transmit completes 1214 * @q_vector: structure containing interrupt and ring information 1215 * @tx_ring: tx ring to clean 1216 **/ 1217 static bool fm10k_clean_tx_irq(struct fm10k_q_vector *q_vector, 1218 struct fm10k_ring *tx_ring) 1219 { 1220 struct fm10k_intfc *interface = q_vector->interface; 1221 struct fm10k_tx_buffer *tx_buffer; 1222 struct fm10k_tx_desc *tx_desc; 1223 unsigned int total_bytes = 0, total_packets = 0; 1224 unsigned int budget = q_vector->tx.work_limit; 1225 unsigned int i = tx_ring->next_to_clean; 1226 1227 if (test_bit(__FM10K_DOWN, &interface->state)) 1228 return true; 1229 1230 tx_buffer = &tx_ring->tx_buffer[i]; 1231 tx_desc = FM10K_TX_DESC(tx_ring, i); 1232 i -= tx_ring->count; 1233 1234 do { 1235 struct fm10k_tx_desc *eop_desc = tx_buffer->next_to_watch; 1236 1237 /* if next_to_watch is not set then there is no work pending */ 1238 if (!eop_desc) 1239 break; 1240 1241 /* prevent any other reads prior to eop_desc */ 1242 read_barrier_depends(); 1243 1244 /* if DD is not set pending work has not been completed */ 1245 if (!(eop_desc->flags & FM10K_TXD_FLAG_DONE)) 1246 break; 1247 1248 /* clear next_to_watch to prevent false hangs */ 1249 tx_buffer->next_to_watch = NULL; 1250 1251 /* update the statistics for this packet */ 1252 total_bytes += tx_buffer->bytecount; 1253 total_packets += tx_buffer->gso_segs; 1254 1255 /* free the skb */ 1256 dev_consume_skb_any(tx_buffer->skb); 1257 1258 /* unmap skb header data */ 1259 dma_unmap_single(tx_ring->dev, 1260 dma_unmap_addr(tx_buffer, dma), 1261 dma_unmap_len(tx_buffer, len), 1262 DMA_TO_DEVICE); 1263 1264 /* clear tx_buffer data */ 1265 tx_buffer->skb = NULL; 1266 dma_unmap_len_set(tx_buffer, len, 0); 1267 1268 /* unmap remaining buffers */ 1269 while (tx_desc != eop_desc) { 1270 tx_buffer++; 1271 tx_desc++; 1272 i++; 1273 if (unlikely(!i)) { 1274 i -= tx_ring->count; 1275 tx_buffer = tx_ring->tx_buffer; 1276 tx_desc = FM10K_TX_DESC(tx_ring, 0); 1277 } 1278 1279 /* unmap any remaining paged data */ 1280 if (dma_unmap_len(tx_buffer, len)) { 1281 dma_unmap_page(tx_ring->dev, 1282 dma_unmap_addr(tx_buffer, dma), 1283 dma_unmap_len(tx_buffer, len), 1284 DMA_TO_DEVICE); 1285 dma_unmap_len_set(tx_buffer, len, 0); 1286 } 1287 } 1288 1289 /* move us one more past the eop_desc for start of next pkt */ 1290 tx_buffer++; 1291 tx_desc++; 1292 i++; 1293 if (unlikely(!i)) { 1294 i -= tx_ring->count; 1295 tx_buffer = tx_ring->tx_buffer; 1296 tx_desc = FM10K_TX_DESC(tx_ring, 0); 1297 } 1298 1299 /* issue prefetch for next Tx descriptor */ 1300 prefetch(tx_desc); 1301 1302 /* update budget accounting */ 1303 budget--; 1304 } while (likely(budget)); 1305 1306 i += tx_ring->count; 1307 tx_ring->next_to_clean = i; 1308 u64_stats_update_begin(&tx_ring->syncp); 1309 tx_ring->stats.bytes += total_bytes; 1310 tx_ring->stats.packets += total_packets; 1311 u64_stats_update_end(&tx_ring->syncp); 1312 q_vector->tx.total_bytes += total_bytes; 1313 q_vector->tx.total_packets += total_packets; 1314 1315 if (check_for_tx_hang(tx_ring) && fm10k_check_tx_hang(tx_ring)) { 1316 /* schedule immediate reset if we believe we hung */ 1317 struct fm10k_hw *hw = &interface->hw; 1318 1319 netif_err(interface, drv, tx_ring->netdev, 1320 "Detected Tx Unit Hang\n" 1321 " Tx Queue <%d>\n" 1322 " TDH, TDT <%x>, <%x>\n" 1323 " next_to_use <%x>\n" 1324 " next_to_clean <%x>\n", 1325 tx_ring->queue_index, 1326 fm10k_read_reg(hw, FM10K_TDH(tx_ring->reg_idx)), 1327 fm10k_read_reg(hw, FM10K_TDT(tx_ring->reg_idx)), 1328 tx_ring->next_to_use, i); 1329 1330 netif_stop_subqueue(tx_ring->netdev, 1331 tx_ring->queue_index); 1332 1333 netif_info(interface, probe, tx_ring->netdev, 1334 "tx hang %d detected on queue %d, resetting interface\n", 1335 interface->tx_timeout_count + 1, 1336 tx_ring->queue_index); 1337 1338 fm10k_tx_timeout_reset(interface); 1339 1340 /* the netdev is about to reset, no point in enabling stuff */ 1341 return true; 1342 } 1343 1344 /* notify netdev of completed buffers */ 1345 netdev_tx_completed_queue(txring_txq(tx_ring), 1346 total_packets, total_bytes); 1347 1348 #define TX_WAKE_THRESHOLD min_t(u16, FM10K_MIN_TXD - 1, DESC_NEEDED * 2) 1349 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 1350 (fm10k_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 1351 /* Make sure that anybody stopping the queue after this 1352 * sees the new next_to_clean. 1353 */ 1354 smp_mb(); 1355 if (__netif_subqueue_stopped(tx_ring->netdev, 1356 tx_ring->queue_index) && 1357 !test_bit(__FM10K_DOWN, &interface->state)) { 1358 netif_wake_subqueue(tx_ring->netdev, 1359 tx_ring->queue_index); 1360 ++tx_ring->tx_stats.restart_queue; 1361 } 1362 } 1363 1364 return !!budget; 1365 } 1366 1367 /** 1368 * fm10k_update_itr - update the dynamic ITR value based on packet size 1369 * 1370 * Stores a new ITR value based on strictly on packet size. The 1371 * divisors and thresholds used by this function were determined based 1372 * on theoretical maximum wire speed and testing data, in order to 1373 * minimize response time while increasing bulk throughput. 1374 * 1375 * @ring_container: Container for rings to have ITR updated 1376 **/ 1377 static void fm10k_update_itr(struct fm10k_ring_container *ring_container) 1378 { 1379 unsigned int avg_wire_size, packets; 1380 1381 /* Only update ITR if we are using adaptive setting */ 1382 if (!(ring_container->itr & FM10K_ITR_ADAPTIVE)) 1383 goto clear_counts; 1384 1385 packets = ring_container->total_packets; 1386 if (!packets) 1387 goto clear_counts; 1388 1389 avg_wire_size = ring_container->total_bytes / packets; 1390 1391 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 1392 avg_wire_size += 24; 1393 1394 /* Don't starve jumbo frames */ 1395 if (avg_wire_size > 3000) 1396 avg_wire_size = 3000; 1397 1398 /* Give a little boost to mid-size frames */ 1399 if ((avg_wire_size > 300) && (avg_wire_size < 1200)) 1400 avg_wire_size /= 3; 1401 else 1402 avg_wire_size /= 2; 1403 1404 /* write back value and retain adaptive flag */ 1405 ring_container->itr = avg_wire_size | FM10K_ITR_ADAPTIVE; 1406 1407 clear_counts: 1408 ring_container->total_bytes = 0; 1409 ring_container->total_packets = 0; 1410 } 1411 1412 static void fm10k_qv_enable(struct fm10k_q_vector *q_vector) 1413 { 1414 /* Enable auto-mask and clear the current mask */ 1415 u32 itr = FM10K_ITR_ENABLE; 1416 1417 /* Update Tx ITR */ 1418 fm10k_update_itr(&q_vector->tx); 1419 1420 /* Update Rx ITR */ 1421 fm10k_update_itr(&q_vector->rx); 1422 1423 /* Store Tx itr in timer slot 0 */ 1424 itr |= (q_vector->tx.itr & FM10K_ITR_MAX); 1425 1426 /* Shift Rx itr to timer slot 1 */ 1427 itr |= (q_vector->rx.itr & FM10K_ITR_MAX) << FM10K_ITR_INTERVAL1_SHIFT; 1428 1429 /* Write the final value to the ITR register */ 1430 writel(itr, q_vector->itr); 1431 } 1432 1433 static int fm10k_poll(struct napi_struct *napi, int budget) 1434 { 1435 struct fm10k_q_vector *q_vector = 1436 container_of(napi, struct fm10k_q_vector, napi); 1437 struct fm10k_ring *ring; 1438 int per_ring_budget; 1439 bool clean_complete = true; 1440 1441 fm10k_for_each_ring(ring, q_vector->tx) 1442 clean_complete &= fm10k_clean_tx_irq(q_vector, ring); 1443 1444 /* attempt to distribute budget to each queue fairly, but don't 1445 * allow the budget to go below 1 because we'll exit polling 1446 */ 1447 if (q_vector->rx.count > 1) 1448 per_ring_budget = max(budget/q_vector->rx.count, 1); 1449 else 1450 per_ring_budget = budget; 1451 1452 fm10k_for_each_ring(ring, q_vector->rx) 1453 clean_complete &= fm10k_clean_rx_irq(q_vector, ring, 1454 per_ring_budget); 1455 1456 /* If all work not completed, return budget and keep polling */ 1457 if (!clean_complete) 1458 return budget; 1459 1460 /* all work done, exit the polling mode */ 1461 napi_complete(napi); 1462 1463 /* re-enable the q_vector */ 1464 fm10k_qv_enable(q_vector); 1465 1466 return 0; 1467 } 1468 1469 /** 1470 * fm10k_set_qos_queues: Allocate queues for a QOS-enabled device 1471 * @interface: board private structure to initialize 1472 * 1473 * When QoS (Quality of Service) is enabled, allocate queues for 1474 * each traffic class. If multiqueue isn't available,then abort QoS 1475 * initialization. 1476 * 1477 * This function handles all combinations of Qos and RSS. 1478 * 1479 **/ 1480 static bool fm10k_set_qos_queues(struct fm10k_intfc *interface) 1481 { 1482 struct net_device *dev = interface->netdev; 1483 struct fm10k_ring_feature *f; 1484 int rss_i, i; 1485 int pcs; 1486 1487 /* Map queue offset and counts onto allocated tx queues */ 1488 pcs = netdev_get_num_tc(dev); 1489 1490 if (pcs <= 1) 1491 return false; 1492 1493 /* set QoS mask and indices */ 1494 f = &interface->ring_feature[RING_F_QOS]; 1495 f->indices = pcs; 1496 f->mask = (1 << fls(pcs - 1)) - 1; 1497 1498 /* determine the upper limit for our current DCB mode */ 1499 rss_i = interface->hw.mac.max_queues / pcs; 1500 rss_i = 1 << (fls(rss_i) - 1); 1501 1502 /* set RSS mask and indices */ 1503 f = &interface->ring_feature[RING_F_RSS]; 1504 rss_i = min_t(u16, rss_i, f->limit); 1505 f->indices = rss_i; 1506 f->mask = (1 << fls(rss_i - 1)) - 1; 1507 1508 /* configure pause class to queue mapping */ 1509 for (i = 0; i < pcs; i++) 1510 netdev_set_tc_queue(dev, i, rss_i, rss_i * i); 1511 1512 interface->num_rx_queues = rss_i * pcs; 1513 interface->num_tx_queues = rss_i * pcs; 1514 1515 return true; 1516 } 1517 1518 /** 1519 * fm10k_set_rss_queues: Allocate queues for RSS 1520 * @interface: board private structure to initialize 1521 * 1522 * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try 1523 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU. 1524 * 1525 **/ 1526 static bool fm10k_set_rss_queues(struct fm10k_intfc *interface) 1527 { 1528 struct fm10k_ring_feature *f; 1529 u16 rss_i; 1530 1531 f = &interface->ring_feature[RING_F_RSS]; 1532 rss_i = min_t(u16, interface->hw.mac.max_queues, f->limit); 1533 1534 /* record indices and power of 2 mask for RSS */ 1535 f->indices = rss_i; 1536 f->mask = (1 << fls(rss_i - 1)) - 1; 1537 1538 interface->num_rx_queues = rss_i; 1539 interface->num_tx_queues = rss_i; 1540 1541 return true; 1542 } 1543 1544 /** 1545 * fm10k_set_num_queues: Allocate queues for device, feature dependent 1546 * @interface: board private structure to initialize 1547 * 1548 * This is the top level queue allocation routine. The order here is very 1549 * important, starting with the "most" number of features turned on at once, 1550 * and ending with the smallest set of features. This way large combinations 1551 * can be allocated if they're turned on, and smaller combinations are the 1552 * fallthrough conditions. 1553 * 1554 **/ 1555 static void fm10k_set_num_queues(struct fm10k_intfc *interface) 1556 { 1557 /* Start with base case */ 1558 interface->num_rx_queues = 1; 1559 interface->num_tx_queues = 1; 1560 1561 if (fm10k_set_qos_queues(interface)) 1562 return; 1563 1564 fm10k_set_rss_queues(interface); 1565 } 1566 1567 /** 1568 * fm10k_alloc_q_vector - Allocate memory for a single interrupt vector 1569 * @interface: board private structure to initialize 1570 * @v_count: q_vectors allocated on interface, used for ring interleaving 1571 * @v_idx: index of vector in interface struct 1572 * @txr_count: total number of Tx rings to allocate 1573 * @txr_idx: index of first Tx ring to allocate 1574 * @rxr_count: total number of Rx rings to allocate 1575 * @rxr_idx: index of first Rx ring to allocate 1576 * 1577 * We allocate one q_vector. If allocation fails we return -ENOMEM. 1578 **/ 1579 static int fm10k_alloc_q_vector(struct fm10k_intfc *interface, 1580 unsigned int v_count, unsigned int v_idx, 1581 unsigned int txr_count, unsigned int txr_idx, 1582 unsigned int rxr_count, unsigned int rxr_idx) 1583 { 1584 struct fm10k_q_vector *q_vector; 1585 struct fm10k_ring *ring; 1586 int ring_count, size; 1587 1588 ring_count = txr_count + rxr_count; 1589 size = sizeof(struct fm10k_q_vector) + 1590 (sizeof(struct fm10k_ring) * ring_count); 1591 1592 /* allocate q_vector and rings */ 1593 q_vector = kzalloc(size, GFP_KERNEL); 1594 if (!q_vector) 1595 return -ENOMEM; 1596 1597 /* initialize NAPI */ 1598 netif_napi_add(interface->netdev, &q_vector->napi, 1599 fm10k_poll, NAPI_POLL_WEIGHT); 1600 1601 /* tie q_vector and interface together */ 1602 interface->q_vector[v_idx] = q_vector; 1603 q_vector->interface = interface; 1604 q_vector->v_idx = v_idx; 1605 1606 /* initialize pointer to rings */ 1607 ring = q_vector->ring; 1608 1609 /* save Tx ring container info */ 1610 q_vector->tx.ring = ring; 1611 q_vector->tx.work_limit = FM10K_DEFAULT_TX_WORK; 1612 q_vector->tx.itr = interface->tx_itr; 1613 q_vector->tx.count = txr_count; 1614 1615 while (txr_count) { 1616 /* assign generic ring traits */ 1617 ring->dev = &interface->pdev->dev; 1618 ring->netdev = interface->netdev; 1619 1620 /* configure backlink on ring */ 1621 ring->q_vector = q_vector; 1622 1623 /* apply Tx specific ring traits */ 1624 ring->count = interface->tx_ring_count; 1625 ring->queue_index = txr_idx; 1626 1627 /* assign ring to interface */ 1628 interface->tx_ring[txr_idx] = ring; 1629 1630 /* update count and index */ 1631 txr_count--; 1632 txr_idx += v_count; 1633 1634 /* push pointer to next ring */ 1635 ring++; 1636 } 1637 1638 /* save Rx ring container info */ 1639 q_vector->rx.ring = ring; 1640 q_vector->rx.itr = interface->rx_itr; 1641 q_vector->rx.count = rxr_count; 1642 1643 while (rxr_count) { 1644 /* assign generic ring traits */ 1645 ring->dev = &interface->pdev->dev; 1646 ring->netdev = interface->netdev; 1647 rcu_assign_pointer(ring->l2_accel, interface->l2_accel); 1648 1649 /* configure backlink on ring */ 1650 ring->q_vector = q_vector; 1651 1652 /* apply Rx specific ring traits */ 1653 ring->count = interface->rx_ring_count; 1654 ring->queue_index = rxr_idx; 1655 1656 /* assign ring to interface */ 1657 interface->rx_ring[rxr_idx] = ring; 1658 1659 /* update count and index */ 1660 rxr_count--; 1661 rxr_idx += v_count; 1662 1663 /* push pointer to next ring */ 1664 ring++; 1665 } 1666 1667 fm10k_dbg_q_vector_init(q_vector); 1668 1669 return 0; 1670 } 1671 1672 /** 1673 * fm10k_free_q_vector - Free memory allocated for specific interrupt vector 1674 * @interface: board private structure to initialize 1675 * @v_idx: Index of vector to be freed 1676 * 1677 * This function frees the memory allocated to the q_vector. In addition if 1678 * NAPI is enabled it will delete any references to the NAPI struct prior 1679 * to freeing the q_vector. 1680 **/ 1681 static void fm10k_free_q_vector(struct fm10k_intfc *interface, int v_idx) 1682 { 1683 struct fm10k_q_vector *q_vector = interface->q_vector[v_idx]; 1684 struct fm10k_ring *ring; 1685 1686 fm10k_dbg_q_vector_exit(q_vector); 1687 1688 fm10k_for_each_ring(ring, q_vector->tx) 1689 interface->tx_ring[ring->queue_index] = NULL; 1690 1691 fm10k_for_each_ring(ring, q_vector->rx) 1692 interface->rx_ring[ring->queue_index] = NULL; 1693 1694 interface->q_vector[v_idx] = NULL; 1695 netif_napi_del(&q_vector->napi); 1696 kfree_rcu(q_vector, rcu); 1697 } 1698 1699 /** 1700 * fm10k_alloc_q_vectors - Allocate memory for interrupt vectors 1701 * @interface: board private structure to initialize 1702 * 1703 * We allocate one q_vector per queue interrupt. If allocation fails we 1704 * return -ENOMEM. 1705 **/ 1706 static int fm10k_alloc_q_vectors(struct fm10k_intfc *interface) 1707 { 1708 unsigned int q_vectors = interface->num_q_vectors; 1709 unsigned int rxr_remaining = interface->num_rx_queues; 1710 unsigned int txr_remaining = interface->num_tx_queues; 1711 unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0; 1712 int err; 1713 1714 if (q_vectors >= (rxr_remaining + txr_remaining)) { 1715 for (; rxr_remaining; v_idx++) { 1716 err = fm10k_alloc_q_vector(interface, q_vectors, v_idx, 1717 0, 0, 1, rxr_idx); 1718 if (err) 1719 goto err_out; 1720 1721 /* update counts and index */ 1722 rxr_remaining--; 1723 rxr_idx++; 1724 } 1725 } 1726 1727 for (; v_idx < q_vectors; v_idx++) { 1728 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 1729 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 1730 1731 err = fm10k_alloc_q_vector(interface, q_vectors, v_idx, 1732 tqpv, txr_idx, 1733 rqpv, rxr_idx); 1734 1735 if (err) 1736 goto err_out; 1737 1738 /* update counts and index */ 1739 rxr_remaining -= rqpv; 1740 txr_remaining -= tqpv; 1741 rxr_idx++; 1742 txr_idx++; 1743 } 1744 1745 return 0; 1746 1747 err_out: 1748 interface->num_tx_queues = 0; 1749 interface->num_rx_queues = 0; 1750 interface->num_q_vectors = 0; 1751 1752 while (v_idx--) 1753 fm10k_free_q_vector(interface, v_idx); 1754 1755 return -ENOMEM; 1756 } 1757 1758 /** 1759 * fm10k_free_q_vectors - Free memory allocated for interrupt vectors 1760 * @interface: board private structure to initialize 1761 * 1762 * This function frees the memory allocated to the q_vectors. In addition if 1763 * NAPI is enabled it will delete any references to the NAPI struct prior 1764 * to freeing the q_vector. 1765 **/ 1766 static void fm10k_free_q_vectors(struct fm10k_intfc *interface) 1767 { 1768 int v_idx = interface->num_q_vectors; 1769 1770 interface->num_tx_queues = 0; 1771 interface->num_rx_queues = 0; 1772 interface->num_q_vectors = 0; 1773 1774 while (v_idx--) 1775 fm10k_free_q_vector(interface, v_idx); 1776 } 1777 1778 /** 1779 * f10k_reset_msix_capability - reset MSI-X capability 1780 * @interface: board private structure to initialize 1781 * 1782 * Reset the MSI-X capability back to its starting state 1783 **/ 1784 static void fm10k_reset_msix_capability(struct fm10k_intfc *interface) 1785 { 1786 pci_disable_msix(interface->pdev); 1787 kfree(interface->msix_entries); 1788 interface->msix_entries = NULL; 1789 } 1790 1791 /** 1792 * f10k_init_msix_capability - configure MSI-X capability 1793 * @interface: board private structure to initialize 1794 * 1795 * Attempt to configure the interrupts using the best available 1796 * capabilities of the hardware and the kernel. 1797 **/ 1798 static int fm10k_init_msix_capability(struct fm10k_intfc *interface) 1799 { 1800 struct fm10k_hw *hw = &interface->hw; 1801 int v_budget, vector; 1802 1803 /* It's easy to be greedy for MSI-X vectors, but it really 1804 * doesn't do us much good if we have a lot more vectors 1805 * than CPU's. So let's be conservative and only ask for 1806 * (roughly) the same number of vectors as there are CPU's. 1807 * the default is to use pairs of vectors 1808 */ 1809 v_budget = max(interface->num_rx_queues, interface->num_tx_queues); 1810 v_budget = min_t(u16, v_budget, num_online_cpus()); 1811 1812 /* account for vectors not related to queues */ 1813 v_budget += NON_Q_VECTORS(hw); 1814 1815 /* At the same time, hardware can only support a maximum of 1816 * hw.mac->max_msix_vectors vectors. With features 1817 * such as RSS and VMDq, we can easily surpass the number of Rx and Tx 1818 * descriptor queues supported by our device. Thus, we cap it off in 1819 * those rare cases where the cpu count also exceeds our vector limit. 1820 */ 1821 v_budget = min_t(int, v_budget, hw->mac.max_msix_vectors); 1822 1823 /* A failure in MSI-X entry allocation is fatal. */ 1824 interface->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 1825 GFP_KERNEL); 1826 if (!interface->msix_entries) 1827 return -ENOMEM; 1828 1829 /* populate entry values */ 1830 for (vector = 0; vector < v_budget; vector++) 1831 interface->msix_entries[vector].entry = vector; 1832 1833 /* Attempt to enable MSI-X with requested value */ 1834 v_budget = pci_enable_msix_range(interface->pdev, 1835 interface->msix_entries, 1836 MIN_MSIX_COUNT(hw), 1837 v_budget); 1838 if (v_budget < 0) { 1839 kfree(interface->msix_entries); 1840 interface->msix_entries = NULL; 1841 return -ENOMEM; 1842 } 1843 1844 /* record the number of queues available for q_vectors */ 1845 interface->num_q_vectors = v_budget - NON_Q_VECTORS(hw); 1846 1847 return 0; 1848 } 1849 1850 /** 1851 * fm10k_cache_ring_qos - Descriptor ring to register mapping for QoS 1852 * @interface: Interface structure continaining rings and devices 1853 * 1854 * Cache the descriptor ring offsets for Qos 1855 **/ 1856 static bool fm10k_cache_ring_qos(struct fm10k_intfc *interface) 1857 { 1858 struct net_device *dev = interface->netdev; 1859 int pc, offset, rss_i, i, q_idx; 1860 u16 pc_stride = interface->ring_feature[RING_F_QOS].mask + 1; 1861 u8 num_pcs = netdev_get_num_tc(dev); 1862 1863 if (num_pcs <= 1) 1864 return false; 1865 1866 rss_i = interface->ring_feature[RING_F_RSS].indices; 1867 1868 for (pc = 0, offset = 0; pc < num_pcs; pc++, offset += rss_i) { 1869 q_idx = pc; 1870 for (i = 0; i < rss_i; i++) { 1871 interface->tx_ring[offset + i]->reg_idx = q_idx; 1872 interface->tx_ring[offset + i]->qos_pc = pc; 1873 interface->rx_ring[offset + i]->reg_idx = q_idx; 1874 interface->rx_ring[offset + i]->qos_pc = pc; 1875 q_idx += pc_stride; 1876 } 1877 } 1878 1879 return true; 1880 } 1881 1882 /** 1883 * fm10k_cache_ring_rss - Descriptor ring to register mapping for RSS 1884 * @interface: Interface structure continaining rings and devices 1885 * 1886 * Cache the descriptor ring offsets for RSS 1887 **/ 1888 static void fm10k_cache_ring_rss(struct fm10k_intfc *interface) 1889 { 1890 int i; 1891 1892 for (i = 0; i < interface->num_rx_queues; i++) 1893 interface->rx_ring[i]->reg_idx = i; 1894 1895 for (i = 0; i < interface->num_tx_queues; i++) 1896 interface->tx_ring[i]->reg_idx = i; 1897 } 1898 1899 /** 1900 * fm10k_assign_rings - Map rings to network devices 1901 * @interface: Interface structure containing rings and devices 1902 * 1903 * This function is meant to go though and configure both the network 1904 * devices so that they contain rings, and configure the rings so that 1905 * they function with their network devices. 1906 **/ 1907 static void fm10k_assign_rings(struct fm10k_intfc *interface) 1908 { 1909 if (fm10k_cache_ring_qos(interface)) 1910 return; 1911 1912 fm10k_cache_ring_rss(interface); 1913 } 1914 1915 static void fm10k_init_reta(struct fm10k_intfc *interface) 1916 { 1917 u16 i, rss_i = interface->ring_feature[RING_F_RSS].indices; 1918 u32 reta, base; 1919 1920 /* If the netdev is initialized we have to maintain table if possible */ 1921 if (interface->netdev->reg_state) { 1922 for (i = FM10K_RETA_SIZE; i--;) { 1923 reta = interface->reta[i]; 1924 if ((((reta << 24) >> 24) < rss_i) && 1925 (((reta << 16) >> 24) < rss_i) && 1926 (((reta << 8) >> 24) < rss_i) && 1927 (((reta) >> 24) < rss_i)) 1928 continue; 1929 goto repopulate_reta; 1930 } 1931 1932 /* do nothing if all of the elements are in bounds */ 1933 return; 1934 } 1935 1936 repopulate_reta: 1937 /* Populate the redirection table 4 entries at a time. To do this 1938 * we are generating the results for n and n+2 and then interleaving 1939 * those with the results with n+1 and n+3. 1940 */ 1941 for (i = FM10K_RETA_SIZE; i--;) { 1942 /* first pass generates n and n+2 */ 1943 base = ((i * 0x00040004) + 0x00020000) * rss_i; 1944 reta = (base & 0x3F803F80) >> 7; 1945 1946 /* second pass generates n+1 and n+3 */ 1947 base += 0x00010001 * rss_i; 1948 reta |= (base & 0x3F803F80) << 1; 1949 1950 interface->reta[i] = reta; 1951 } 1952 } 1953 1954 /** 1955 * fm10k_init_queueing_scheme - Determine proper queueing scheme 1956 * @interface: board private structure to initialize 1957 * 1958 * We determine which queueing scheme to use based on... 1959 * - Hardware queue count (num_*_queues) 1960 * - defined by miscellaneous hardware support/features (RSS, etc.) 1961 **/ 1962 int fm10k_init_queueing_scheme(struct fm10k_intfc *interface) 1963 { 1964 int err; 1965 1966 /* Number of supported queues */ 1967 fm10k_set_num_queues(interface); 1968 1969 /* Configure MSI-X capability */ 1970 err = fm10k_init_msix_capability(interface); 1971 if (err) { 1972 dev_err(&interface->pdev->dev, 1973 "Unable to initialize MSI-X capability\n"); 1974 return err; 1975 } 1976 1977 /* Allocate memory for queues */ 1978 err = fm10k_alloc_q_vectors(interface); 1979 if (err) 1980 return err; 1981 1982 /* Map rings to devices, and map devices to physical queues */ 1983 fm10k_assign_rings(interface); 1984 1985 /* Initialize RSS redirection table */ 1986 fm10k_init_reta(interface); 1987 1988 return 0; 1989 } 1990 1991 /** 1992 * fm10k_clear_queueing_scheme - Clear the current queueing scheme settings 1993 * @interface: board private structure to clear queueing scheme on 1994 * 1995 * We go through and clear queueing specific resources and reset the structure 1996 * to pre-load conditions 1997 **/ 1998 void fm10k_clear_queueing_scheme(struct fm10k_intfc *interface) 1999 { 2000 fm10k_free_q_vectors(interface); 2001 fm10k_reset_msix_capability(interface); 2002 } 2003