1 /******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2014 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 26 *******************************************************************************/ 27 28 29 /****************************************************************************** 30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 31 ******************************************************************************/ 32 33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35 #include <linux/types.h> 36 #include <linux/bitops.h> 37 #include <linux/module.h> 38 #include <linux/pci.h> 39 #include <linux/netdevice.h> 40 #include <linux/vmalloc.h> 41 #include <linux/string.h> 42 #include <linux/in.h> 43 #include <linux/ip.h> 44 #include <linux/tcp.h> 45 #include <linux/sctp.h> 46 #include <linux/ipv6.h> 47 #include <linux/slab.h> 48 #include <net/checksum.h> 49 #include <net/ip6_checksum.h> 50 #include <linux/ethtool.h> 51 #include <linux/if.h> 52 #include <linux/if_vlan.h> 53 #include <linux/prefetch.h> 54 55 #include "ixgbevf.h" 56 57 const char ixgbevf_driver_name[] = "ixgbevf"; 58 static const char ixgbevf_driver_string[] = 59 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 60 61 #define DRV_VERSION "2.12.1-k" 62 const char ixgbevf_driver_version[] = DRV_VERSION; 63 static char ixgbevf_copyright[] = 64 "Copyright (c) 2009 - 2012 Intel Corporation."; 65 66 static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 67 [board_82599_vf] = &ixgbevf_82599_vf_info, 68 [board_X540_vf] = &ixgbevf_X540_vf_info, 69 [board_X550_vf] = &ixgbevf_X550_vf_info, 70 [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info, 71 }; 72 73 /* ixgbevf_pci_tbl - PCI Device ID Table 74 * 75 * Wildcard entries (PCI_ANY_ID) should come last 76 * Last entry must be all 0s 77 * 78 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 79 * Class, Class Mask, private data (not used) } 80 */ 81 static const struct pci_device_id ixgbevf_pci_tbl[] = { 82 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf }, 83 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf }, 84 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf }, 85 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf }, 86 /* required last entry */ 87 {0, } 88 }; 89 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 90 91 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 92 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); 93 MODULE_LICENSE("GPL"); 94 MODULE_VERSION(DRV_VERSION); 95 96 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 97 static int debug = -1; 98 module_param(debug, int, 0); 99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 100 101 static void ixgbevf_service_event_schedule(struct ixgbevf_adapter *adapter) 102 { 103 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 104 !test_bit(__IXGBEVF_REMOVING, &adapter->state) && 105 !test_and_set_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)) 106 schedule_work(&adapter->service_task); 107 } 108 109 static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter) 110 { 111 BUG_ON(!test_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)); 112 113 /* flush memory to make sure state is correct before next watchdog */ 114 smp_mb__before_atomic(); 115 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 116 } 117 118 /* forward decls */ 119 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); 120 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); 121 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); 122 123 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) 124 { 125 struct ixgbevf_adapter *adapter = hw->back; 126 127 if (!hw->hw_addr) 128 return; 129 hw->hw_addr = NULL; 130 dev_err(&adapter->pdev->dev, "Adapter removed\n"); 131 if (test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 132 ixgbevf_service_event_schedule(adapter); 133 } 134 135 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg) 136 { 137 u32 value; 138 139 /* The following check not only optimizes a bit by not 140 * performing a read on the status register when the 141 * register just read was a status register read that 142 * returned IXGBE_FAILED_READ_REG. It also blocks any 143 * potential recursion. 144 */ 145 if (reg == IXGBE_VFSTATUS) { 146 ixgbevf_remove_adapter(hw); 147 return; 148 } 149 value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS); 150 if (value == IXGBE_FAILED_READ_REG) 151 ixgbevf_remove_adapter(hw); 152 } 153 154 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg) 155 { 156 u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr); 157 u32 value; 158 159 if (IXGBE_REMOVED(reg_addr)) 160 return IXGBE_FAILED_READ_REG; 161 value = readl(reg_addr + reg); 162 if (unlikely(value == IXGBE_FAILED_READ_REG)) 163 ixgbevf_check_remove(hw, reg); 164 return value; 165 } 166 167 /** 168 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 169 * @adapter: pointer to adapter struct 170 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 171 * @queue: queue to map the corresponding interrupt to 172 * @msix_vector: the vector to map to the corresponding queue 173 */ 174 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 175 u8 queue, u8 msix_vector) 176 { 177 u32 ivar, index; 178 struct ixgbe_hw *hw = &adapter->hw; 179 if (direction == -1) { 180 /* other causes */ 181 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 182 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 183 ivar &= ~0xFF; 184 ivar |= msix_vector; 185 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 186 } else { 187 /* tx or rx causes */ 188 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 189 index = ((16 * (queue & 1)) + (8 * direction)); 190 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 191 ivar &= ~(0xFF << index); 192 ivar |= (msix_vector << index); 193 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 194 } 195 } 196 197 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring, 198 struct ixgbevf_tx_buffer *tx_buffer) 199 { 200 if (tx_buffer->skb) { 201 dev_kfree_skb_any(tx_buffer->skb); 202 if (dma_unmap_len(tx_buffer, len)) 203 dma_unmap_single(tx_ring->dev, 204 dma_unmap_addr(tx_buffer, dma), 205 dma_unmap_len(tx_buffer, len), 206 DMA_TO_DEVICE); 207 } else if (dma_unmap_len(tx_buffer, len)) { 208 dma_unmap_page(tx_ring->dev, 209 dma_unmap_addr(tx_buffer, dma), 210 dma_unmap_len(tx_buffer, len), 211 DMA_TO_DEVICE); 212 } 213 tx_buffer->next_to_watch = NULL; 214 tx_buffer->skb = NULL; 215 dma_unmap_len_set(tx_buffer, len, 0); 216 /* tx_buffer must be completely set up in the transmit path */ 217 } 218 219 static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring) 220 { 221 return ring->stats.packets; 222 } 223 224 static u32 ixgbevf_get_tx_pending(struct ixgbevf_ring *ring) 225 { 226 struct ixgbevf_adapter *adapter = netdev_priv(ring->netdev); 227 struct ixgbe_hw *hw = &adapter->hw; 228 229 u32 head = IXGBE_READ_REG(hw, IXGBE_VFTDH(ring->reg_idx)); 230 u32 tail = IXGBE_READ_REG(hw, IXGBE_VFTDT(ring->reg_idx)); 231 232 if (head != tail) 233 return (head < tail) ? 234 tail - head : (tail + ring->count - head); 235 236 return 0; 237 } 238 239 static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring) 240 { 241 u32 tx_done = ixgbevf_get_tx_completed(tx_ring); 242 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 243 u32 tx_pending = ixgbevf_get_tx_pending(tx_ring); 244 245 clear_check_for_tx_hang(tx_ring); 246 247 /* Check for a hung queue, but be thorough. This verifies 248 * that a transmit has been completed since the previous 249 * check AND there is at least one packet pending. The 250 * ARMED bit is set to indicate a potential hang. 251 */ 252 if ((tx_done_old == tx_done) && tx_pending) { 253 /* make sure it is true for two checks in a row */ 254 return test_and_set_bit(__IXGBEVF_HANG_CHECK_ARMED, 255 &tx_ring->state); 256 } 257 /* reset the countdown */ 258 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &tx_ring->state); 259 260 /* update completed stats and continue */ 261 tx_ring->tx_stats.tx_done_old = tx_done; 262 263 return false; 264 } 265 266 static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter) 267 { 268 /* Do the reset outside of interrupt context */ 269 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 270 adapter->flags |= IXGBEVF_FLAG_RESET_REQUESTED; 271 ixgbevf_service_event_schedule(adapter); 272 } 273 } 274 275 /** 276 * ixgbevf_tx_timeout - Respond to a Tx Hang 277 * @netdev: network interface device structure 278 **/ 279 static void ixgbevf_tx_timeout(struct net_device *netdev) 280 { 281 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 282 283 ixgbevf_tx_timeout_reset(adapter); 284 } 285 286 /** 287 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 288 * @q_vector: board private structure 289 * @tx_ring: tx ring to clean 290 **/ 291 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, 292 struct ixgbevf_ring *tx_ring) 293 { 294 struct ixgbevf_adapter *adapter = q_vector->adapter; 295 struct ixgbevf_tx_buffer *tx_buffer; 296 union ixgbe_adv_tx_desc *tx_desc; 297 unsigned int total_bytes = 0, total_packets = 0; 298 unsigned int budget = tx_ring->count / 2; 299 unsigned int i = tx_ring->next_to_clean; 300 301 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 302 return true; 303 304 tx_buffer = &tx_ring->tx_buffer_info[i]; 305 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 306 i -= tx_ring->count; 307 308 do { 309 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 310 311 /* if next_to_watch is not set then there is no work pending */ 312 if (!eop_desc) 313 break; 314 315 /* prevent any other reads prior to eop_desc */ 316 read_barrier_depends(); 317 318 /* if DD is not set pending work has not been completed */ 319 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 320 break; 321 322 /* clear next_to_watch to prevent false hangs */ 323 tx_buffer->next_to_watch = NULL; 324 325 /* update the statistics for this packet */ 326 total_bytes += tx_buffer->bytecount; 327 total_packets += tx_buffer->gso_segs; 328 329 /* free the skb */ 330 dev_kfree_skb_any(tx_buffer->skb); 331 332 /* unmap skb header data */ 333 dma_unmap_single(tx_ring->dev, 334 dma_unmap_addr(tx_buffer, dma), 335 dma_unmap_len(tx_buffer, len), 336 DMA_TO_DEVICE); 337 338 /* clear tx_buffer data */ 339 tx_buffer->skb = NULL; 340 dma_unmap_len_set(tx_buffer, len, 0); 341 342 /* unmap remaining buffers */ 343 while (tx_desc != eop_desc) { 344 tx_buffer++; 345 tx_desc++; 346 i++; 347 if (unlikely(!i)) { 348 i -= tx_ring->count; 349 tx_buffer = tx_ring->tx_buffer_info; 350 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 351 } 352 353 /* unmap any remaining paged data */ 354 if (dma_unmap_len(tx_buffer, len)) { 355 dma_unmap_page(tx_ring->dev, 356 dma_unmap_addr(tx_buffer, dma), 357 dma_unmap_len(tx_buffer, len), 358 DMA_TO_DEVICE); 359 dma_unmap_len_set(tx_buffer, len, 0); 360 } 361 } 362 363 /* move us one more past the eop_desc for start of next pkt */ 364 tx_buffer++; 365 tx_desc++; 366 i++; 367 if (unlikely(!i)) { 368 i -= tx_ring->count; 369 tx_buffer = tx_ring->tx_buffer_info; 370 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 371 } 372 373 /* issue prefetch for next Tx descriptor */ 374 prefetch(tx_desc); 375 376 /* update budget accounting */ 377 budget--; 378 } while (likely(budget)); 379 380 i += tx_ring->count; 381 tx_ring->next_to_clean = i; 382 u64_stats_update_begin(&tx_ring->syncp); 383 tx_ring->stats.bytes += total_bytes; 384 tx_ring->stats.packets += total_packets; 385 u64_stats_update_end(&tx_ring->syncp); 386 q_vector->tx.total_bytes += total_bytes; 387 q_vector->tx.total_packets += total_packets; 388 389 if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) { 390 struct ixgbe_hw *hw = &adapter->hw; 391 union ixgbe_adv_tx_desc *eop_desc; 392 393 eop_desc = tx_ring->tx_buffer_info[i].next_to_watch; 394 395 pr_err("Detected Tx Unit Hang\n" 396 " Tx Queue <%d>\n" 397 " TDH, TDT <%x>, <%x>\n" 398 " next_to_use <%x>\n" 399 " next_to_clean <%x>\n" 400 "tx_buffer_info[next_to_clean]\n" 401 " next_to_watch <%p>\n" 402 " eop_desc->wb.status <%x>\n" 403 " time_stamp <%lx>\n" 404 " jiffies <%lx>\n", 405 tx_ring->queue_index, 406 IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)), 407 IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)), 408 tx_ring->next_to_use, i, 409 eop_desc, (eop_desc ? eop_desc->wb.status : 0), 410 tx_ring->tx_buffer_info[i].time_stamp, jiffies); 411 412 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 413 414 /* schedule immediate reset if we believe we hung */ 415 ixgbevf_tx_timeout_reset(adapter); 416 417 return true; 418 } 419 420 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 421 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 422 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 423 /* Make sure that anybody stopping the queue after this 424 * sees the new next_to_clean. 425 */ 426 smp_mb(); 427 428 if (__netif_subqueue_stopped(tx_ring->netdev, 429 tx_ring->queue_index) && 430 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 431 netif_wake_subqueue(tx_ring->netdev, 432 tx_ring->queue_index); 433 ++tx_ring->tx_stats.restart_queue; 434 } 435 } 436 437 return !!budget; 438 } 439 440 /** 441 * ixgbevf_rx_skb - Helper function to determine proper Rx method 442 * @q_vector: structure containing interrupt and ring information 443 * @skb: packet to send up 444 **/ 445 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, 446 struct sk_buff *skb) 447 { 448 #ifdef CONFIG_NET_RX_BUSY_POLL 449 skb_mark_napi_id(skb, &q_vector->napi); 450 451 if (ixgbevf_qv_busy_polling(q_vector)) { 452 netif_receive_skb(skb); 453 /* exit early if we busy polled */ 454 return; 455 } 456 #endif /* CONFIG_NET_RX_BUSY_POLL */ 457 458 napi_gro_receive(&q_vector->napi, skb); 459 } 460 461 /* ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 462 * @ring: structure containig ring specific data 463 * @rx_desc: current Rx descriptor being processed 464 * @skb: skb currently being received and modified 465 */ 466 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, 467 union ixgbe_adv_rx_desc *rx_desc, 468 struct sk_buff *skb) 469 { 470 skb_checksum_none_assert(skb); 471 472 /* Rx csum disabled */ 473 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 474 return; 475 476 /* if IP and error */ 477 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 478 ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 479 ring->rx_stats.csum_err++; 480 return; 481 } 482 483 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 484 return; 485 486 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 487 ring->rx_stats.csum_err++; 488 return; 489 } 490 491 /* It must be a TCP or UDP packet with a valid checksum */ 492 skb->ip_summed = CHECKSUM_UNNECESSARY; 493 } 494 495 /* ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor 496 * @rx_ring: rx descriptor ring packet is being transacted on 497 * @rx_desc: pointer to the EOP Rx descriptor 498 * @skb: pointer to current skb being populated 499 * 500 * This function checks the ring, descriptor, and packet information in 501 * order to populate the checksum, VLAN, protocol, and other fields within 502 * the skb. 503 */ 504 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, 505 union ixgbe_adv_rx_desc *rx_desc, 506 struct sk_buff *skb) 507 { 508 ixgbevf_rx_checksum(rx_ring, rx_desc, skb); 509 510 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 511 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 512 unsigned long *active_vlans = netdev_priv(rx_ring->netdev); 513 514 if (test_bit(vid & VLAN_VID_MASK, active_vlans)) 515 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 516 } 517 518 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 519 } 520 521 /** 522 * ixgbevf_is_non_eop - process handling of non-EOP buffers 523 * @rx_ring: Rx ring being processed 524 * @rx_desc: Rx descriptor for current buffer 525 * @skb: current socket buffer containing buffer in progress 526 * 527 * This function updates next to clean. If the buffer is an EOP buffer 528 * this function exits returning false, otherwise it will place the 529 * sk_buff in the next buffer to be chained and return true indicating 530 * that this is in fact a non-EOP buffer. 531 **/ 532 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, 533 union ixgbe_adv_rx_desc *rx_desc) 534 { 535 u32 ntc = rx_ring->next_to_clean + 1; 536 537 /* fetch, update, and store next to clean */ 538 ntc = (ntc < rx_ring->count) ? ntc : 0; 539 rx_ring->next_to_clean = ntc; 540 541 prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); 542 543 if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 544 return false; 545 546 return true; 547 } 548 549 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, 550 struct ixgbevf_rx_buffer *bi) 551 { 552 struct page *page = bi->page; 553 dma_addr_t dma = bi->dma; 554 555 /* since we are recycling buffers we should seldom need to alloc */ 556 if (likely(page)) 557 return true; 558 559 /* alloc new page for storage */ 560 page = dev_alloc_page(); 561 if (unlikely(!page)) { 562 rx_ring->rx_stats.alloc_rx_page_failed++; 563 return false; 564 } 565 566 /* map page for use */ 567 dma = dma_map_page(rx_ring->dev, page, 0, 568 PAGE_SIZE, DMA_FROM_DEVICE); 569 570 /* if mapping failed free memory back to system since 571 * there isn't much point in holding memory we can't use 572 */ 573 if (dma_mapping_error(rx_ring->dev, dma)) { 574 __free_page(page); 575 576 rx_ring->rx_stats.alloc_rx_buff_failed++; 577 return false; 578 } 579 580 bi->dma = dma; 581 bi->page = page; 582 bi->page_offset = 0; 583 584 return true; 585 } 586 587 /** 588 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 589 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on 590 * @cleaned_count: number of buffers to replace 591 **/ 592 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, 593 u16 cleaned_count) 594 { 595 union ixgbe_adv_rx_desc *rx_desc; 596 struct ixgbevf_rx_buffer *bi; 597 unsigned int i = rx_ring->next_to_use; 598 599 /* nothing to do or no valid netdev defined */ 600 if (!cleaned_count || !rx_ring->netdev) 601 return; 602 603 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 604 bi = &rx_ring->rx_buffer_info[i]; 605 i -= rx_ring->count; 606 607 do { 608 if (!ixgbevf_alloc_mapped_page(rx_ring, bi)) 609 break; 610 611 /* Refresh the desc even if pkt_addr didn't change 612 * because each write-back erases this info. 613 */ 614 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 615 616 rx_desc++; 617 bi++; 618 i++; 619 if (unlikely(!i)) { 620 rx_desc = IXGBEVF_RX_DESC(rx_ring, 0); 621 bi = rx_ring->rx_buffer_info; 622 i -= rx_ring->count; 623 } 624 625 /* clear the hdr_addr for the next_to_use descriptor */ 626 rx_desc->read.hdr_addr = 0; 627 628 cleaned_count--; 629 } while (cleaned_count); 630 631 i += rx_ring->count; 632 633 if (rx_ring->next_to_use != i) { 634 /* record the next descriptor to use */ 635 rx_ring->next_to_use = i; 636 637 /* update next to alloc since we have filled the ring */ 638 rx_ring->next_to_alloc = i; 639 640 /* Force memory writes to complete before letting h/w 641 * know there are new descriptors to fetch. (Only 642 * applicable for weak-ordered memory model archs, 643 * such as IA-64). 644 */ 645 wmb(); 646 ixgbevf_write_tail(rx_ring, i); 647 } 648 } 649 650 /* ixgbevf_pull_tail - ixgbevf specific version of skb_pull_tail 651 * @rx_ring: rx descriptor ring packet is being transacted on 652 * @skb: pointer to current skb being adjusted 653 * 654 * This function is an ixgbevf specific version of __pskb_pull_tail. The 655 * main difference between this version and the original function is that 656 * this function can make several assumptions about the state of things 657 * that allow for significant optimizations versus the standard function. 658 * As a result we can do things like drop a frag and maintain an accurate 659 * truesize for the skb. 660 */ 661 static void ixgbevf_pull_tail(struct ixgbevf_ring *rx_ring, 662 struct sk_buff *skb) 663 { 664 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 665 unsigned char *va; 666 unsigned int pull_len; 667 668 /* it is valid to use page_address instead of kmap since we are 669 * working with pages allocated out of the lomem pool per 670 * alloc_page(GFP_ATOMIC) 671 */ 672 va = skb_frag_address(frag); 673 674 /* we need the header to contain the greater of either ETH_HLEN or 675 * 60 bytes if the skb->len is less than 60 for skb_pad. 676 */ 677 pull_len = eth_get_headlen(va, IXGBEVF_RX_HDR_SIZE); 678 679 /* align pull length to size of long to optimize memcpy performance */ 680 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); 681 682 /* update all of the pointers */ 683 skb_frag_size_sub(frag, pull_len); 684 frag->page_offset += pull_len; 685 skb->data_len -= pull_len; 686 skb->tail += pull_len; 687 } 688 689 /* ixgbevf_cleanup_headers - Correct corrupted or empty headers 690 * @rx_ring: rx descriptor ring packet is being transacted on 691 * @rx_desc: pointer to the EOP Rx descriptor 692 * @skb: pointer to current skb being fixed 693 * 694 * Check for corrupted packet headers caused by senders on the local L2 695 * embedded NIC switch not setting up their Tx Descriptors right. These 696 * should be very rare. 697 * 698 * Also address the case where we are pulling data in on pages only 699 * and as such no data is present in the skb header. 700 * 701 * In addition if skb is not at least 60 bytes we need to pad it so that 702 * it is large enough to qualify as a valid Ethernet frame. 703 * 704 * Returns true if an error was encountered and skb was freed. 705 */ 706 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, 707 union ixgbe_adv_rx_desc *rx_desc, 708 struct sk_buff *skb) 709 { 710 /* verify that the packet does not have any known errors */ 711 if (unlikely(ixgbevf_test_staterr(rx_desc, 712 IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { 713 struct net_device *netdev = rx_ring->netdev; 714 715 if (!(netdev->features & NETIF_F_RXALL)) { 716 dev_kfree_skb_any(skb); 717 return true; 718 } 719 } 720 721 /* place header in linear portion of buffer */ 722 if (skb_is_nonlinear(skb)) 723 ixgbevf_pull_tail(rx_ring, skb); 724 725 /* if eth_skb_pad returns an error the skb was freed */ 726 if (eth_skb_pad(skb)) 727 return true; 728 729 return false; 730 } 731 732 /* ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring 733 * @rx_ring: rx descriptor ring to store buffers on 734 * @old_buff: donor buffer to have page reused 735 * 736 * Synchronizes page for reuse by the adapter 737 */ 738 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, 739 struct ixgbevf_rx_buffer *old_buff) 740 { 741 struct ixgbevf_rx_buffer *new_buff; 742 u16 nta = rx_ring->next_to_alloc; 743 744 new_buff = &rx_ring->rx_buffer_info[nta]; 745 746 /* update, and store next to alloc */ 747 nta++; 748 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 749 750 /* transfer page from old buffer to new buffer */ 751 new_buff->page = old_buff->page; 752 new_buff->dma = old_buff->dma; 753 new_buff->page_offset = old_buff->page_offset; 754 755 /* sync the buffer for use by the device */ 756 dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma, 757 new_buff->page_offset, 758 IXGBEVF_RX_BUFSZ, 759 DMA_FROM_DEVICE); 760 } 761 762 static inline bool ixgbevf_page_is_reserved(struct page *page) 763 { 764 return (page_to_nid(page) != numa_mem_id()) || page->pfmemalloc; 765 } 766 767 /* ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff 768 * @rx_ring: rx descriptor ring to transact packets on 769 * @rx_buffer: buffer containing page to add 770 * @rx_desc: descriptor containing length of buffer written by hardware 771 * @skb: sk_buff to place the data into 772 * 773 * This function will add the data contained in rx_buffer->page to the skb. 774 * This is done either through a direct copy if the data in the buffer is 775 * less than the skb header size, otherwise it will just attach the page as 776 * a frag to the skb. 777 * 778 * The function will then update the page offset if necessary and return 779 * true if the buffer can be reused by the adapter. 780 */ 781 static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, 782 struct ixgbevf_rx_buffer *rx_buffer, 783 union ixgbe_adv_rx_desc *rx_desc, 784 struct sk_buff *skb) 785 { 786 struct page *page = rx_buffer->page; 787 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length); 788 #if (PAGE_SIZE < 8192) 789 unsigned int truesize = IXGBEVF_RX_BUFSZ; 790 #else 791 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES); 792 #endif 793 794 if ((size <= IXGBEVF_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) { 795 unsigned char *va = page_address(page) + rx_buffer->page_offset; 796 797 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 798 799 /* page is not reserved, we can reuse buffer as is */ 800 if (likely(!ixgbevf_page_is_reserved(page))) 801 return true; 802 803 /* this page cannot be reused so discard it */ 804 put_page(page); 805 return false; 806 } 807 808 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 809 rx_buffer->page_offset, size, truesize); 810 811 /* avoid re-using remote pages */ 812 if (unlikely(ixgbevf_page_is_reserved(page))) 813 return false; 814 815 #if (PAGE_SIZE < 8192) 816 /* if we are only owner of page we can reuse it */ 817 if (unlikely(page_count(page) != 1)) 818 return false; 819 820 /* flip page offset to other buffer */ 821 rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ; 822 823 #else 824 /* move offset up to the next cache line */ 825 rx_buffer->page_offset += truesize; 826 827 if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ)) 828 return false; 829 830 #endif 831 /* Even if we own the page, we are not allowed to use atomic_set() 832 * This would break get_page_unless_zero() users. 833 */ 834 atomic_inc(&page->_count); 835 836 return true; 837 } 838 839 static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring, 840 union ixgbe_adv_rx_desc *rx_desc, 841 struct sk_buff *skb) 842 { 843 struct ixgbevf_rx_buffer *rx_buffer; 844 struct page *page; 845 846 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 847 page = rx_buffer->page; 848 prefetchw(page); 849 850 if (likely(!skb)) { 851 void *page_addr = page_address(page) + 852 rx_buffer->page_offset; 853 854 /* prefetch first cache line of first page */ 855 prefetch(page_addr); 856 #if L1_CACHE_BYTES < 128 857 prefetch(page_addr + L1_CACHE_BYTES); 858 #endif 859 860 /* allocate a skb to store the frags */ 861 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 862 IXGBEVF_RX_HDR_SIZE); 863 if (unlikely(!skb)) { 864 rx_ring->rx_stats.alloc_rx_buff_failed++; 865 return NULL; 866 } 867 868 /* we will be copying header into skb->data in 869 * pskb_may_pull so it is in our interest to prefetch 870 * it now to avoid a possible cache miss 871 */ 872 prefetchw(skb->data); 873 } 874 875 /* we are reusing so sync this buffer for CPU use */ 876 dma_sync_single_range_for_cpu(rx_ring->dev, 877 rx_buffer->dma, 878 rx_buffer->page_offset, 879 IXGBEVF_RX_BUFSZ, 880 DMA_FROM_DEVICE); 881 882 /* pull page into skb */ 883 if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) { 884 /* hand second half of page back to the ring */ 885 ixgbevf_reuse_rx_page(rx_ring, rx_buffer); 886 } else { 887 /* we are not reusing the buffer so unmap it */ 888 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 889 PAGE_SIZE, DMA_FROM_DEVICE); 890 } 891 892 /* clear contents of buffer_info */ 893 rx_buffer->dma = 0; 894 rx_buffer->page = NULL; 895 896 return skb; 897 } 898 899 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 900 u32 qmask) 901 { 902 struct ixgbe_hw *hw = &adapter->hw; 903 904 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); 905 } 906 907 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 908 struct ixgbevf_ring *rx_ring, 909 int budget) 910 { 911 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 912 u16 cleaned_count = ixgbevf_desc_unused(rx_ring); 913 struct sk_buff *skb = rx_ring->skb; 914 915 while (likely(total_rx_packets < budget)) { 916 union ixgbe_adv_rx_desc *rx_desc; 917 918 /* return some buffers to hardware, one at a time is too slow */ 919 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 920 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count); 921 cleaned_count = 0; 922 } 923 924 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); 925 926 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD)) 927 break; 928 929 /* This memory barrier is needed to keep us from reading 930 * any other fields out of the rx_desc until we know the 931 * RXD_STAT_DD bit is set 932 */ 933 rmb(); 934 935 /* retrieve a buffer from the ring */ 936 skb = ixgbevf_fetch_rx_buffer(rx_ring, rx_desc, skb); 937 938 /* exit if we failed to retrieve a buffer */ 939 if (!skb) 940 break; 941 942 cleaned_count++; 943 944 /* fetch next buffer in frame if non-eop */ 945 if (ixgbevf_is_non_eop(rx_ring, rx_desc)) 946 continue; 947 948 /* verify the packet layout is correct */ 949 if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) { 950 skb = NULL; 951 continue; 952 } 953 954 /* probably a little skewed due to removing CRC */ 955 total_rx_bytes += skb->len; 956 957 /* Workaround hardware that can't do proper VEPA multicast 958 * source pruning. 959 */ 960 if ((skb->pkt_type == PACKET_BROADCAST || 961 skb->pkt_type == PACKET_MULTICAST) && 962 ether_addr_equal(rx_ring->netdev->dev_addr, 963 eth_hdr(skb)->h_source)) { 964 dev_kfree_skb_irq(skb); 965 continue; 966 } 967 968 /* populate checksum, VLAN, and protocol */ 969 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); 970 971 ixgbevf_rx_skb(q_vector, skb); 972 973 /* reset skb pointer */ 974 skb = NULL; 975 976 /* update budget accounting */ 977 total_rx_packets++; 978 } 979 980 /* place incomplete frames back on ring for completion */ 981 rx_ring->skb = skb; 982 983 u64_stats_update_begin(&rx_ring->syncp); 984 rx_ring->stats.packets += total_rx_packets; 985 rx_ring->stats.bytes += total_rx_bytes; 986 u64_stats_update_end(&rx_ring->syncp); 987 q_vector->rx.total_packets += total_rx_packets; 988 q_vector->rx.total_bytes += total_rx_bytes; 989 990 return total_rx_packets; 991 } 992 993 /** 994 * ixgbevf_poll - NAPI polling calback 995 * @napi: napi struct with our devices info in it 996 * @budget: amount of work driver is allowed to do this pass, in packets 997 * 998 * This function will clean more than one or more rings associated with a 999 * q_vector. 1000 **/ 1001 static int ixgbevf_poll(struct napi_struct *napi, int budget) 1002 { 1003 struct ixgbevf_q_vector *q_vector = 1004 container_of(napi, struct ixgbevf_q_vector, napi); 1005 struct ixgbevf_adapter *adapter = q_vector->adapter; 1006 struct ixgbevf_ring *ring; 1007 int per_ring_budget; 1008 bool clean_complete = true; 1009 1010 ixgbevf_for_each_ring(ring, q_vector->tx) 1011 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring); 1012 1013 #ifdef CONFIG_NET_RX_BUSY_POLL 1014 if (!ixgbevf_qv_lock_napi(q_vector)) 1015 return budget; 1016 #endif 1017 1018 /* attempt to distribute budget to each queue fairly, but don't allow 1019 * the budget to go below 1 because we'll exit polling */ 1020 if (q_vector->rx.count > 1) 1021 per_ring_budget = max(budget/q_vector->rx.count, 1); 1022 else 1023 per_ring_budget = budget; 1024 1025 ixgbevf_for_each_ring(ring, q_vector->rx) 1026 clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring, 1027 per_ring_budget) 1028 < per_ring_budget); 1029 1030 #ifdef CONFIG_NET_RX_BUSY_POLL 1031 ixgbevf_qv_unlock_napi(q_vector); 1032 #endif 1033 1034 /* If all work not completed, return budget and keep polling */ 1035 if (!clean_complete) 1036 return budget; 1037 /* all work done, exit the polling mode */ 1038 napi_complete(napi); 1039 if (adapter->rx_itr_setting & 1) 1040 ixgbevf_set_itr(q_vector); 1041 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 1042 !test_bit(__IXGBEVF_REMOVING, &adapter->state)) 1043 ixgbevf_irq_enable_queues(adapter, 1044 1 << q_vector->v_idx); 1045 1046 return 0; 1047 } 1048 1049 /** 1050 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 1051 * @q_vector: structure containing interrupt and ring information 1052 */ 1053 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) 1054 { 1055 struct ixgbevf_adapter *adapter = q_vector->adapter; 1056 struct ixgbe_hw *hw = &adapter->hw; 1057 int v_idx = q_vector->v_idx; 1058 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 1059 1060 /* 1061 * set the WDIS bit to not clear the timer bits and cause an 1062 * immediate assertion of the interrupt 1063 */ 1064 itr_reg |= IXGBE_EITR_CNT_WDIS; 1065 1066 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 1067 } 1068 1069 #ifdef CONFIG_NET_RX_BUSY_POLL 1070 /* must be called with local_bh_disable()d */ 1071 static int ixgbevf_busy_poll_recv(struct napi_struct *napi) 1072 { 1073 struct ixgbevf_q_vector *q_vector = 1074 container_of(napi, struct ixgbevf_q_vector, napi); 1075 struct ixgbevf_adapter *adapter = q_vector->adapter; 1076 struct ixgbevf_ring *ring; 1077 int found = 0; 1078 1079 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 1080 return LL_FLUSH_FAILED; 1081 1082 if (!ixgbevf_qv_lock_poll(q_vector)) 1083 return LL_FLUSH_BUSY; 1084 1085 ixgbevf_for_each_ring(ring, q_vector->rx) { 1086 found = ixgbevf_clean_rx_irq(q_vector, ring, 4); 1087 #ifdef BP_EXTENDED_STATS 1088 if (found) 1089 ring->stats.cleaned += found; 1090 else 1091 ring->stats.misses++; 1092 #endif 1093 if (found) 1094 break; 1095 } 1096 1097 ixgbevf_qv_unlock_poll(q_vector); 1098 1099 return found; 1100 } 1101 #endif /* CONFIG_NET_RX_BUSY_POLL */ 1102 1103 /** 1104 * ixgbevf_configure_msix - Configure MSI-X hardware 1105 * @adapter: board private structure 1106 * 1107 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 1108 * interrupts. 1109 **/ 1110 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 1111 { 1112 struct ixgbevf_q_vector *q_vector; 1113 int q_vectors, v_idx; 1114 1115 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1116 adapter->eims_enable_mask = 0; 1117 1118 /* 1119 * Populate the IVAR table and set the ITR values to the 1120 * corresponding register. 1121 */ 1122 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 1123 struct ixgbevf_ring *ring; 1124 q_vector = adapter->q_vector[v_idx]; 1125 1126 ixgbevf_for_each_ring(ring, q_vector->rx) 1127 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx); 1128 1129 ixgbevf_for_each_ring(ring, q_vector->tx) 1130 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx); 1131 1132 if (q_vector->tx.ring && !q_vector->rx.ring) { 1133 /* tx only vector */ 1134 if (adapter->tx_itr_setting == 1) 1135 q_vector->itr = IXGBE_10K_ITR; 1136 else 1137 q_vector->itr = adapter->tx_itr_setting; 1138 } else { 1139 /* rx or rx/tx vector */ 1140 if (adapter->rx_itr_setting == 1) 1141 q_vector->itr = IXGBE_20K_ITR; 1142 else 1143 q_vector->itr = adapter->rx_itr_setting; 1144 } 1145 1146 /* add q_vector eims value to global eims_enable_mask */ 1147 adapter->eims_enable_mask |= 1 << v_idx; 1148 1149 ixgbevf_write_eitr(q_vector); 1150 } 1151 1152 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 1153 /* setup eims_other and add value to global eims_enable_mask */ 1154 adapter->eims_other = 1 << v_idx; 1155 adapter->eims_enable_mask |= adapter->eims_other; 1156 } 1157 1158 enum latency_range { 1159 lowest_latency = 0, 1160 low_latency = 1, 1161 bulk_latency = 2, 1162 latency_invalid = 255 1163 }; 1164 1165 /** 1166 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 1167 * @q_vector: structure containing interrupt and ring information 1168 * @ring_container: structure containing ring performance data 1169 * 1170 * Stores a new ITR value based on packets and byte 1171 * counts during the last interrupt. The advantage of per interrupt 1172 * computation is faster updates and more accurate ITR for the current 1173 * traffic pattern. Constants in this function were computed 1174 * based on theoretical maximum wire speed and thresholds were set based 1175 * on testing data as well as attempting to minimize response time 1176 * while increasing bulk throughput. 1177 **/ 1178 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector, 1179 struct ixgbevf_ring_container *ring_container) 1180 { 1181 int bytes = ring_container->total_bytes; 1182 int packets = ring_container->total_packets; 1183 u32 timepassed_us; 1184 u64 bytes_perint; 1185 u8 itr_setting = ring_container->itr; 1186 1187 if (packets == 0) 1188 return; 1189 1190 /* simple throttlerate management 1191 * 0-20MB/s lowest (100000 ints/s) 1192 * 20-100MB/s low (20000 ints/s) 1193 * 100-1249MB/s bulk (8000 ints/s) 1194 */ 1195 /* what was last interrupt timeslice? */ 1196 timepassed_us = q_vector->itr >> 2; 1197 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 1198 1199 switch (itr_setting) { 1200 case lowest_latency: 1201 if (bytes_perint > 10) 1202 itr_setting = low_latency; 1203 break; 1204 case low_latency: 1205 if (bytes_perint > 20) 1206 itr_setting = bulk_latency; 1207 else if (bytes_perint <= 10) 1208 itr_setting = lowest_latency; 1209 break; 1210 case bulk_latency: 1211 if (bytes_perint <= 20) 1212 itr_setting = low_latency; 1213 break; 1214 } 1215 1216 /* clear work counters since we have the values we need */ 1217 ring_container->total_bytes = 0; 1218 ring_container->total_packets = 0; 1219 1220 /* write updated itr to ring container */ 1221 ring_container->itr = itr_setting; 1222 } 1223 1224 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector) 1225 { 1226 u32 new_itr = q_vector->itr; 1227 u8 current_itr; 1228 1229 ixgbevf_update_itr(q_vector, &q_vector->tx); 1230 ixgbevf_update_itr(q_vector, &q_vector->rx); 1231 1232 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 1233 1234 switch (current_itr) { 1235 /* counts and packets in update_itr are dependent on these numbers */ 1236 case lowest_latency: 1237 new_itr = IXGBE_100K_ITR; 1238 break; 1239 case low_latency: 1240 new_itr = IXGBE_20K_ITR; 1241 break; 1242 case bulk_latency: 1243 default: 1244 new_itr = IXGBE_8K_ITR; 1245 break; 1246 } 1247 1248 if (new_itr != q_vector->itr) { 1249 /* do an exponential smoothing */ 1250 new_itr = (10 * new_itr * q_vector->itr) / 1251 ((9 * new_itr) + q_vector->itr); 1252 1253 /* save the algorithm value here */ 1254 q_vector->itr = new_itr; 1255 1256 ixgbevf_write_eitr(q_vector); 1257 } 1258 } 1259 1260 static irqreturn_t ixgbevf_msix_other(int irq, void *data) 1261 { 1262 struct ixgbevf_adapter *adapter = data; 1263 struct ixgbe_hw *hw = &adapter->hw; 1264 1265 hw->mac.get_link_status = 1; 1266 1267 ixgbevf_service_event_schedule(adapter); 1268 1269 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); 1270 1271 return IRQ_HANDLED; 1272 } 1273 1274 /** 1275 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) 1276 * @irq: unused 1277 * @data: pointer to our q_vector struct for this interrupt vector 1278 **/ 1279 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data) 1280 { 1281 struct ixgbevf_q_vector *q_vector = data; 1282 1283 /* EIAM disabled interrupts (on this vector) for us */ 1284 if (q_vector->rx.ring || q_vector->tx.ring) 1285 napi_schedule(&q_vector->napi); 1286 1287 return IRQ_HANDLED; 1288 } 1289 1290 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, 1291 int r_idx) 1292 { 1293 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1294 1295 a->rx_ring[r_idx]->next = q_vector->rx.ring; 1296 q_vector->rx.ring = a->rx_ring[r_idx]; 1297 q_vector->rx.count++; 1298 } 1299 1300 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, 1301 int t_idx) 1302 { 1303 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1304 1305 a->tx_ring[t_idx]->next = q_vector->tx.ring; 1306 q_vector->tx.ring = a->tx_ring[t_idx]; 1307 q_vector->tx.count++; 1308 } 1309 1310 /** 1311 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors 1312 * @adapter: board private structure to initialize 1313 * 1314 * This function maps descriptor rings to the queue-specific vectors 1315 * we were allotted through the MSI-X enabling code. Ideally, we'd have 1316 * one vector per ring/queue, but on a constrained vector budget, we 1317 * group the rings as "efficiently" as possible. You would add new 1318 * mapping configurations in here. 1319 **/ 1320 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter) 1321 { 1322 int q_vectors; 1323 int v_start = 0; 1324 int rxr_idx = 0, txr_idx = 0; 1325 int rxr_remaining = adapter->num_rx_queues; 1326 int txr_remaining = adapter->num_tx_queues; 1327 int i, j; 1328 int rqpv, tqpv; 1329 int err = 0; 1330 1331 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1332 1333 /* 1334 * The ideal configuration... 1335 * We have enough vectors to map one per queue. 1336 */ 1337 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) { 1338 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) 1339 map_vector_to_rxq(adapter, v_start, rxr_idx); 1340 1341 for (; txr_idx < txr_remaining; v_start++, txr_idx++) 1342 map_vector_to_txq(adapter, v_start, txr_idx); 1343 goto out; 1344 } 1345 1346 /* 1347 * If we don't have enough vectors for a 1-to-1 1348 * mapping, we'll have to group them so there are 1349 * multiple queues per vector. 1350 */ 1351 /* Re-adjusting *qpv takes care of the remainder. */ 1352 for (i = v_start; i < q_vectors; i++) { 1353 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i); 1354 for (j = 0; j < rqpv; j++) { 1355 map_vector_to_rxq(adapter, i, rxr_idx); 1356 rxr_idx++; 1357 rxr_remaining--; 1358 } 1359 } 1360 for (i = v_start; i < q_vectors; i++) { 1361 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i); 1362 for (j = 0; j < tqpv; j++) { 1363 map_vector_to_txq(adapter, i, txr_idx); 1364 txr_idx++; 1365 txr_remaining--; 1366 } 1367 } 1368 1369 out: 1370 return err; 1371 } 1372 1373 /** 1374 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 1375 * @adapter: board private structure 1376 * 1377 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 1378 * interrupts from the kernel. 1379 **/ 1380 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 1381 { 1382 struct net_device *netdev = adapter->netdev; 1383 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1384 int vector, err; 1385 int ri = 0, ti = 0; 1386 1387 for (vector = 0; vector < q_vectors; vector++) { 1388 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector]; 1389 struct msix_entry *entry = &adapter->msix_entries[vector]; 1390 1391 if (q_vector->tx.ring && q_vector->rx.ring) { 1392 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1393 "%s-%s-%d", netdev->name, "TxRx", ri++); 1394 ti++; 1395 } else if (q_vector->rx.ring) { 1396 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1397 "%s-%s-%d", netdev->name, "rx", ri++); 1398 } else if (q_vector->tx.ring) { 1399 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1400 "%s-%s-%d", netdev->name, "tx", ti++); 1401 } else { 1402 /* skip this unused q_vector */ 1403 continue; 1404 } 1405 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0, 1406 q_vector->name, q_vector); 1407 if (err) { 1408 hw_dbg(&adapter->hw, 1409 "request_irq failed for MSIX interrupt " 1410 "Error: %d\n", err); 1411 goto free_queue_irqs; 1412 } 1413 } 1414 1415 err = request_irq(adapter->msix_entries[vector].vector, 1416 &ixgbevf_msix_other, 0, netdev->name, adapter); 1417 if (err) { 1418 hw_dbg(&adapter->hw, 1419 "request_irq for msix_other failed: %d\n", err); 1420 goto free_queue_irqs; 1421 } 1422 1423 return 0; 1424 1425 free_queue_irqs: 1426 while (vector) { 1427 vector--; 1428 free_irq(adapter->msix_entries[vector].vector, 1429 adapter->q_vector[vector]); 1430 } 1431 /* This failure is non-recoverable - it indicates the system is 1432 * out of MSIX vector resources and the VF driver cannot run 1433 * without them. Set the number of msix vectors to zero 1434 * indicating that not enough can be allocated. The error 1435 * will be returned to the user indicating device open failed. 1436 * Any further attempts to force the driver to open will also 1437 * fail. The only way to recover is to unload the driver and 1438 * reload it again. If the system has recovered some MSIX 1439 * vectors then it may succeed. 1440 */ 1441 adapter->num_msix_vectors = 0; 1442 return err; 1443 } 1444 1445 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter) 1446 { 1447 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1448 1449 for (i = 0; i < q_vectors; i++) { 1450 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i]; 1451 q_vector->rx.ring = NULL; 1452 q_vector->tx.ring = NULL; 1453 q_vector->rx.count = 0; 1454 q_vector->tx.count = 0; 1455 } 1456 } 1457 1458 /** 1459 * ixgbevf_request_irq - initialize interrupts 1460 * @adapter: board private structure 1461 * 1462 * Attempts to configure interrupts using the best available 1463 * capabilities of the hardware and kernel. 1464 **/ 1465 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 1466 { 1467 int err = 0; 1468 1469 err = ixgbevf_request_msix_irqs(adapter); 1470 1471 if (err) 1472 hw_dbg(&adapter->hw, 1473 "request_irq failed, Error %d\n", err); 1474 1475 return err; 1476 } 1477 1478 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 1479 { 1480 int i, q_vectors; 1481 1482 q_vectors = adapter->num_msix_vectors; 1483 i = q_vectors - 1; 1484 1485 free_irq(adapter->msix_entries[i].vector, adapter); 1486 i--; 1487 1488 for (; i >= 0; i--) { 1489 /* free only the irqs that were actually requested */ 1490 if (!adapter->q_vector[i]->rx.ring && 1491 !adapter->q_vector[i]->tx.ring) 1492 continue; 1493 1494 free_irq(adapter->msix_entries[i].vector, 1495 adapter->q_vector[i]); 1496 } 1497 1498 ixgbevf_reset_q_vectors(adapter); 1499 } 1500 1501 /** 1502 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1503 * @adapter: board private structure 1504 **/ 1505 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1506 { 1507 struct ixgbe_hw *hw = &adapter->hw; 1508 int i; 1509 1510 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0); 1511 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1512 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0); 1513 1514 IXGBE_WRITE_FLUSH(hw); 1515 1516 for (i = 0; i < adapter->num_msix_vectors; i++) 1517 synchronize_irq(adapter->msix_entries[i].vector); 1518 } 1519 1520 /** 1521 * ixgbevf_irq_enable - Enable default interrupt generation settings 1522 * @adapter: board private structure 1523 **/ 1524 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) 1525 { 1526 struct ixgbe_hw *hw = &adapter->hw; 1527 1528 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask); 1529 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask); 1530 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); 1531 } 1532 1533 /** 1534 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset 1535 * @adapter: board private structure 1536 * @ring: structure containing ring specific data 1537 * 1538 * Configure the Tx descriptor ring after a reset. 1539 **/ 1540 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 1541 struct ixgbevf_ring *ring) 1542 { 1543 struct ixgbe_hw *hw = &adapter->hw; 1544 u64 tdba = ring->dma; 1545 int wait_loop = 10; 1546 u32 txdctl = IXGBE_TXDCTL_ENABLE; 1547 u8 reg_idx = ring->reg_idx; 1548 1549 /* disable queue to avoid issues while updating state */ 1550 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 1551 IXGBE_WRITE_FLUSH(hw); 1552 1553 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 1554 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32); 1555 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx), 1556 ring->count * sizeof(union ixgbe_adv_tx_desc)); 1557 1558 /* disable head writeback */ 1559 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0); 1560 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0); 1561 1562 /* enable relaxed ordering */ 1563 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx), 1564 (IXGBE_DCA_TXCTRL_DESC_RRO_EN | 1565 IXGBE_DCA_TXCTRL_DATA_RRO_EN)); 1566 1567 /* reset head and tail pointers */ 1568 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0); 1569 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0); 1570 ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx); 1571 1572 /* reset ntu and ntc to place SW in sync with hardwdare */ 1573 ring->next_to_clean = 0; 1574 ring->next_to_use = 0; 1575 1576 /* In order to avoid issues WTHRESH + PTHRESH should always be equal 1577 * to or less than the number of on chip descriptors, which is 1578 * currently 40. 1579 */ 1580 txdctl |= (8 << 16); /* WTHRESH = 8 */ 1581 1582 /* Setting PTHRESH to 32 both improves performance */ 1583 txdctl |= (1 << 8) | /* HTHRESH = 1 */ 1584 32; /* PTHRESH = 32 */ 1585 1586 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); 1587 1588 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl); 1589 1590 /* poll to verify queue is enabled */ 1591 do { 1592 usleep_range(1000, 2000); 1593 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx)); 1594 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 1595 if (!wait_loop) 1596 pr_err("Could not enable Tx Queue %d\n", reg_idx); 1597 } 1598 1599 /** 1600 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1601 * @adapter: board private structure 1602 * 1603 * Configure the Tx unit of the MAC after a reset. 1604 **/ 1605 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1606 { 1607 u32 i; 1608 1609 /* Setup the HW Tx Head and Tail descriptor pointers */ 1610 for (i = 0; i < adapter->num_tx_queues; i++) 1611 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]); 1612 } 1613 1614 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1615 1616 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) 1617 { 1618 struct ixgbe_hw *hw = &adapter->hw; 1619 u32 srrctl; 1620 1621 srrctl = IXGBE_SRRCTL_DROP_EN; 1622 1623 srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; 1624 srrctl |= IXGBEVF_RX_BUFSZ >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1625 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1626 1627 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1628 } 1629 1630 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) 1631 { 1632 struct ixgbe_hw *hw = &adapter->hw; 1633 1634 /* PSRTYPE must be initialized in 82599 */ 1635 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 1636 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR | 1637 IXGBE_PSRTYPE_L2HDR; 1638 1639 if (adapter->num_rx_queues > 1) 1640 psrtype |= 1 << 29; 1641 1642 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1643 } 1644 1645 #define IXGBEVF_MAX_RX_DESC_POLL 10 1646 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, 1647 struct ixgbevf_ring *ring) 1648 { 1649 struct ixgbe_hw *hw = &adapter->hw; 1650 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1651 u32 rxdctl; 1652 u8 reg_idx = ring->reg_idx; 1653 1654 if (IXGBE_REMOVED(hw->hw_addr)) 1655 return; 1656 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1657 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1658 1659 /* write value back with RXDCTL.ENABLE bit cleared */ 1660 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1661 1662 /* the hardware may take up to 100us to really disable the rx queue */ 1663 do { 1664 udelay(10); 1665 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1666 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 1667 1668 if (!wait_loop) 1669 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", 1670 reg_idx); 1671 } 1672 1673 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1674 struct ixgbevf_ring *ring) 1675 { 1676 struct ixgbe_hw *hw = &adapter->hw; 1677 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1678 u32 rxdctl; 1679 u8 reg_idx = ring->reg_idx; 1680 1681 if (IXGBE_REMOVED(hw->hw_addr)) 1682 return; 1683 do { 1684 usleep_range(1000, 2000); 1685 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1686 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 1687 1688 if (!wait_loop) 1689 pr_err("RXDCTL.ENABLE queue %d not set while polling\n", 1690 reg_idx); 1691 } 1692 1693 static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) 1694 { 1695 struct ixgbe_hw *hw = &adapter->hw; 1696 u32 vfmrqc = 0, vfreta = 0; 1697 u32 rss_key[10]; 1698 u16 rss_i = adapter->num_rx_queues; 1699 int i, j; 1700 1701 /* Fill out hash function seeds */ 1702 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 1703 for (i = 0; i < 10; i++) 1704 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1705 1706 /* Fill out redirection table */ 1707 for (i = 0, j = 0; i < 64; i++, j++) { 1708 if (j == rss_i) 1709 j = 0; 1710 vfreta = (vfreta << 8) | (j * 0x1); 1711 if ((i & 3) == 3) 1712 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), vfreta); 1713 } 1714 1715 /* Perform hash on these packet types */ 1716 vfmrqc |= IXGBE_VFMRQC_RSS_FIELD_IPV4 | 1717 IXGBE_VFMRQC_RSS_FIELD_IPV4_TCP | 1718 IXGBE_VFMRQC_RSS_FIELD_IPV6 | 1719 IXGBE_VFMRQC_RSS_FIELD_IPV6_TCP; 1720 1721 vfmrqc |= IXGBE_VFMRQC_RSSEN; 1722 1723 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); 1724 } 1725 1726 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, 1727 struct ixgbevf_ring *ring) 1728 { 1729 struct ixgbe_hw *hw = &adapter->hw; 1730 u64 rdba = ring->dma; 1731 u32 rxdctl; 1732 u8 reg_idx = ring->reg_idx; 1733 1734 /* disable queue to avoid issues while updating state */ 1735 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1736 ixgbevf_disable_rx_queue(adapter, ring); 1737 1738 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 1739 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32); 1740 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx), 1741 ring->count * sizeof(union ixgbe_adv_rx_desc)); 1742 1743 /* enable relaxed ordering */ 1744 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1745 IXGBE_DCA_RXCTRL_DESC_RRO_EN); 1746 1747 /* reset head and tail pointers */ 1748 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0); 1749 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); 1750 ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); 1751 1752 /* reset ntu and ntc to place SW in sync with hardwdare */ 1753 ring->next_to_clean = 0; 1754 ring->next_to_use = 0; 1755 ring->next_to_alloc = 0; 1756 1757 ixgbevf_configure_srrctl(adapter, reg_idx); 1758 1759 /* allow any size packet since we can handle overflow */ 1760 rxdctl &= ~IXGBE_RXDCTL_RLPML_EN; 1761 1762 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1763 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1764 1765 ixgbevf_rx_desc_queue_enable(adapter, ring); 1766 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); 1767 } 1768 1769 /** 1770 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1771 * @adapter: board private structure 1772 * 1773 * Configure the Rx unit of the MAC after a reset. 1774 **/ 1775 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1776 { 1777 int i; 1778 struct ixgbe_hw *hw = &adapter->hw; 1779 struct net_device *netdev = adapter->netdev; 1780 1781 ixgbevf_setup_psrtype(adapter); 1782 if (hw->mac.type >= ixgbe_mac_X550_vf) 1783 ixgbevf_setup_vfmrqc(adapter); 1784 1785 /* notify the PF of our intent to use this size of frame */ 1786 ixgbevf_rlpml_set_vf(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN); 1787 1788 /* Setup the HW Rx Head and Tail Descriptor Pointers and 1789 * the Base and Length of the Rx Descriptor Ring */ 1790 for (i = 0; i < adapter->num_rx_queues; i++) 1791 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]); 1792 } 1793 1794 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, 1795 __be16 proto, u16 vid) 1796 { 1797 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1798 struct ixgbe_hw *hw = &adapter->hw; 1799 int err; 1800 1801 spin_lock_bh(&adapter->mbx_lock); 1802 1803 /* add VID to filter table */ 1804 err = hw->mac.ops.set_vfta(hw, vid, 0, true); 1805 1806 spin_unlock_bh(&adapter->mbx_lock); 1807 1808 /* translate error return types so error makes sense */ 1809 if (err == IXGBE_ERR_MBX) 1810 return -EIO; 1811 1812 if (err == IXGBE_ERR_INVALID_ARGUMENT) 1813 return -EACCES; 1814 1815 set_bit(vid, adapter->active_vlans); 1816 1817 return err; 1818 } 1819 1820 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, 1821 __be16 proto, u16 vid) 1822 { 1823 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1824 struct ixgbe_hw *hw = &adapter->hw; 1825 int err = -EOPNOTSUPP; 1826 1827 spin_lock_bh(&adapter->mbx_lock); 1828 1829 /* remove VID from filter table */ 1830 err = hw->mac.ops.set_vfta(hw, vid, 0, false); 1831 1832 spin_unlock_bh(&adapter->mbx_lock); 1833 1834 clear_bit(vid, adapter->active_vlans); 1835 1836 return err; 1837 } 1838 1839 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 1840 { 1841 u16 vid; 1842 1843 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1844 ixgbevf_vlan_rx_add_vid(adapter->netdev, 1845 htons(ETH_P_8021Q), vid); 1846 } 1847 1848 static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 1849 { 1850 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1851 struct ixgbe_hw *hw = &adapter->hw; 1852 int count = 0; 1853 1854 if ((netdev_uc_count(netdev)) > 10) { 1855 pr_err("Too many unicast filters - No Space\n"); 1856 return -ENOSPC; 1857 } 1858 1859 if (!netdev_uc_empty(netdev)) { 1860 struct netdev_hw_addr *ha; 1861 netdev_for_each_uc_addr(ha, netdev) { 1862 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 1863 udelay(200); 1864 } 1865 } else { 1866 /* 1867 * If the list is empty then send message to PF driver to 1868 * clear all macvlans on this VF. 1869 */ 1870 hw->mac.ops.set_uc_addr(hw, 0, NULL); 1871 } 1872 1873 return count; 1874 } 1875 1876 /** 1877 * ixgbevf_set_rx_mode - Multicast and unicast set 1878 * @netdev: network interface device structure 1879 * 1880 * The set_rx_method entry point is called whenever the multicast address 1881 * list, unicast address list or the network interface flags are updated. 1882 * This routine is responsible for configuring the hardware for proper 1883 * multicast mode and configuring requested unicast filters. 1884 **/ 1885 static void ixgbevf_set_rx_mode(struct net_device *netdev) 1886 { 1887 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1888 struct ixgbe_hw *hw = &adapter->hw; 1889 1890 spin_lock_bh(&adapter->mbx_lock); 1891 1892 /* reprogram multicast list */ 1893 hw->mac.ops.update_mc_addr_list(hw, netdev); 1894 1895 ixgbevf_write_uc_addr_list(netdev); 1896 1897 spin_unlock_bh(&adapter->mbx_lock); 1898 } 1899 1900 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 1901 { 1902 int q_idx; 1903 struct ixgbevf_q_vector *q_vector; 1904 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1905 1906 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1907 q_vector = adapter->q_vector[q_idx]; 1908 #ifdef CONFIG_NET_RX_BUSY_POLL 1909 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]); 1910 #endif 1911 napi_enable(&q_vector->napi); 1912 } 1913 } 1914 1915 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 1916 { 1917 int q_idx; 1918 struct ixgbevf_q_vector *q_vector; 1919 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1920 1921 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1922 q_vector = adapter->q_vector[q_idx]; 1923 napi_disable(&q_vector->napi); 1924 #ifdef CONFIG_NET_RX_BUSY_POLL 1925 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) { 1926 pr_info("QV %d locked\n", q_idx); 1927 usleep_range(1000, 20000); 1928 } 1929 #endif /* CONFIG_NET_RX_BUSY_POLL */ 1930 } 1931 } 1932 1933 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter) 1934 { 1935 struct ixgbe_hw *hw = &adapter->hw; 1936 unsigned int def_q = 0; 1937 unsigned int num_tcs = 0; 1938 unsigned int num_rx_queues = adapter->num_rx_queues; 1939 unsigned int num_tx_queues = adapter->num_tx_queues; 1940 int err; 1941 1942 spin_lock_bh(&adapter->mbx_lock); 1943 1944 /* fetch queue configuration from the PF */ 1945 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 1946 1947 spin_unlock_bh(&adapter->mbx_lock); 1948 1949 if (err) 1950 return err; 1951 1952 if (num_tcs > 1) { 1953 /* we need only one Tx queue */ 1954 num_tx_queues = 1; 1955 1956 /* update default Tx ring register index */ 1957 adapter->tx_ring[0]->reg_idx = def_q; 1958 1959 /* we need as many queues as traffic classes */ 1960 num_rx_queues = num_tcs; 1961 } 1962 1963 /* if we have a bad config abort request queue reset */ 1964 if ((adapter->num_rx_queues != num_rx_queues) || 1965 (adapter->num_tx_queues != num_tx_queues)) { 1966 /* force mailbox timeout to prevent further messages */ 1967 hw->mbx.timeout = 0; 1968 1969 /* wait for watchdog to come around and bail us out */ 1970 adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED; 1971 } 1972 1973 return 0; 1974 } 1975 1976 static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 1977 { 1978 ixgbevf_configure_dcb(adapter); 1979 1980 ixgbevf_set_rx_mode(adapter->netdev); 1981 1982 ixgbevf_restore_vlan(adapter); 1983 1984 ixgbevf_configure_tx(adapter); 1985 ixgbevf_configure_rx(adapter); 1986 } 1987 1988 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 1989 { 1990 /* Only save pre-reset stats if there are some */ 1991 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 1992 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 1993 adapter->stats.base_vfgprc; 1994 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 1995 adapter->stats.base_vfgptc; 1996 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 1997 adapter->stats.base_vfgorc; 1998 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 1999 adapter->stats.base_vfgotc; 2000 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 2001 adapter->stats.base_vfmprc; 2002 } 2003 } 2004 2005 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 2006 { 2007 struct ixgbe_hw *hw = &adapter->hw; 2008 2009 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2010 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2011 adapter->stats.last_vfgorc |= 2012 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2013 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2014 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2015 adapter->stats.last_vfgotc |= 2016 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2017 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2018 2019 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 2020 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 2021 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 2022 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 2023 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 2024 } 2025 2026 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) 2027 { 2028 struct ixgbe_hw *hw = &adapter->hw; 2029 int api[] = { ixgbe_mbox_api_11, 2030 ixgbe_mbox_api_10, 2031 ixgbe_mbox_api_unknown }; 2032 int err = 0, idx = 0; 2033 2034 spin_lock_bh(&adapter->mbx_lock); 2035 2036 while (api[idx] != ixgbe_mbox_api_unknown) { 2037 err = ixgbevf_negotiate_api_version(hw, api[idx]); 2038 if (!err) 2039 break; 2040 idx++; 2041 } 2042 2043 spin_unlock_bh(&adapter->mbx_lock); 2044 } 2045 2046 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 2047 { 2048 struct net_device *netdev = adapter->netdev; 2049 struct ixgbe_hw *hw = &adapter->hw; 2050 2051 ixgbevf_configure_msix(adapter); 2052 2053 spin_lock_bh(&adapter->mbx_lock); 2054 2055 if (is_valid_ether_addr(hw->mac.addr)) 2056 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 2057 else 2058 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 2059 2060 spin_unlock_bh(&adapter->mbx_lock); 2061 2062 smp_mb__before_atomic(); 2063 clear_bit(__IXGBEVF_DOWN, &adapter->state); 2064 ixgbevf_napi_enable_all(adapter); 2065 2066 /* clear any pending interrupts, may auto mask */ 2067 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2068 ixgbevf_irq_enable(adapter); 2069 2070 /* enable transmits */ 2071 netif_tx_start_all_queues(netdev); 2072 2073 ixgbevf_save_reset_stats(adapter); 2074 ixgbevf_init_last_counter_stats(adapter); 2075 2076 hw->mac.get_link_status = 1; 2077 mod_timer(&adapter->service_timer, jiffies); 2078 } 2079 2080 void ixgbevf_up(struct ixgbevf_adapter *adapter) 2081 { 2082 ixgbevf_configure(adapter); 2083 2084 ixgbevf_up_complete(adapter); 2085 } 2086 2087 /** 2088 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 2089 * @rx_ring: ring to free buffers from 2090 **/ 2091 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) 2092 { 2093 struct device *dev = rx_ring->dev; 2094 unsigned long size; 2095 unsigned int i; 2096 2097 /* Free Rx ring sk_buff */ 2098 if (rx_ring->skb) { 2099 dev_kfree_skb(rx_ring->skb); 2100 rx_ring->skb = NULL; 2101 } 2102 2103 /* ring already cleared, nothing to do */ 2104 if (!rx_ring->rx_buffer_info) 2105 return; 2106 2107 /* Free all the Rx ring pages */ 2108 for (i = 0; i < rx_ring->count; i++) { 2109 struct ixgbevf_rx_buffer *rx_buffer; 2110 2111 rx_buffer = &rx_ring->rx_buffer_info[i]; 2112 if (rx_buffer->dma) 2113 dma_unmap_page(dev, rx_buffer->dma, 2114 PAGE_SIZE, DMA_FROM_DEVICE); 2115 rx_buffer->dma = 0; 2116 if (rx_buffer->page) 2117 __free_page(rx_buffer->page); 2118 rx_buffer->page = NULL; 2119 } 2120 2121 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 2122 memset(rx_ring->rx_buffer_info, 0, size); 2123 2124 /* Zero out the descriptor ring */ 2125 memset(rx_ring->desc, 0, rx_ring->size); 2126 } 2127 2128 /** 2129 * ixgbevf_clean_tx_ring - Free Tx Buffers 2130 * @tx_ring: ring to be cleaned 2131 **/ 2132 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) 2133 { 2134 struct ixgbevf_tx_buffer *tx_buffer_info; 2135 unsigned long size; 2136 unsigned int i; 2137 2138 if (!tx_ring->tx_buffer_info) 2139 return; 2140 2141 /* Free all the Tx ring sk_buffs */ 2142 for (i = 0; i < tx_ring->count; i++) { 2143 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2144 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 2145 } 2146 2147 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2148 memset(tx_ring->tx_buffer_info, 0, size); 2149 2150 memset(tx_ring->desc, 0, tx_ring->size); 2151 } 2152 2153 /** 2154 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 2155 * @adapter: board private structure 2156 **/ 2157 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 2158 { 2159 int i; 2160 2161 for (i = 0; i < adapter->num_rx_queues; i++) 2162 ixgbevf_clean_rx_ring(adapter->rx_ring[i]); 2163 } 2164 2165 /** 2166 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 2167 * @adapter: board private structure 2168 **/ 2169 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 2170 { 2171 int i; 2172 2173 for (i = 0; i < adapter->num_tx_queues; i++) 2174 ixgbevf_clean_tx_ring(adapter->tx_ring[i]); 2175 } 2176 2177 void ixgbevf_down(struct ixgbevf_adapter *adapter) 2178 { 2179 struct net_device *netdev = adapter->netdev; 2180 struct ixgbe_hw *hw = &adapter->hw; 2181 int i; 2182 2183 /* signal that we are down to the interrupt handler */ 2184 if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state)) 2185 return; /* do nothing if already down */ 2186 2187 /* disable all enabled rx queues */ 2188 for (i = 0; i < adapter->num_rx_queues; i++) 2189 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]); 2190 2191 usleep_range(10000, 20000); 2192 2193 netif_tx_stop_all_queues(netdev); 2194 2195 /* call carrier off first to avoid false dev_watchdog timeouts */ 2196 netif_carrier_off(netdev); 2197 netif_tx_disable(netdev); 2198 2199 ixgbevf_irq_disable(adapter); 2200 2201 ixgbevf_napi_disable_all(adapter); 2202 2203 del_timer_sync(&adapter->service_timer); 2204 2205 /* disable transmits in the hardware now that interrupts are off */ 2206 for (i = 0; i < adapter->num_tx_queues; i++) { 2207 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 2208 2209 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), 2210 IXGBE_TXDCTL_SWFLSH); 2211 } 2212 2213 if (!pci_channel_offline(adapter->pdev)) 2214 ixgbevf_reset(adapter); 2215 2216 ixgbevf_clean_all_tx_rings(adapter); 2217 ixgbevf_clean_all_rx_rings(adapter); 2218 } 2219 2220 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 2221 { 2222 WARN_ON(in_interrupt()); 2223 2224 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 2225 msleep(1); 2226 2227 ixgbevf_down(adapter); 2228 ixgbevf_up(adapter); 2229 2230 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 2231 } 2232 2233 void ixgbevf_reset(struct ixgbevf_adapter *adapter) 2234 { 2235 struct ixgbe_hw *hw = &adapter->hw; 2236 struct net_device *netdev = adapter->netdev; 2237 2238 if (hw->mac.ops.reset_hw(hw)) { 2239 hw_dbg(hw, "PF still resetting\n"); 2240 } else { 2241 hw->mac.ops.init_hw(hw); 2242 ixgbevf_negotiate_api(adapter); 2243 } 2244 2245 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 2246 memcpy(netdev->dev_addr, adapter->hw.mac.addr, 2247 netdev->addr_len); 2248 memcpy(netdev->perm_addr, adapter->hw.mac.addr, 2249 netdev->addr_len); 2250 } 2251 2252 adapter->last_reset = jiffies; 2253 } 2254 2255 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 2256 int vectors) 2257 { 2258 int vector_threshold; 2259 2260 /* We'll want at least 2 (vector_threshold): 2261 * 1) TxQ[0] + RxQ[0] handler 2262 * 2) Other (Link Status Change, etc.) 2263 */ 2264 vector_threshold = MIN_MSIX_COUNT; 2265 2266 /* The more we get, the more we will assign to Tx/Rx Cleanup 2267 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 2268 * Right now, we simply care about how many we'll get; we'll 2269 * set them up later while requesting irq's. 2270 */ 2271 vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 2272 vector_threshold, vectors); 2273 2274 if (vectors < 0) { 2275 dev_err(&adapter->pdev->dev, 2276 "Unable to allocate MSI-X interrupts\n"); 2277 kfree(adapter->msix_entries); 2278 adapter->msix_entries = NULL; 2279 return vectors; 2280 } 2281 2282 /* Adjust for only the vectors we'll use, which is minimum 2283 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 2284 * vectors we were allocated. 2285 */ 2286 adapter->num_msix_vectors = vectors; 2287 2288 return 0; 2289 } 2290 2291 /** 2292 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent 2293 * @adapter: board private structure to initialize 2294 * 2295 * This is the top level queue allocation routine. The order here is very 2296 * important, starting with the "most" number of features turned on at once, 2297 * and ending with the smallest set of features. This way large combinations 2298 * can be allocated if they're turned on, and smaller combinations are the 2299 * fallthrough conditions. 2300 * 2301 **/ 2302 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 2303 { 2304 struct ixgbe_hw *hw = &adapter->hw; 2305 unsigned int def_q = 0; 2306 unsigned int num_tcs = 0; 2307 int err; 2308 2309 /* Start with base case */ 2310 adapter->num_rx_queues = 1; 2311 adapter->num_tx_queues = 1; 2312 2313 spin_lock_bh(&adapter->mbx_lock); 2314 2315 /* fetch queue configuration from the PF */ 2316 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2317 2318 spin_unlock_bh(&adapter->mbx_lock); 2319 2320 if (err) 2321 return; 2322 2323 /* we need as many queues as traffic classes */ 2324 if (num_tcs > 1) { 2325 adapter->num_rx_queues = num_tcs; 2326 } else { 2327 u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES); 2328 2329 switch (hw->api_version) { 2330 case ixgbe_mbox_api_11: 2331 adapter->num_rx_queues = rss; 2332 adapter->num_tx_queues = rss; 2333 default: 2334 break; 2335 } 2336 } 2337 } 2338 2339 /** 2340 * ixgbevf_alloc_queues - Allocate memory for all rings 2341 * @adapter: board private structure to initialize 2342 * 2343 * We allocate one ring per queue at run-time since we don't know the 2344 * number of queues at compile-time. The polling_netdev array is 2345 * intended for Multiqueue, but should work fine with a single queue. 2346 **/ 2347 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) 2348 { 2349 struct ixgbevf_ring *ring; 2350 int rx = 0, tx = 0; 2351 2352 for (; tx < adapter->num_tx_queues; tx++) { 2353 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 2354 if (!ring) 2355 goto err_allocation; 2356 2357 ring->dev = &adapter->pdev->dev; 2358 ring->netdev = adapter->netdev; 2359 ring->count = adapter->tx_ring_count; 2360 ring->queue_index = tx; 2361 ring->reg_idx = tx; 2362 2363 adapter->tx_ring[tx] = ring; 2364 } 2365 2366 for (; rx < adapter->num_rx_queues; rx++) { 2367 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 2368 if (!ring) 2369 goto err_allocation; 2370 2371 ring->dev = &adapter->pdev->dev; 2372 ring->netdev = adapter->netdev; 2373 2374 ring->count = adapter->rx_ring_count; 2375 ring->queue_index = rx; 2376 ring->reg_idx = rx; 2377 2378 adapter->rx_ring[rx] = ring; 2379 } 2380 2381 return 0; 2382 2383 err_allocation: 2384 while (tx) { 2385 kfree(adapter->tx_ring[--tx]); 2386 adapter->tx_ring[tx] = NULL; 2387 } 2388 2389 while (rx) { 2390 kfree(adapter->rx_ring[--rx]); 2391 adapter->rx_ring[rx] = NULL; 2392 } 2393 return -ENOMEM; 2394 } 2395 2396 /** 2397 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 2398 * @adapter: board private structure to initialize 2399 * 2400 * Attempt to configure the interrupts using the best available 2401 * capabilities of the hardware and the kernel. 2402 **/ 2403 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 2404 { 2405 struct net_device *netdev = adapter->netdev; 2406 int err = 0; 2407 int vector, v_budget; 2408 2409 /* 2410 * It's easy to be greedy for MSI-X vectors, but it really 2411 * doesn't do us much good if we have a lot more vectors 2412 * than CPU's. So let's be conservative and only ask for 2413 * (roughly) the same number of vectors as there are CPU's. 2414 * The default is to use pairs of vectors. 2415 */ 2416 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); 2417 v_budget = min_t(int, v_budget, num_online_cpus()); 2418 v_budget += NON_Q_VECTORS; 2419 2420 /* A failure in MSI-X entry allocation isn't fatal, but it does 2421 * mean we disable MSI-X capabilities of the adapter. */ 2422 adapter->msix_entries = kcalloc(v_budget, 2423 sizeof(struct msix_entry), GFP_KERNEL); 2424 if (!adapter->msix_entries) { 2425 err = -ENOMEM; 2426 goto out; 2427 } 2428 2429 for (vector = 0; vector < v_budget; vector++) 2430 adapter->msix_entries[vector].entry = vector; 2431 2432 err = ixgbevf_acquire_msix_vectors(adapter, v_budget); 2433 if (err) 2434 goto out; 2435 2436 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 2437 if (err) 2438 goto out; 2439 2440 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 2441 2442 out: 2443 return err; 2444 } 2445 2446 /** 2447 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 2448 * @adapter: board private structure to initialize 2449 * 2450 * We allocate one q_vector per queue interrupt. If allocation fails we 2451 * return -ENOMEM. 2452 **/ 2453 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 2454 { 2455 int q_idx, num_q_vectors; 2456 struct ixgbevf_q_vector *q_vector; 2457 2458 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2459 2460 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2461 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL); 2462 if (!q_vector) 2463 goto err_out; 2464 q_vector->adapter = adapter; 2465 q_vector->v_idx = q_idx; 2466 netif_napi_add(adapter->netdev, &q_vector->napi, 2467 ixgbevf_poll, 64); 2468 #ifdef CONFIG_NET_RX_BUSY_POLL 2469 napi_hash_add(&q_vector->napi); 2470 #endif 2471 adapter->q_vector[q_idx] = q_vector; 2472 } 2473 2474 return 0; 2475 2476 err_out: 2477 while (q_idx) { 2478 q_idx--; 2479 q_vector = adapter->q_vector[q_idx]; 2480 #ifdef CONFIG_NET_RX_BUSY_POLL 2481 napi_hash_del(&q_vector->napi); 2482 #endif 2483 netif_napi_del(&q_vector->napi); 2484 kfree(q_vector); 2485 adapter->q_vector[q_idx] = NULL; 2486 } 2487 return -ENOMEM; 2488 } 2489 2490 /** 2491 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 2492 * @adapter: board private structure to initialize 2493 * 2494 * This function frees the memory allocated to the q_vectors. In addition if 2495 * NAPI is enabled it will delete any references to the NAPI struct prior 2496 * to freeing the q_vector. 2497 **/ 2498 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 2499 { 2500 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2501 2502 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2503 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; 2504 2505 adapter->q_vector[q_idx] = NULL; 2506 #ifdef CONFIG_NET_RX_BUSY_POLL 2507 napi_hash_del(&q_vector->napi); 2508 #endif 2509 netif_napi_del(&q_vector->napi); 2510 kfree(q_vector); 2511 } 2512 } 2513 2514 /** 2515 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 2516 * @adapter: board private structure 2517 * 2518 **/ 2519 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 2520 { 2521 pci_disable_msix(adapter->pdev); 2522 kfree(adapter->msix_entries); 2523 adapter->msix_entries = NULL; 2524 } 2525 2526 /** 2527 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 2528 * @adapter: board private structure to initialize 2529 * 2530 **/ 2531 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 2532 { 2533 int err; 2534 2535 /* Number of supported queues */ 2536 ixgbevf_set_num_queues(adapter); 2537 2538 err = ixgbevf_set_interrupt_capability(adapter); 2539 if (err) { 2540 hw_dbg(&adapter->hw, 2541 "Unable to setup interrupt capabilities\n"); 2542 goto err_set_interrupt; 2543 } 2544 2545 err = ixgbevf_alloc_q_vectors(adapter); 2546 if (err) { 2547 hw_dbg(&adapter->hw, "Unable to allocate memory for queue " 2548 "vectors\n"); 2549 goto err_alloc_q_vectors; 2550 } 2551 2552 err = ixgbevf_alloc_queues(adapter); 2553 if (err) { 2554 pr_err("Unable to allocate memory for queues\n"); 2555 goto err_alloc_queues; 2556 } 2557 2558 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, " 2559 "Tx Queue count = %u\n", 2560 (adapter->num_rx_queues > 1) ? "Enabled" : 2561 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues); 2562 2563 set_bit(__IXGBEVF_DOWN, &adapter->state); 2564 2565 return 0; 2566 err_alloc_queues: 2567 ixgbevf_free_q_vectors(adapter); 2568 err_alloc_q_vectors: 2569 ixgbevf_reset_interrupt_capability(adapter); 2570 err_set_interrupt: 2571 return err; 2572 } 2573 2574 /** 2575 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings 2576 * @adapter: board private structure to clear interrupt scheme on 2577 * 2578 * We go through and clear interrupt specific resources and reset the structure 2579 * to pre-load conditions 2580 **/ 2581 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) 2582 { 2583 int i; 2584 2585 for (i = 0; i < adapter->num_tx_queues; i++) { 2586 kfree(adapter->tx_ring[i]); 2587 adapter->tx_ring[i] = NULL; 2588 } 2589 for (i = 0; i < adapter->num_rx_queues; i++) { 2590 kfree(adapter->rx_ring[i]); 2591 adapter->rx_ring[i] = NULL; 2592 } 2593 2594 adapter->num_tx_queues = 0; 2595 adapter->num_rx_queues = 0; 2596 2597 ixgbevf_free_q_vectors(adapter); 2598 ixgbevf_reset_interrupt_capability(adapter); 2599 } 2600 2601 /** 2602 * ixgbevf_sw_init - Initialize general software structures 2603 * (struct ixgbevf_adapter) 2604 * @adapter: board private structure to initialize 2605 * 2606 * ixgbevf_sw_init initializes the Adapter private data structure. 2607 * Fields are initialized based on PCI device information and 2608 * OS network device settings (MTU size). 2609 **/ 2610 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 2611 { 2612 struct ixgbe_hw *hw = &adapter->hw; 2613 struct pci_dev *pdev = adapter->pdev; 2614 struct net_device *netdev = adapter->netdev; 2615 int err; 2616 2617 /* PCI config space info */ 2618 2619 hw->vendor_id = pdev->vendor; 2620 hw->device_id = pdev->device; 2621 hw->revision_id = pdev->revision; 2622 hw->subsystem_vendor_id = pdev->subsystem_vendor; 2623 hw->subsystem_device_id = pdev->subsystem_device; 2624 2625 hw->mbx.ops.init_params(hw); 2626 2627 /* assume legacy case in which PF would only give VF 2 queues */ 2628 hw->mac.max_tx_queues = 2; 2629 hw->mac.max_rx_queues = 2; 2630 2631 /* lock to protect mailbox accesses */ 2632 spin_lock_init(&adapter->mbx_lock); 2633 2634 err = hw->mac.ops.reset_hw(hw); 2635 if (err) { 2636 dev_info(&pdev->dev, 2637 "PF still in reset state. Is the PF interface up?\n"); 2638 } else { 2639 err = hw->mac.ops.init_hw(hw); 2640 if (err) { 2641 pr_err("init_shared_code failed: %d\n", err); 2642 goto out; 2643 } 2644 ixgbevf_negotiate_api(adapter); 2645 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr); 2646 if (err) 2647 dev_info(&pdev->dev, "Error reading MAC address\n"); 2648 else if (is_zero_ether_addr(adapter->hw.mac.addr)) 2649 dev_info(&pdev->dev, 2650 "MAC address not assigned by administrator.\n"); 2651 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 2652 } 2653 2654 if (!is_valid_ether_addr(netdev->dev_addr)) { 2655 dev_info(&pdev->dev, "Assigning random MAC address\n"); 2656 eth_hw_addr_random(netdev); 2657 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len); 2658 } 2659 2660 /* Enable dynamic interrupt throttling rates */ 2661 adapter->rx_itr_setting = 1; 2662 adapter->tx_itr_setting = 1; 2663 2664 /* set default ring sizes */ 2665 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 2666 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 2667 2668 set_bit(__IXGBEVF_DOWN, &adapter->state); 2669 return 0; 2670 2671 out: 2672 return err; 2673 } 2674 2675 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 2676 { \ 2677 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 2678 if (current_counter < last_counter) \ 2679 counter += 0x100000000LL; \ 2680 last_counter = current_counter; \ 2681 counter &= 0xFFFFFFFF00000000LL; \ 2682 counter |= current_counter; \ 2683 } 2684 2685 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 2686 { \ 2687 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 2688 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 2689 u64 current_counter = (current_counter_msb << 32) | \ 2690 current_counter_lsb; \ 2691 if (current_counter < last_counter) \ 2692 counter += 0x1000000000LL; \ 2693 last_counter = current_counter; \ 2694 counter &= 0xFFFFFFF000000000LL; \ 2695 counter |= current_counter; \ 2696 } 2697 /** 2698 * ixgbevf_update_stats - Update the board statistics counters. 2699 * @adapter: board private structure 2700 **/ 2701 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 2702 { 2703 struct ixgbe_hw *hw = &adapter->hw; 2704 int i; 2705 2706 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2707 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2708 return; 2709 2710 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 2711 adapter->stats.vfgprc); 2712 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 2713 adapter->stats.vfgptc); 2714 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2715 adapter->stats.last_vfgorc, 2716 adapter->stats.vfgorc); 2717 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2718 adapter->stats.last_vfgotc, 2719 adapter->stats.vfgotc); 2720 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 2721 adapter->stats.vfmprc); 2722 2723 for (i = 0; i < adapter->num_rx_queues; i++) { 2724 adapter->hw_csum_rx_error += 2725 adapter->rx_ring[i]->hw_csum_rx_error; 2726 adapter->rx_ring[i]->hw_csum_rx_error = 0; 2727 } 2728 } 2729 2730 /** 2731 * ixgbevf_service_timer - Timer Call-back 2732 * @data: pointer to adapter cast into an unsigned long 2733 **/ 2734 static void ixgbevf_service_timer(unsigned long data) 2735 { 2736 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data; 2737 2738 /* Reset the timer */ 2739 mod_timer(&adapter->service_timer, (HZ * 2) + jiffies); 2740 2741 ixgbevf_service_event_schedule(adapter); 2742 } 2743 2744 static void ixgbevf_reset_subtask(struct ixgbevf_adapter *adapter) 2745 { 2746 if (!(adapter->flags & IXGBEVF_FLAG_RESET_REQUESTED)) 2747 return; 2748 2749 adapter->flags &= ~IXGBEVF_FLAG_RESET_REQUESTED; 2750 2751 /* If we're already down or resetting, just bail */ 2752 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2753 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2754 return; 2755 2756 adapter->tx_timeout_count++; 2757 2758 ixgbevf_reinit_locked(adapter); 2759 } 2760 2761 /* ixgbevf_check_hang_subtask - check for hung queues and dropped interrupts 2762 * @adapter - pointer to the device adapter structure 2763 * 2764 * This function serves two purposes. First it strobes the interrupt lines 2765 * in order to make certain interrupts are occurring. Secondly it sets the 2766 * bits needed to check for TX hangs. As a result we should immediately 2767 * determine if a hang has occurred. 2768 */ 2769 static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter) 2770 { 2771 struct ixgbe_hw *hw = &adapter->hw; 2772 u32 eics = 0; 2773 int i; 2774 2775 /* If we're down or resetting, just bail */ 2776 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2777 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2778 return; 2779 2780 /* Force detection of hung controller */ 2781 if (netif_carrier_ok(adapter->netdev)) { 2782 for (i = 0; i < adapter->num_tx_queues; i++) 2783 set_check_for_tx_hang(adapter->tx_ring[i]); 2784 } 2785 2786 /* get one bit for every active tx/rx interrupt vector */ 2787 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 2788 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 2789 2790 if (qv->rx.ring || qv->tx.ring) 2791 eics |= 1 << i; 2792 } 2793 2794 /* Cause software interrupt to ensure rings are cleaned */ 2795 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics); 2796 } 2797 2798 /** 2799 * ixgbevf_watchdog_update_link - update the link status 2800 * @adapter - pointer to the device adapter structure 2801 **/ 2802 static void ixgbevf_watchdog_update_link(struct ixgbevf_adapter *adapter) 2803 { 2804 struct ixgbe_hw *hw = &adapter->hw; 2805 u32 link_speed = adapter->link_speed; 2806 bool link_up = adapter->link_up; 2807 s32 err; 2808 2809 spin_lock_bh(&adapter->mbx_lock); 2810 2811 err = hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 2812 2813 spin_unlock_bh(&adapter->mbx_lock); 2814 2815 /* if check for link returns error we will need to reset */ 2816 if (err && time_after(jiffies, adapter->last_reset + (10 * HZ))) { 2817 adapter->flags |= IXGBEVF_FLAG_RESET_REQUESTED; 2818 link_up = false; 2819 } 2820 2821 adapter->link_up = link_up; 2822 adapter->link_speed = link_speed; 2823 } 2824 2825 /** 2826 * ixgbevf_watchdog_link_is_up - update netif_carrier status and 2827 * print link up message 2828 * @adapter - pointer to the device adapter structure 2829 **/ 2830 static void ixgbevf_watchdog_link_is_up(struct ixgbevf_adapter *adapter) 2831 { 2832 struct net_device *netdev = adapter->netdev; 2833 2834 /* only continue if link was previously down */ 2835 if (netif_carrier_ok(netdev)) 2836 return; 2837 2838 dev_info(&adapter->pdev->dev, "NIC Link is Up %s\n", 2839 (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 2840 "10 Gbps" : 2841 (adapter->link_speed == IXGBE_LINK_SPEED_1GB_FULL) ? 2842 "1 Gbps" : 2843 (adapter->link_speed == IXGBE_LINK_SPEED_100_FULL) ? 2844 "100 Mbps" : 2845 "unknown speed"); 2846 2847 netif_carrier_on(netdev); 2848 } 2849 2850 /** 2851 * ixgbevf_watchdog_link_is_down - update netif_carrier status and 2852 * print link down message 2853 * @adapter - pointer to the adapter structure 2854 **/ 2855 static void ixgbevf_watchdog_link_is_down(struct ixgbevf_adapter *adapter) 2856 { 2857 struct net_device *netdev = adapter->netdev; 2858 2859 adapter->link_speed = 0; 2860 2861 /* only continue if link was up previously */ 2862 if (!netif_carrier_ok(netdev)) 2863 return; 2864 2865 dev_info(&adapter->pdev->dev, "NIC Link is Down\n"); 2866 2867 netif_carrier_off(netdev); 2868 } 2869 2870 /** 2871 * ixgbevf_watchdog_subtask - worker thread to bring link up 2872 * @work: pointer to work_struct containing our data 2873 **/ 2874 static void ixgbevf_watchdog_subtask(struct ixgbevf_adapter *adapter) 2875 { 2876 /* if interface is down do nothing */ 2877 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2878 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2879 return; 2880 2881 ixgbevf_watchdog_update_link(adapter); 2882 2883 if (adapter->link_up) 2884 ixgbevf_watchdog_link_is_up(adapter); 2885 else 2886 ixgbevf_watchdog_link_is_down(adapter); 2887 2888 ixgbevf_update_stats(adapter); 2889 } 2890 2891 /** 2892 * ixgbevf_service_task - manages and runs subtasks 2893 * @work: pointer to work_struct containing our data 2894 **/ 2895 static void ixgbevf_service_task(struct work_struct *work) 2896 { 2897 struct ixgbevf_adapter *adapter = container_of(work, 2898 struct ixgbevf_adapter, 2899 service_task); 2900 struct ixgbe_hw *hw = &adapter->hw; 2901 2902 if (IXGBE_REMOVED(hw->hw_addr)) { 2903 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 2904 rtnl_lock(); 2905 ixgbevf_down(adapter); 2906 rtnl_unlock(); 2907 } 2908 return; 2909 } 2910 2911 ixgbevf_queue_reset_subtask(adapter); 2912 ixgbevf_reset_subtask(adapter); 2913 ixgbevf_watchdog_subtask(adapter); 2914 ixgbevf_check_hang_subtask(adapter); 2915 2916 ixgbevf_service_event_complete(adapter); 2917 } 2918 2919 /** 2920 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 2921 * @tx_ring: Tx descriptor ring for a specific queue 2922 * 2923 * Free all transmit software resources 2924 **/ 2925 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) 2926 { 2927 ixgbevf_clean_tx_ring(tx_ring); 2928 2929 vfree(tx_ring->tx_buffer_info); 2930 tx_ring->tx_buffer_info = NULL; 2931 2932 /* if not set, then don't free */ 2933 if (!tx_ring->desc) 2934 return; 2935 2936 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, 2937 tx_ring->dma); 2938 2939 tx_ring->desc = NULL; 2940 } 2941 2942 /** 2943 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 2944 * @adapter: board private structure 2945 * 2946 * Free all transmit software resources 2947 **/ 2948 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 2949 { 2950 int i; 2951 2952 for (i = 0; i < adapter->num_tx_queues; i++) 2953 if (adapter->tx_ring[i]->desc) 2954 ixgbevf_free_tx_resources(adapter->tx_ring[i]); 2955 } 2956 2957 /** 2958 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 2959 * @tx_ring: tx descriptor ring (for a specific queue) to setup 2960 * 2961 * Return 0 on success, negative on failure 2962 **/ 2963 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) 2964 { 2965 int size; 2966 2967 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2968 tx_ring->tx_buffer_info = vzalloc(size); 2969 if (!tx_ring->tx_buffer_info) 2970 goto err; 2971 2972 /* round up to nearest 4K */ 2973 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 2974 tx_ring->size = ALIGN(tx_ring->size, 4096); 2975 2976 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, 2977 &tx_ring->dma, GFP_KERNEL); 2978 if (!tx_ring->desc) 2979 goto err; 2980 2981 return 0; 2982 2983 err: 2984 vfree(tx_ring->tx_buffer_info); 2985 tx_ring->tx_buffer_info = NULL; 2986 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit " 2987 "descriptor ring\n"); 2988 return -ENOMEM; 2989 } 2990 2991 /** 2992 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 2993 * @adapter: board private structure 2994 * 2995 * If this function returns with an error, then it's possible one or 2996 * more of the rings is populated (while the rest are not). It is the 2997 * callers duty to clean those orphaned rings. 2998 * 2999 * Return 0 on success, negative on failure 3000 **/ 3001 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 3002 { 3003 int i, err = 0; 3004 3005 for (i = 0; i < adapter->num_tx_queues; i++) { 3006 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]); 3007 if (!err) 3008 continue; 3009 hw_dbg(&adapter->hw, 3010 "Allocation for Tx Queue %u failed\n", i); 3011 break; 3012 } 3013 3014 return err; 3015 } 3016 3017 /** 3018 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 3019 * @rx_ring: rx descriptor ring (for a specific queue) to setup 3020 * 3021 * Returns 0 on success, negative on failure 3022 **/ 3023 int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring) 3024 { 3025 int size; 3026 3027 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 3028 rx_ring->rx_buffer_info = vzalloc(size); 3029 if (!rx_ring->rx_buffer_info) 3030 goto err; 3031 3032 /* Round up to nearest 4K */ 3033 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 3034 rx_ring->size = ALIGN(rx_ring->size, 4096); 3035 3036 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, 3037 &rx_ring->dma, GFP_KERNEL); 3038 3039 if (!rx_ring->desc) 3040 goto err; 3041 3042 return 0; 3043 err: 3044 vfree(rx_ring->rx_buffer_info); 3045 rx_ring->rx_buffer_info = NULL; 3046 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); 3047 return -ENOMEM; 3048 } 3049 3050 /** 3051 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 3052 * @adapter: board private structure 3053 * 3054 * If this function returns with an error, then it's possible one or 3055 * more of the rings is populated (while the rest are not). It is the 3056 * callers duty to clean those orphaned rings. 3057 * 3058 * Return 0 on success, negative on failure 3059 **/ 3060 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 3061 { 3062 int i, err = 0; 3063 3064 for (i = 0; i < adapter->num_rx_queues; i++) { 3065 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]); 3066 if (!err) 3067 continue; 3068 hw_dbg(&adapter->hw, 3069 "Allocation for Rx Queue %u failed\n", i); 3070 break; 3071 } 3072 return err; 3073 } 3074 3075 /** 3076 * ixgbevf_free_rx_resources - Free Rx Resources 3077 * @rx_ring: ring to clean the resources from 3078 * 3079 * Free all receive software resources 3080 **/ 3081 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) 3082 { 3083 ixgbevf_clean_rx_ring(rx_ring); 3084 3085 vfree(rx_ring->rx_buffer_info); 3086 rx_ring->rx_buffer_info = NULL; 3087 3088 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, 3089 rx_ring->dma); 3090 3091 rx_ring->desc = NULL; 3092 } 3093 3094 /** 3095 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 3096 * @adapter: board private structure 3097 * 3098 * Free all receive software resources 3099 **/ 3100 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 3101 { 3102 int i; 3103 3104 for (i = 0; i < adapter->num_rx_queues; i++) 3105 if (adapter->rx_ring[i]->desc) 3106 ixgbevf_free_rx_resources(adapter->rx_ring[i]); 3107 } 3108 3109 /** 3110 * ixgbevf_open - Called when a network interface is made active 3111 * @netdev: network interface device structure 3112 * 3113 * Returns 0 on success, negative value on failure 3114 * 3115 * The open entry point is called when a network interface is made 3116 * active by the system (IFF_UP). At this point all resources needed 3117 * for transmit and receive operations are allocated, the interrupt 3118 * handler is registered with the OS, the watchdog timer is started, 3119 * and the stack is notified that the interface is ready. 3120 **/ 3121 static int ixgbevf_open(struct net_device *netdev) 3122 { 3123 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3124 struct ixgbe_hw *hw = &adapter->hw; 3125 int err; 3126 3127 /* A previous failure to open the device because of a lack of 3128 * available MSIX vector resources may have reset the number 3129 * of msix vectors variable to zero. The only way to recover 3130 * is to unload/reload the driver and hope that the system has 3131 * been able to recover some MSIX vector resources. 3132 */ 3133 if (!adapter->num_msix_vectors) 3134 return -ENOMEM; 3135 3136 if (hw->adapter_stopped) { 3137 ixgbevf_reset(adapter); 3138 /* if adapter is still stopped then PF isn't up and 3139 * the vf can't start. */ 3140 if (hw->adapter_stopped) { 3141 err = IXGBE_ERR_MBX; 3142 pr_err("Unable to start - perhaps the PF Driver isn't " 3143 "up yet\n"); 3144 goto err_setup_reset; 3145 } 3146 } 3147 3148 /* disallow open during test */ 3149 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 3150 return -EBUSY; 3151 3152 netif_carrier_off(netdev); 3153 3154 /* allocate transmit descriptors */ 3155 err = ixgbevf_setup_all_tx_resources(adapter); 3156 if (err) 3157 goto err_setup_tx; 3158 3159 /* allocate receive descriptors */ 3160 err = ixgbevf_setup_all_rx_resources(adapter); 3161 if (err) 3162 goto err_setup_rx; 3163 3164 ixgbevf_configure(adapter); 3165 3166 /* 3167 * Map the Tx/Rx rings to the vectors we were allotted. 3168 * if request_irq will be called in this function map_rings 3169 * must be called *before* up_complete 3170 */ 3171 ixgbevf_map_rings_to_vectors(adapter); 3172 3173 err = ixgbevf_request_irq(adapter); 3174 if (err) 3175 goto err_req_irq; 3176 3177 ixgbevf_up_complete(adapter); 3178 3179 return 0; 3180 3181 err_req_irq: 3182 ixgbevf_down(adapter); 3183 err_setup_rx: 3184 ixgbevf_free_all_rx_resources(adapter); 3185 err_setup_tx: 3186 ixgbevf_free_all_tx_resources(adapter); 3187 ixgbevf_reset(adapter); 3188 3189 err_setup_reset: 3190 3191 return err; 3192 } 3193 3194 /** 3195 * ixgbevf_close - Disables a network interface 3196 * @netdev: network interface device structure 3197 * 3198 * Returns 0, this is not allowed to fail 3199 * 3200 * The close entry point is called when an interface is de-activated 3201 * by the OS. The hardware is still under the drivers control, but 3202 * needs to be disabled. A global MAC reset is issued to stop the 3203 * hardware, and all transmit and receive resources are freed. 3204 **/ 3205 static int ixgbevf_close(struct net_device *netdev) 3206 { 3207 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3208 3209 ixgbevf_down(adapter); 3210 ixgbevf_free_irq(adapter); 3211 3212 ixgbevf_free_all_tx_resources(adapter); 3213 ixgbevf_free_all_rx_resources(adapter); 3214 3215 return 0; 3216 } 3217 3218 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter) 3219 { 3220 struct net_device *dev = adapter->netdev; 3221 3222 if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED)) 3223 return; 3224 3225 adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED; 3226 3227 /* if interface is down do nothing */ 3228 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3229 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3230 return; 3231 3232 /* Hardware has to reinitialize queues and interrupts to 3233 * match packet buffer alignment. Unfortunately, the 3234 * hardware is not flexible enough to do this dynamically. 3235 */ 3236 if (netif_running(dev)) 3237 ixgbevf_close(dev); 3238 3239 ixgbevf_clear_interrupt_scheme(adapter); 3240 ixgbevf_init_interrupt_scheme(adapter); 3241 3242 if (netif_running(dev)) 3243 ixgbevf_open(dev); 3244 } 3245 3246 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, 3247 u32 vlan_macip_lens, u32 type_tucmd, 3248 u32 mss_l4len_idx) 3249 { 3250 struct ixgbe_adv_tx_context_desc *context_desc; 3251 u16 i = tx_ring->next_to_use; 3252 3253 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i); 3254 3255 i++; 3256 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 3257 3258 /* set bits to identify this as an advanced context descriptor */ 3259 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3260 3261 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 3262 context_desc->seqnum_seed = 0; 3263 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 3264 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 3265 } 3266 3267 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, 3268 struct ixgbevf_tx_buffer *first, 3269 u8 *hdr_len) 3270 { 3271 struct sk_buff *skb = first->skb; 3272 u32 vlan_macip_lens, type_tucmd; 3273 u32 mss_l4len_idx, l4len; 3274 int err; 3275 3276 if (skb->ip_summed != CHECKSUM_PARTIAL) 3277 return 0; 3278 3279 if (!skb_is_gso(skb)) 3280 return 0; 3281 3282 err = skb_cow_head(skb, 0); 3283 if (err < 0) 3284 return err; 3285 3286 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 3287 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3288 3289 if (first->protocol == htons(ETH_P_IP)) { 3290 struct iphdr *iph = ip_hdr(skb); 3291 iph->tot_len = 0; 3292 iph->check = 0; 3293 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 3294 iph->daddr, 0, 3295 IPPROTO_TCP, 3296 0); 3297 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3298 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3299 IXGBE_TX_FLAGS_CSUM | 3300 IXGBE_TX_FLAGS_IPV4; 3301 } else if (skb_is_gso_v6(skb)) { 3302 ipv6_hdr(skb)->payload_len = 0; 3303 tcp_hdr(skb)->check = 3304 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 3305 &ipv6_hdr(skb)->daddr, 3306 0, IPPROTO_TCP, 0); 3307 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3308 IXGBE_TX_FLAGS_CSUM; 3309 } 3310 3311 /* compute header lengths */ 3312 l4len = tcp_hdrlen(skb); 3313 *hdr_len += l4len; 3314 *hdr_len = skb_transport_offset(skb) + l4len; 3315 3316 /* update gso size and bytecount with header size */ 3317 first->gso_segs = skb_shinfo(skb)->gso_segs; 3318 first->bytecount += (first->gso_segs - 1) * *hdr_len; 3319 3320 /* mss_l4len_id: use 1 as index for TSO */ 3321 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; 3322 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 3323 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT; 3324 3325 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 3326 vlan_macip_lens = skb_network_header_len(skb); 3327 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3328 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3329 3330 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3331 type_tucmd, mss_l4len_idx); 3332 3333 return 1; 3334 } 3335 3336 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, 3337 struct ixgbevf_tx_buffer *first) 3338 { 3339 struct sk_buff *skb = first->skb; 3340 u32 vlan_macip_lens = 0; 3341 u32 mss_l4len_idx = 0; 3342 u32 type_tucmd = 0; 3343 3344 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3345 u8 l4_hdr = 0; 3346 switch (first->protocol) { 3347 case htons(ETH_P_IP): 3348 vlan_macip_lens |= skb_network_header_len(skb); 3349 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3350 l4_hdr = ip_hdr(skb)->protocol; 3351 break; 3352 case htons(ETH_P_IPV6): 3353 vlan_macip_lens |= skb_network_header_len(skb); 3354 l4_hdr = ipv6_hdr(skb)->nexthdr; 3355 break; 3356 default: 3357 if (unlikely(net_ratelimit())) { 3358 dev_warn(tx_ring->dev, 3359 "partial checksum but proto=%x!\n", 3360 first->protocol); 3361 } 3362 break; 3363 } 3364 3365 switch (l4_hdr) { 3366 case IPPROTO_TCP: 3367 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3368 mss_l4len_idx = tcp_hdrlen(skb) << 3369 IXGBE_ADVTXD_L4LEN_SHIFT; 3370 break; 3371 case IPPROTO_SCTP: 3372 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 3373 mss_l4len_idx = sizeof(struct sctphdr) << 3374 IXGBE_ADVTXD_L4LEN_SHIFT; 3375 break; 3376 case IPPROTO_UDP: 3377 mss_l4len_idx = sizeof(struct udphdr) << 3378 IXGBE_ADVTXD_L4LEN_SHIFT; 3379 break; 3380 default: 3381 if (unlikely(net_ratelimit())) { 3382 dev_warn(tx_ring->dev, 3383 "partial checksum but l4 proto=%x!\n", 3384 l4_hdr); 3385 } 3386 break; 3387 } 3388 3389 /* update TX checksum flag */ 3390 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 3391 } 3392 3393 /* vlan_macip_lens: MACLEN, VLAN tag */ 3394 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3395 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3396 3397 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3398 type_tucmd, mss_l4len_idx); 3399 } 3400 3401 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags) 3402 { 3403 /* set type for advanced descriptor with frame checksum insertion */ 3404 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | 3405 IXGBE_ADVTXD_DCMD_IFCS | 3406 IXGBE_ADVTXD_DCMD_DEXT); 3407 3408 /* set HW vlan bit if vlan is present */ 3409 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 3410 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); 3411 3412 /* set segmentation enable bits for TSO/FSO */ 3413 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3414 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); 3415 3416 return cmd_type; 3417 } 3418 3419 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 3420 u32 tx_flags, unsigned int paylen) 3421 { 3422 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT); 3423 3424 /* enable L4 checksum for TSO and TX checksum offload */ 3425 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 3426 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM); 3427 3428 /* enble IPv4 checksum for TSO */ 3429 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 3430 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM); 3431 3432 /* use index 1 context for TSO/FSO/FCOE */ 3433 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3434 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT); 3435 3436 /* Check Context must be set if Tx switch is enabled, which it 3437 * always is for case where virtual functions are running 3438 */ 3439 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC); 3440 3441 tx_desc->read.olinfo_status = olinfo_status; 3442 } 3443 3444 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, 3445 struct ixgbevf_tx_buffer *first, 3446 const u8 hdr_len) 3447 { 3448 dma_addr_t dma; 3449 struct sk_buff *skb = first->skb; 3450 struct ixgbevf_tx_buffer *tx_buffer; 3451 union ixgbe_adv_tx_desc *tx_desc; 3452 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 3453 unsigned int data_len = skb->data_len; 3454 unsigned int size = skb_headlen(skb); 3455 unsigned int paylen = skb->len - hdr_len; 3456 u32 tx_flags = first->tx_flags; 3457 __le32 cmd_type; 3458 u16 i = tx_ring->next_to_use; 3459 3460 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 3461 3462 ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen); 3463 cmd_type = ixgbevf_tx_cmd_type(tx_flags); 3464 3465 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 3466 if (dma_mapping_error(tx_ring->dev, dma)) 3467 goto dma_error; 3468 3469 /* record length, and DMA address */ 3470 dma_unmap_len_set(first, len, size); 3471 dma_unmap_addr_set(first, dma, dma); 3472 3473 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3474 3475 for (;;) { 3476 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 3477 tx_desc->read.cmd_type_len = 3478 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); 3479 3480 i++; 3481 tx_desc++; 3482 if (i == tx_ring->count) { 3483 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3484 i = 0; 3485 } 3486 3487 dma += IXGBE_MAX_DATA_PER_TXD; 3488 size -= IXGBE_MAX_DATA_PER_TXD; 3489 3490 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3491 tx_desc->read.olinfo_status = 0; 3492 } 3493 3494 if (likely(!data_len)) 3495 break; 3496 3497 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); 3498 3499 i++; 3500 tx_desc++; 3501 if (i == tx_ring->count) { 3502 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3503 i = 0; 3504 } 3505 3506 size = skb_frag_size(frag); 3507 data_len -= size; 3508 3509 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 3510 DMA_TO_DEVICE); 3511 if (dma_mapping_error(tx_ring->dev, dma)) 3512 goto dma_error; 3513 3514 tx_buffer = &tx_ring->tx_buffer_info[i]; 3515 dma_unmap_len_set(tx_buffer, len, size); 3516 dma_unmap_addr_set(tx_buffer, dma, dma); 3517 3518 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3519 tx_desc->read.olinfo_status = 0; 3520 3521 frag++; 3522 } 3523 3524 /* write last descriptor with RS and EOP bits */ 3525 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); 3526 tx_desc->read.cmd_type_len = cmd_type; 3527 3528 /* set the timestamp */ 3529 first->time_stamp = jiffies; 3530 3531 /* Force memory writes to complete before letting h/w know there 3532 * are new descriptors to fetch. (Only applicable for weak-ordered 3533 * memory model archs, such as IA-64). 3534 * 3535 * We also need this memory barrier (wmb) to make certain all of the 3536 * status bits have been updated before next_to_watch is written. 3537 */ 3538 wmb(); 3539 3540 /* set next_to_watch value indicating a packet is present */ 3541 first->next_to_watch = tx_desc; 3542 3543 i++; 3544 if (i == tx_ring->count) 3545 i = 0; 3546 3547 tx_ring->next_to_use = i; 3548 3549 /* notify HW of packet */ 3550 ixgbevf_write_tail(tx_ring, i); 3551 3552 return; 3553 dma_error: 3554 dev_err(tx_ring->dev, "TX DMA map failed\n"); 3555 3556 /* clear dma mappings for failed tx_buffer_info map */ 3557 for (;;) { 3558 tx_buffer = &tx_ring->tx_buffer_info[i]; 3559 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer); 3560 if (tx_buffer == first) 3561 break; 3562 if (i == 0) 3563 i = tx_ring->count; 3564 i--; 3565 } 3566 3567 tx_ring->next_to_use = i; 3568 } 3569 3570 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 3571 { 3572 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 3573 /* Herbert's original patch had: 3574 * smp_mb__after_netif_stop_queue(); 3575 * but since that doesn't exist yet, just open code it. */ 3576 smp_mb(); 3577 3578 /* We need to check again in a case another CPU has just 3579 * made room available. */ 3580 if (likely(ixgbevf_desc_unused(tx_ring) < size)) 3581 return -EBUSY; 3582 3583 /* A reprieve! - use start_queue because it doesn't call schedule */ 3584 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 3585 ++tx_ring->tx_stats.restart_queue; 3586 3587 return 0; 3588 } 3589 3590 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 3591 { 3592 if (likely(ixgbevf_desc_unused(tx_ring) >= size)) 3593 return 0; 3594 return __ixgbevf_maybe_stop_tx(tx_ring, size); 3595 } 3596 3597 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 3598 { 3599 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3600 struct ixgbevf_tx_buffer *first; 3601 struct ixgbevf_ring *tx_ring; 3602 int tso; 3603 u32 tx_flags = 0; 3604 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 3605 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3606 unsigned short f; 3607 #endif 3608 u8 hdr_len = 0; 3609 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL); 3610 3611 if (!dst_mac || is_link_local_ether_addr(dst_mac)) { 3612 dev_kfree_skb(skb); 3613 return NETDEV_TX_OK; 3614 } 3615 3616 tx_ring = adapter->tx_ring[skb->queue_mapping]; 3617 3618 /* 3619 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 3620 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 3621 * + 2 desc gap to keep tail from touching head, 3622 * + 1 desc for context descriptor, 3623 * otherwise try next time 3624 */ 3625 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3626 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 3627 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 3628 #else 3629 count += skb_shinfo(skb)->nr_frags; 3630 #endif 3631 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { 3632 tx_ring->tx_stats.tx_busy++; 3633 return NETDEV_TX_BUSY; 3634 } 3635 3636 /* record the location of the first descriptor for this packet */ 3637 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 3638 first->skb = skb; 3639 first->bytecount = skb->len; 3640 first->gso_segs = 1; 3641 3642 if (skb_vlan_tag_present(skb)) { 3643 tx_flags |= skb_vlan_tag_get(skb); 3644 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 3645 tx_flags |= IXGBE_TX_FLAGS_VLAN; 3646 } 3647 3648 /* record initial flags and protocol */ 3649 first->tx_flags = tx_flags; 3650 first->protocol = vlan_get_protocol(skb); 3651 3652 tso = ixgbevf_tso(tx_ring, first, &hdr_len); 3653 if (tso < 0) 3654 goto out_drop; 3655 else if (!tso) 3656 ixgbevf_tx_csum(tx_ring, first); 3657 3658 ixgbevf_tx_map(tx_ring, first, hdr_len); 3659 3660 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); 3661 3662 return NETDEV_TX_OK; 3663 3664 out_drop: 3665 dev_kfree_skb_any(first->skb); 3666 first->skb = NULL; 3667 3668 return NETDEV_TX_OK; 3669 } 3670 3671 /** 3672 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 3673 * @netdev: network interface device structure 3674 * @p: pointer to an address structure 3675 * 3676 * Returns 0 on success, negative on failure 3677 **/ 3678 static int ixgbevf_set_mac(struct net_device *netdev, void *p) 3679 { 3680 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3681 struct ixgbe_hw *hw = &adapter->hw; 3682 struct sockaddr *addr = p; 3683 3684 if (!is_valid_ether_addr(addr->sa_data)) 3685 return -EADDRNOTAVAIL; 3686 3687 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 3688 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 3689 3690 spin_lock_bh(&adapter->mbx_lock); 3691 3692 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 3693 3694 spin_unlock_bh(&adapter->mbx_lock); 3695 3696 return 0; 3697 } 3698 3699 /** 3700 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 3701 * @netdev: network interface device structure 3702 * @new_mtu: new value for maximum frame size 3703 * 3704 * Returns 0 on success, negative on failure 3705 **/ 3706 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 3707 { 3708 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3709 struct ixgbe_hw *hw = &adapter->hw; 3710 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 3711 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; 3712 3713 switch (adapter->hw.api_version) { 3714 case ixgbe_mbox_api_11: 3715 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3716 break; 3717 default: 3718 if (adapter->hw.mac.type != ixgbe_mac_82599_vf) 3719 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3720 break; 3721 } 3722 3723 /* MTU < 68 is an error and causes problems on some kernels */ 3724 if ((new_mtu < 68) || (max_frame > max_possible_frame)) 3725 return -EINVAL; 3726 3727 hw_dbg(hw, "changing MTU from %d to %d\n", 3728 netdev->mtu, new_mtu); 3729 /* must set new MTU before calling down or up */ 3730 netdev->mtu = new_mtu; 3731 3732 /* notify the PF of our intent to use this size of frame */ 3733 ixgbevf_rlpml_set_vf(hw, max_frame); 3734 3735 return 0; 3736 } 3737 3738 #ifdef CONFIG_NET_POLL_CONTROLLER 3739 /* Polling 'interrupt' - used by things like netconsole to send skbs 3740 * without having to re-enable interrupts. It's not called while 3741 * the interrupt routine is executing. 3742 */ 3743 static void ixgbevf_netpoll(struct net_device *netdev) 3744 { 3745 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3746 int i; 3747 3748 /* if interface is down do nothing */ 3749 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 3750 return; 3751 for (i = 0; i < adapter->num_rx_queues; i++) 3752 ixgbevf_msix_clean_rings(0, adapter->q_vector[i]); 3753 } 3754 #endif /* CONFIG_NET_POLL_CONTROLLER */ 3755 3756 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) 3757 { 3758 struct net_device *netdev = pci_get_drvdata(pdev); 3759 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3760 #ifdef CONFIG_PM 3761 int retval = 0; 3762 #endif 3763 3764 netif_device_detach(netdev); 3765 3766 if (netif_running(netdev)) { 3767 rtnl_lock(); 3768 ixgbevf_down(adapter); 3769 ixgbevf_free_irq(adapter); 3770 ixgbevf_free_all_tx_resources(adapter); 3771 ixgbevf_free_all_rx_resources(adapter); 3772 rtnl_unlock(); 3773 } 3774 3775 ixgbevf_clear_interrupt_scheme(adapter); 3776 3777 #ifdef CONFIG_PM 3778 retval = pci_save_state(pdev); 3779 if (retval) 3780 return retval; 3781 3782 #endif 3783 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 3784 pci_disable_device(pdev); 3785 3786 return 0; 3787 } 3788 3789 #ifdef CONFIG_PM 3790 static int ixgbevf_resume(struct pci_dev *pdev) 3791 { 3792 struct net_device *netdev = pci_get_drvdata(pdev); 3793 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3794 u32 err; 3795 3796 pci_restore_state(pdev); 3797 /* 3798 * pci_restore_state clears dev->state_saved so call 3799 * pci_save_state to restore it. 3800 */ 3801 pci_save_state(pdev); 3802 3803 err = pci_enable_device_mem(pdev); 3804 if (err) { 3805 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 3806 return err; 3807 } 3808 smp_mb__before_atomic(); 3809 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 3810 pci_set_master(pdev); 3811 3812 ixgbevf_reset(adapter); 3813 3814 rtnl_lock(); 3815 err = ixgbevf_init_interrupt_scheme(adapter); 3816 rtnl_unlock(); 3817 if (err) { 3818 dev_err(&pdev->dev, "Cannot initialize interrupts\n"); 3819 return err; 3820 } 3821 3822 if (netif_running(netdev)) { 3823 err = ixgbevf_open(netdev); 3824 if (err) 3825 return err; 3826 } 3827 3828 netif_device_attach(netdev); 3829 3830 return err; 3831 } 3832 3833 #endif /* CONFIG_PM */ 3834 static void ixgbevf_shutdown(struct pci_dev *pdev) 3835 { 3836 ixgbevf_suspend(pdev, PMSG_SUSPEND); 3837 } 3838 3839 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, 3840 struct rtnl_link_stats64 *stats) 3841 { 3842 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3843 unsigned int start; 3844 u64 bytes, packets; 3845 const struct ixgbevf_ring *ring; 3846 int i; 3847 3848 ixgbevf_update_stats(adapter); 3849 3850 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3851 3852 for (i = 0; i < adapter->num_rx_queues; i++) { 3853 ring = adapter->rx_ring[i]; 3854 do { 3855 start = u64_stats_fetch_begin_irq(&ring->syncp); 3856 bytes = ring->stats.bytes; 3857 packets = ring->stats.packets; 3858 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 3859 stats->rx_bytes += bytes; 3860 stats->rx_packets += packets; 3861 } 3862 3863 for (i = 0; i < adapter->num_tx_queues; i++) { 3864 ring = adapter->tx_ring[i]; 3865 do { 3866 start = u64_stats_fetch_begin_irq(&ring->syncp); 3867 bytes = ring->stats.bytes; 3868 packets = ring->stats.packets; 3869 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 3870 stats->tx_bytes += bytes; 3871 stats->tx_packets += packets; 3872 } 3873 3874 return stats; 3875 } 3876 3877 static const struct net_device_ops ixgbevf_netdev_ops = { 3878 .ndo_open = ixgbevf_open, 3879 .ndo_stop = ixgbevf_close, 3880 .ndo_start_xmit = ixgbevf_xmit_frame, 3881 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 3882 .ndo_get_stats64 = ixgbevf_get_stats, 3883 .ndo_validate_addr = eth_validate_addr, 3884 .ndo_set_mac_address = ixgbevf_set_mac, 3885 .ndo_change_mtu = ixgbevf_change_mtu, 3886 .ndo_tx_timeout = ixgbevf_tx_timeout, 3887 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 3888 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 3889 #ifdef CONFIG_NET_RX_BUSY_POLL 3890 .ndo_busy_poll = ixgbevf_busy_poll_recv, 3891 #endif 3892 #ifdef CONFIG_NET_POLL_CONTROLLER 3893 .ndo_poll_controller = ixgbevf_netpoll, 3894 #endif 3895 }; 3896 3897 static void ixgbevf_assign_netdev_ops(struct net_device *dev) 3898 { 3899 dev->netdev_ops = &ixgbevf_netdev_ops; 3900 ixgbevf_set_ethtool_ops(dev); 3901 dev->watchdog_timeo = 5 * HZ; 3902 } 3903 3904 /** 3905 * ixgbevf_probe - Device Initialization Routine 3906 * @pdev: PCI device information struct 3907 * @ent: entry in ixgbevf_pci_tbl 3908 * 3909 * Returns 0 on success, negative on failure 3910 * 3911 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 3912 * The OS initialization, configuring of the adapter private structure, 3913 * and a hardware reset occur. 3914 **/ 3915 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3916 { 3917 struct net_device *netdev; 3918 struct ixgbevf_adapter *adapter = NULL; 3919 struct ixgbe_hw *hw = NULL; 3920 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 3921 int err, pci_using_dac; 3922 bool disable_dev = false; 3923 3924 err = pci_enable_device(pdev); 3925 if (err) 3926 return err; 3927 3928 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 3929 pci_using_dac = 1; 3930 } else { 3931 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 3932 if (err) { 3933 dev_err(&pdev->dev, "No usable DMA " 3934 "configuration, aborting\n"); 3935 goto err_dma; 3936 } 3937 pci_using_dac = 0; 3938 } 3939 3940 err = pci_request_regions(pdev, ixgbevf_driver_name); 3941 if (err) { 3942 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 3943 goto err_pci_reg; 3944 } 3945 3946 pci_set_master(pdev); 3947 3948 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 3949 MAX_TX_QUEUES); 3950 if (!netdev) { 3951 err = -ENOMEM; 3952 goto err_alloc_etherdev; 3953 } 3954 3955 SET_NETDEV_DEV(netdev, &pdev->dev); 3956 3957 adapter = netdev_priv(netdev); 3958 3959 adapter->netdev = netdev; 3960 adapter->pdev = pdev; 3961 hw = &adapter->hw; 3962 hw->back = adapter; 3963 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 3964 3965 /* 3966 * call save state here in standalone driver because it relies on 3967 * adapter struct to exist, and needs to call netdev_priv 3968 */ 3969 pci_save_state(pdev); 3970 3971 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3972 pci_resource_len(pdev, 0)); 3973 adapter->io_addr = hw->hw_addr; 3974 if (!hw->hw_addr) { 3975 err = -EIO; 3976 goto err_ioremap; 3977 } 3978 3979 ixgbevf_assign_netdev_ops(netdev); 3980 3981 /* Setup hw api */ 3982 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 3983 hw->mac.type = ii->mac; 3984 3985 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 3986 sizeof(struct ixgbe_mbx_operations)); 3987 3988 /* setup the private structure */ 3989 err = ixgbevf_sw_init(adapter); 3990 if (err) 3991 goto err_sw_init; 3992 3993 /* The HW MAC address was set and/or determined in sw_init */ 3994 if (!is_valid_ether_addr(netdev->dev_addr)) { 3995 pr_err("invalid MAC address\n"); 3996 err = -EIO; 3997 goto err_sw_init; 3998 } 3999 4000 netdev->hw_features = NETIF_F_SG | 4001 NETIF_F_IP_CSUM | 4002 NETIF_F_IPV6_CSUM | 4003 NETIF_F_TSO | 4004 NETIF_F_TSO6 | 4005 NETIF_F_RXCSUM; 4006 4007 netdev->features = netdev->hw_features | 4008 NETIF_F_HW_VLAN_CTAG_TX | 4009 NETIF_F_HW_VLAN_CTAG_RX | 4010 NETIF_F_HW_VLAN_CTAG_FILTER; 4011 4012 netdev->vlan_features |= NETIF_F_TSO | 4013 NETIF_F_TSO6 | 4014 NETIF_F_IP_CSUM | 4015 NETIF_F_IPV6_CSUM | 4016 NETIF_F_SG; 4017 4018 if (pci_using_dac) 4019 netdev->features |= NETIF_F_HIGHDMA; 4020 4021 netdev->priv_flags |= IFF_UNICAST_FLT; 4022 4023 if (IXGBE_REMOVED(hw->hw_addr)) { 4024 err = -EIO; 4025 goto err_sw_init; 4026 } 4027 4028 setup_timer(&adapter->service_timer, &ixgbevf_service_timer, 4029 (unsigned long)adapter); 4030 4031 INIT_WORK(&adapter->service_task, ixgbevf_service_task); 4032 set_bit(__IXGBEVF_SERVICE_INITED, &adapter->state); 4033 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 4034 4035 err = ixgbevf_init_interrupt_scheme(adapter); 4036 if (err) 4037 goto err_sw_init; 4038 4039 strcpy(netdev->name, "eth%d"); 4040 4041 err = register_netdev(netdev); 4042 if (err) 4043 goto err_register; 4044 4045 pci_set_drvdata(pdev, netdev); 4046 netif_carrier_off(netdev); 4047 4048 ixgbevf_init_last_counter_stats(adapter); 4049 4050 /* print the VF info */ 4051 dev_info(&pdev->dev, "%pM\n", netdev->dev_addr); 4052 dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type); 4053 4054 switch (hw->mac.type) { 4055 case ixgbe_mac_X550_vf: 4056 dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n"); 4057 break; 4058 case ixgbe_mac_X540_vf: 4059 dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n"); 4060 break; 4061 case ixgbe_mac_82599_vf: 4062 default: 4063 dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n"); 4064 break; 4065 } 4066 4067 return 0; 4068 4069 err_register: 4070 ixgbevf_clear_interrupt_scheme(adapter); 4071 err_sw_init: 4072 ixgbevf_reset_interrupt_capability(adapter); 4073 iounmap(adapter->io_addr); 4074 err_ioremap: 4075 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4076 free_netdev(netdev); 4077 err_alloc_etherdev: 4078 pci_release_regions(pdev); 4079 err_pci_reg: 4080 err_dma: 4081 if (!adapter || disable_dev) 4082 pci_disable_device(pdev); 4083 return err; 4084 } 4085 4086 /** 4087 * ixgbevf_remove - Device Removal Routine 4088 * @pdev: PCI device information struct 4089 * 4090 * ixgbevf_remove is called by the PCI subsystem to alert the driver 4091 * that it should release a PCI device. The could be caused by a 4092 * Hot-Plug event, or because the driver is going to be removed from 4093 * memory. 4094 **/ 4095 static void ixgbevf_remove(struct pci_dev *pdev) 4096 { 4097 struct net_device *netdev = pci_get_drvdata(pdev); 4098 struct ixgbevf_adapter *adapter; 4099 bool disable_dev; 4100 4101 if (!netdev) 4102 return; 4103 4104 adapter = netdev_priv(netdev); 4105 4106 set_bit(__IXGBEVF_REMOVING, &adapter->state); 4107 cancel_work_sync(&adapter->service_task); 4108 4109 if (netdev->reg_state == NETREG_REGISTERED) 4110 unregister_netdev(netdev); 4111 4112 ixgbevf_clear_interrupt_scheme(adapter); 4113 ixgbevf_reset_interrupt_capability(adapter); 4114 4115 iounmap(adapter->io_addr); 4116 pci_release_regions(pdev); 4117 4118 hw_dbg(&adapter->hw, "Remove complete\n"); 4119 4120 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4121 free_netdev(netdev); 4122 4123 if (disable_dev) 4124 pci_disable_device(pdev); 4125 } 4126 4127 /** 4128 * ixgbevf_io_error_detected - called when PCI error is detected 4129 * @pdev: Pointer to PCI device 4130 * @state: The current pci connection state 4131 * 4132 * This function is called after a PCI bus error affecting 4133 * this device has been detected. 4134 */ 4135 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, 4136 pci_channel_state_t state) 4137 { 4138 struct net_device *netdev = pci_get_drvdata(pdev); 4139 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4140 4141 if (!test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 4142 return PCI_ERS_RESULT_DISCONNECT; 4143 4144 rtnl_lock(); 4145 netif_device_detach(netdev); 4146 4147 if (state == pci_channel_io_perm_failure) { 4148 rtnl_unlock(); 4149 return PCI_ERS_RESULT_DISCONNECT; 4150 } 4151 4152 if (netif_running(netdev)) 4153 ixgbevf_down(adapter); 4154 4155 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 4156 pci_disable_device(pdev); 4157 rtnl_unlock(); 4158 4159 /* Request a slot slot reset. */ 4160 return PCI_ERS_RESULT_NEED_RESET; 4161 } 4162 4163 /** 4164 * ixgbevf_io_slot_reset - called after the pci bus has been reset. 4165 * @pdev: Pointer to PCI device 4166 * 4167 * Restart the card from scratch, as if from a cold-boot. Implementation 4168 * resembles the first-half of the ixgbevf_resume routine. 4169 */ 4170 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) 4171 { 4172 struct net_device *netdev = pci_get_drvdata(pdev); 4173 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4174 4175 if (pci_enable_device_mem(pdev)) { 4176 dev_err(&pdev->dev, 4177 "Cannot re-enable PCI device after reset.\n"); 4178 return PCI_ERS_RESULT_DISCONNECT; 4179 } 4180 4181 smp_mb__before_atomic(); 4182 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 4183 pci_set_master(pdev); 4184 4185 ixgbevf_reset(adapter); 4186 4187 return PCI_ERS_RESULT_RECOVERED; 4188 } 4189 4190 /** 4191 * ixgbevf_io_resume - called when traffic can start flowing again. 4192 * @pdev: Pointer to PCI device 4193 * 4194 * This callback is called when the error recovery driver tells us that 4195 * its OK to resume normal operation. Implementation resembles the 4196 * second-half of the ixgbevf_resume routine. 4197 */ 4198 static void ixgbevf_io_resume(struct pci_dev *pdev) 4199 { 4200 struct net_device *netdev = pci_get_drvdata(pdev); 4201 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4202 4203 if (netif_running(netdev)) 4204 ixgbevf_up(adapter); 4205 4206 netif_device_attach(netdev); 4207 } 4208 4209 /* PCI Error Recovery (ERS) */ 4210 static const struct pci_error_handlers ixgbevf_err_handler = { 4211 .error_detected = ixgbevf_io_error_detected, 4212 .slot_reset = ixgbevf_io_slot_reset, 4213 .resume = ixgbevf_io_resume, 4214 }; 4215 4216 static struct pci_driver ixgbevf_driver = { 4217 .name = ixgbevf_driver_name, 4218 .id_table = ixgbevf_pci_tbl, 4219 .probe = ixgbevf_probe, 4220 .remove = ixgbevf_remove, 4221 #ifdef CONFIG_PM 4222 /* Power Management Hooks */ 4223 .suspend = ixgbevf_suspend, 4224 .resume = ixgbevf_resume, 4225 #endif 4226 .shutdown = ixgbevf_shutdown, 4227 .err_handler = &ixgbevf_err_handler 4228 }; 4229 4230 /** 4231 * ixgbevf_init_module - Driver Registration Routine 4232 * 4233 * ixgbevf_init_module is the first routine called when the driver is 4234 * loaded. All it does is register with the PCI subsystem. 4235 **/ 4236 static int __init ixgbevf_init_module(void) 4237 { 4238 int ret; 4239 pr_info("%s - version %s\n", ixgbevf_driver_string, 4240 ixgbevf_driver_version); 4241 4242 pr_info("%s\n", ixgbevf_copyright); 4243 4244 ret = pci_register_driver(&ixgbevf_driver); 4245 return ret; 4246 } 4247 4248 module_init(ixgbevf_init_module); 4249 4250 /** 4251 * ixgbevf_exit_module - Driver Exit Cleanup Routine 4252 * 4253 * ixgbevf_exit_module is called just before the driver is removed 4254 * from memory. 4255 **/ 4256 static void __exit ixgbevf_exit_module(void) 4257 { 4258 pci_unregister_driver(&ixgbevf_driver); 4259 } 4260 4261 #ifdef DEBUG 4262 /** 4263 * ixgbevf_get_hw_dev_name - return device name string 4264 * used by hardware layer to print debugging information 4265 **/ 4266 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 4267 { 4268 struct ixgbevf_adapter *adapter = hw->back; 4269 return adapter->netdev->name; 4270 } 4271 4272 #endif 4273 module_exit(ixgbevf_exit_module); 4274 4275 /* ixgbevf_main.c */ 4276