1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2018 Intel Corporation. */ 3 4 #include "iavf.h" 5 #include "iavf_prototype.h" 6 #include "iavf_client.h" 7 /* All iavf tracepoints are defined by the include below, which must 8 * be included exactly once across the whole kernel with 9 * CREATE_TRACE_POINTS defined 10 */ 11 #define CREATE_TRACE_POINTS 12 #include "iavf_trace.h" 13 14 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter); 15 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter); 16 static int iavf_close(struct net_device *netdev); 17 18 char iavf_driver_name[] = "iavf"; 19 static const char iavf_driver_string[] = 20 "Intel(R) Ethernet Adaptive Virtual Function Network Driver"; 21 22 #define DRV_KERN "-k" 23 24 #define DRV_VERSION_MAJOR 3 25 #define DRV_VERSION_MINOR 2 26 #define DRV_VERSION_BUILD 3 27 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ 28 __stringify(DRV_VERSION_MINOR) "." \ 29 __stringify(DRV_VERSION_BUILD) \ 30 DRV_KERN 31 const char iavf_driver_version[] = DRV_VERSION; 32 static const char iavf_copyright[] = 33 "Copyright (c) 2013 - 2018 Intel Corporation."; 34 35 /* iavf_pci_tbl - PCI Device ID Table 36 * 37 * Wildcard entries (PCI_ANY_ID) should come last 38 * Last entry must be all 0s 39 * 40 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 41 * Class, Class Mask, private data (not used) } 42 */ 43 static const struct pci_device_id iavf_pci_tbl[] = { 44 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0}, 45 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0}, 46 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0}, 47 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0}, 48 /* required last entry */ 49 {0, } 50 }; 51 52 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl); 53 54 MODULE_ALIAS("i40evf"); 55 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 56 MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver"); 57 MODULE_LICENSE("GPL v2"); 58 MODULE_VERSION(DRV_VERSION); 59 60 static struct workqueue_struct *iavf_wq; 61 62 /** 63 * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code 64 * @hw: pointer to the HW structure 65 * @mem: ptr to mem struct to fill out 66 * @size: size of memory requested 67 * @alignment: what to align the allocation to 68 **/ 69 iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw, 70 struct iavf_dma_mem *mem, 71 u64 size, u32 alignment) 72 { 73 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back; 74 75 if (!mem) 76 return I40E_ERR_PARAM; 77 78 mem->size = ALIGN(size, alignment); 79 mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size, 80 (dma_addr_t *)&mem->pa, GFP_KERNEL); 81 if (mem->va) 82 return 0; 83 else 84 return I40E_ERR_NO_MEMORY; 85 } 86 87 /** 88 * iavf_free_dma_mem_d - OS specific memory free for shared code 89 * @hw: pointer to the HW structure 90 * @mem: ptr to mem struct to free 91 **/ 92 iavf_status iavf_free_dma_mem_d(struct iavf_hw *hw, struct iavf_dma_mem *mem) 93 { 94 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back; 95 96 if (!mem || !mem->va) 97 return I40E_ERR_PARAM; 98 dma_free_coherent(&adapter->pdev->dev, mem->size, 99 mem->va, (dma_addr_t)mem->pa); 100 return 0; 101 } 102 103 /** 104 * iavf_allocate_virt_mem_d - OS specific memory alloc for shared code 105 * @hw: pointer to the HW structure 106 * @mem: ptr to mem struct to fill out 107 * @size: size of memory requested 108 **/ 109 iavf_status iavf_allocate_virt_mem_d(struct iavf_hw *hw, 110 struct iavf_virt_mem *mem, u32 size) 111 { 112 if (!mem) 113 return I40E_ERR_PARAM; 114 115 mem->size = size; 116 mem->va = kzalloc(size, GFP_KERNEL); 117 118 if (mem->va) 119 return 0; 120 else 121 return I40E_ERR_NO_MEMORY; 122 } 123 124 /** 125 * iavf_free_virt_mem_d - OS specific memory free for shared code 126 * @hw: pointer to the HW structure 127 * @mem: ptr to mem struct to free 128 **/ 129 iavf_status iavf_free_virt_mem_d(struct iavf_hw *hw, struct iavf_virt_mem *mem) 130 { 131 if (!mem) 132 return I40E_ERR_PARAM; 133 134 /* it's ok to kfree a NULL pointer */ 135 kfree(mem->va); 136 137 return 0; 138 } 139 140 /** 141 * iavf_debug_d - OS dependent version of debug printing 142 * @hw: pointer to the HW structure 143 * @mask: debug level mask 144 * @fmt_str: printf-type format description 145 **/ 146 void iavf_debug_d(void *hw, u32 mask, char *fmt_str, ...) 147 { 148 char buf[512]; 149 va_list argptr; 150 151 if (!(mask & ((struct iavf_hw *)hw)->debug_mask)) 152 return; 153 154 va_start(argptr, fmt_str); 155 vsnprintf(buf, sizeof(buf), fmt_str, argptr); 156 va_end(argptr); 157 158 /* the debug string is already formatted with a newline */ 159 pr_info("%s", buf); 160 } 161 162 /** 163 * iavf_schedule_reset - Set the flags and schedule a reset event 164 * @adapter: board private structure 165 **/ 166 void iavf_schedule_reset(struct iavf_adapter *adapter) 167 { 168 if (!(adapter->flags & 169 (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) { 170 adapter->flags |= IAVF_FLAG_RESET_NEEDED; 171 schedule_work(&adapter->reset_task); 172 } 173 } 174 175 /** 176 * iavf_tx_timeout - Respond to a Tx Hang 177 * @netdev: network interface device structure 178 **/ 179 static void iavf_tx_timeout(struct net_device *netdev) 180 { 181 struct iavf_adapter *adapter = netdev_priv(netdev); 182 183 adapter->tx_timeout_count++; 184 iavf_schedule_reset(adapter); 185 } 186 187 /** 188 * iavf_misc_irq_disable - Mask off interrupt generation on the NIC 189 * @adapter: board private structure 190 **/ 191 static void iavf_misc_irq_disable(struct iavf_adapter *adapter) 192 { 193 struct iavf_hw *hw = &adapter->hw; 194 195 if (!adapter->msix_entries) 196 return; 197 198 wr32(hw, IAVF_VFINT_DYN_CTL01, 0); 199 200 iavf_flush(hw); 201 202 synchronize_irq(adapter->msix_entries[0].vector); 203 } 204 205 /** 206 * iavf_misc_irq_enable - Enable default interrupt generation settings 207 * @adapter: board private structure 208 **/ 209 static void iavf_misc_irq_enable(struct iavf_adapter *adapter) 210 { 211 struct iavf_hw *hw = &adapter->hw; 212 213 wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK | 214 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 215 wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 216 217 iavf_flush(hw); 218 } 219 220 /** 221 * iavf_irq_disable - Mask off interrupt generation on the NIC 222 * @adapter: board private structure 223 **/ 224 static void iavf_irq_disable(struct iavf_adapter *adapter) 225 { 226 int i; 227 struct iavf_hw *hw = &adapter->hw; 228 229 if (!adapter->msix_entries) 230 return; 231 232 for (i = 1; i < adapter->num_msix_vectors; i++) { 233 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0); 234 synchronize_irq(adapter->msix_entries[i].vector); 235 } 236 iavf_flush(hw); 237 } 238 239 /** 240 * iavf_irq_enable_queues - Enable interrupt for specified queues 241 * @adapter: board private structure 242 * @mask: bitmap of queues to enable 243 **/ 244 void iavf_irq_enable_queues(struct iavf_adapter *adapter, u32 mask) 245 { 246 struct iavf_hw *hw = &adapter->hw; 247 int i; 248 249 for (i = 1; i < adapter->num_msix_vectors; i++) { 250 if (mask & BIT(i - 1)) { 251 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 252 IAVF_VFINT_DYN_CTLN1_INTENA_MASK | 253 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK); 254 } 255 } 256 } 257 258 /** 259 * iavf_irq_enable - Enable default interrupt generation settings 260 * @adapter: board private structure 261 * @flush: boolean value whether to run rd32() 262 **/ 263 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush) 264 { 265 struct iavf_hw *hw = &adapter->hw; 266 267 iavf_misc_irq_enable(adapter); 268 iavf_irq_enable_queues(adapter, ~0); 269 270 if (flush) 271 iavf_flush(hw); 272 } 273 274 /** 275 * iavf_msix_aq - Interrupt handler for vector 0 276 * @irq: interrupt number 277 * @data: pointer to netdev 278 **/ 279 static irqreturn_t iavf_msix_aq(int irq, void *data) 280 { 281 struct net_device *netdev = data; 282 struct iavf_adapter *adapter = netdev_priv(netdev); 283 struct iavf_hw *hw = &adapter->hw; 284 285 /* handle non-queue interrupts, these reads clear the registers */ 286 rd32(hw, IAVF_VFINT_ICR01); 287 rd32(hw, IAVF_VFINT_ICR0_ENA1); 288 289 /* schedule work on the private workqueue */ 290 schedule_work(&adapter->adminq_task); 291 292 return IRQ_HANDLED; 293 } 294 295 /** 296 * iavf_msix_clean_rings - MSIX mode Interrupt Handler 297 * @irq: interrupt number 298 * @data: pointer to a q_vector 299 **/ 300 static irqreturn_t iavf_msix_clean_rings(int irq, void *data) 301 { 302 struct iavf_q_vector *q_vector = data; 303 304 if (!q_vector->tx.ring && !q_vector->rx.ring) 305 return IRQ_HANDLED; 306 307 napi_schedule_irqoff(&q_vector->napi); 308 309 return IRQ_HANDLED; 310 } 311 312 /** 313 * iavf_map_vector_to_rxq - associate irqs with rx queues 314 * @adapter: board private structure 315 * @v_idx: interrupt number 316 * @r_idx: queue number 317 **/ 318 static void 319 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx) 320 { 321 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx]; 322 struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx]; 323 struct iavf_hw *hw = &adapter->hw; 324 325 rx_ring->q_vector = q_vector; 326 rx_ring->next = q_vector->rx.ring; 327 rx_ring->vsi = &adapter->vsi; 328 q_vector->rx.ring = rx_ring; 329 q_vector->rx.count++; 330 q_vector->rx.next_update = jiffies + 1; 331 q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting); 332 q_vector->ring_mask |= BIT(r_idx); 333 wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx), 334 q_vector->rx.current_itr); 335 q_vector->rx.current_itr = q_vector->rx.target_itr; 336 } 337 338 /** 339 * iavf_map_vector_to_txq - associate irqs with tx queues 340 * @adapter: board private structure 341 * @v_idx: interrupt number 342 * @t_idx: queue number 343 **/ 344 static void 345 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx) 346 { 347 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx]; 348 struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx]; 349 struct iavf_hw *hw = &adapter->hw; 350 351 tx_ring->q_vector = q_vector; 352 tx_ring->next = q_vector->tx.ring; 353 tx_ring->vsi = &adapter->vsi; 354 q_vector->tx.ring = tx_ring; 355 q_vector->tx.count++; 356 q_vector->tx.next_update = jiffies + 1; 357 q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting); 358 q_vector->num_ringpairs++; 359 wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx), 360 q_vector->tx.target_itr); 361 q_vector->tx.current_itr = q_vector->tx.target_itr; 362 } 363 364 /** 365 * iavf_map_rings_to_vectors - Maps descriptor rings to vectors 366 * @adapter: board private structure to initialize 367 * 368 * This function maps descriptor rings to the queue-specific vectors 369 * we were allotted through the MSI-X enabling code. Ideally, we'd have 370 * one vector per ring/queue, but on a constrained vector budget, we 371 * group the rings as "efficiently" as possible. You would add new 372 * mapping configurations in here. 373 **/ 374 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter) 375 { 376 int rings_remaining = adapter->num_active_queues; 377 int ridx = 0, vidx = 0; 378 int q_vectors; 379 380 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 381 382 for (; ridx < rings_remaining; ridx++) { 383 iavf_map_vector_to_rxq(adapter, vidx, ridx); 384 iavf_map_vector_to_txq(adapter, vidx, ridx); 385 386 /* In the case where we have more queues than vectors, continue 387 * round-robin on vectors until all queues are mapped. 388 */ 389 if (++vidx >= q_vectors) 390 vidx = 0; 391 } 392 393 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS; 394 } 395 396 /** 397 * iavf_irq_affinity_notify - Callback for affinity changes 398 * @notify: context as to what irq was changed 399 * @mask: the new affinity mask 400 * 401 * This is a callback function used by the irq_set_affinity_notifier function 402 * so that we may register to receive changes to the irq affinity masks. 403 **/ 404 static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify, 405 const cpumask_t *mask) 406 { 407 struct iavf_q_vector *q_vector = 408 container_of(notify, struct iavf_q_vector, affinity_notify); 409 410 cpumask_copy(&q_vector->affinity_mask, mask); 411 } 412 413 /** 414 * iavf_irq_affinity_release - Callback for affinity notifier release 415 * @ref: internal core kernel usage 416 * 417 * This is a callback function used by the irq_set_affinity_notifier function 418 * to inform the current notification subscriber that they will no longer 419 * receive notifications. 420 **/ 421 static void iavf_irq_affinity_release(struct kref *ref) {} 422 423 /** 424 * iavf_request_traffic_irqs - Initialize MSI-X interrupts 425 * @adapter: board private structure 426 * @basename: device basename 427 * 428 * Allocates MSI-X vectors for tx and rx handling, and requests 429 * interrupts from the kernel. 430 **/ 431 static int 432 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename) 433 { 434 unsigned int vector, q_vectors; 435 unsigned int rx_int_idx = 0, tx_int_idx = 0; 436 int irq_num, err; 437 int cpu; 438 439 iavf_irq_disable(adapter); 440 /* Decrement for Other and TCP Timer vectors */ 441 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 442 443 for (vector = 0; vector < q_vectors; vector++) { 444 struct iavf_q_vector *q_vector = &adapter->q_vectors[vector]; 445 446 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 447 448 if (q_vector->tx.ring && q_vector->rx.ring) { 449 snprintf(q_vector->name, sizeof(q_vector->name), 450 "iavf-%s-TxRx-%d", basename, rx_int_idx++); 451 tx_int_idx++; 452 } else if (q_vector->rx.ring) { 453 snprintf(q_vector->name, sizeof(q_vector->name), 454 "iavf-%s-rx-%d", basename, rx_int_idx++); 455 } else if (q_vector->tx.ring) { 456 snprintf(q_vector->name, sizeof(q_vector->name), 457 "iavf-%s-tx-%d", basename, tx_int_idx++); 458 } else { 459 /* skip this unused q_vector */ 460 continue; 461 } 462 err = request_irq(irq_num, 463 iavf_msix_clean_rings, 464 0, 465 q_vector->name, 466 q_vector); 467 if (err) { 468 dev_info(&adapter->pdev->dev, 469 "Request_irq failed, error: %d\n", err); 470 goto free_queue_irqs; 471 } 472 /* register for affinity change notifications */ 473 q_vector->affinity_notify.notify = iavf_irq_affinity_notify; 474 q_vector->affinity_notify.release = 475 iavf_irq_affinity_release; 476 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 477 /* Spread the IRQ affinity hints across online CPUs. Note that 478 * get_cpu_mask returns a mask with a permanent lifetime so 479 * it's safe to use as a hint for irq_set_affinity_hint. 480 */ 481 cpu = cpumask_local_spread(q_vector->v_idx, -1); 482 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu)); 483 } 484 485 return 0; 486 487 free_queue_irqs: 488 while (vector) { 489 vector--; 490 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 491 irq_set_affinity_notifier(irq_num, NULL); 492 irq_set_affinity_hint(irq_num, NULL); 493 free_irq(irq_num, &adapter->q_vectors[vector]); 494 } 495 return err; 496 } 497 498 /** 499 * iavf_request_misc_irq - Initialize MSI-X interrupts 500 * @adapter: board private structure 501 * 502 * Allocates MSI-X vector 0 and requests interrupts from the kernel. This 503 * vector is only for the admin queue, and stays active even when the netdev 504 * is closed. 505 **/ 506 static int iavf_request_misc_irq(struct iavf_adapter *adapter) 507 { 508 struct net_device *netdev = adapter->netdev; 509 int err; 510 511 snprintf(adapter->misc_vector_name, 512 sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx", 513 dev_name(&adapter->pdev->dev)); 514 err = request_irq(adapter->msix_entries[0].vector, 515 &iavf_msix_aq, 0, 516 adapter->misc_vector_name, netdev); 517 if (err) { 518 dev_err(&adapter->pdev->dev, 519 "request_irq for %s failed: %d\n", 520 adapter->misc_vector_name, err); 521 free_irq(adapter->msix_entries[0].vector, netdev); 522 } 523 return err; 524 } 525 526 /** 527 * iavf_free_traffic_irqs - Free MSI-X interrupts 528 * @adapter: board private structure 529 * 530 * Frees all MSI-X vectors other than 0. 531 **/ 532 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter) 533 { 534 int vector, irq_num, q_vectors; 535 536 if (!adapter->msix_entries) 537 return; 538 539 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 540 541 for (vector = 0; vector < q_vectors; vector++) { 542 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 543 irq_set_affinity_notifier(irq_num, NULL); 544 irq_set_affinity_hint(irq_num, NULL); 545 free_irq(irq_num, &adapter->q_vectors[vector]); 546 } 547 } 548 549 /** 550 * iavf_free_misc_irq - Free MSI-X miscellaneous vector 551 * @adapter: board private structure 552 * 553 * Frees MSI-X vector 0. 554 **/ 555 static void iavf_free_misc_irq(struct iavf_adapter *adapter) 556 { 557 struct net_device *netdev = adapter->netdev; 558 559 if (!adapter->msix_entries) 560 return; 561 562 free_irq(adapter->msix_entries[0].vector, netdev); 563 } 564 565 /** 566 * iavf_configure_tx - Configure Transmit Unit after Reset 567 * @adapter: board private structure 568 * 569 * Configure the Tx unit of the MAC after a reset. 570 **/ 571 static void iavf_configure_tx(struct iavf_adapter *adapter) 572 { 573 struct iavf_hw *hw = &adapter->hw; 574 int i; 575 576 for (i = 0; i < adapter->num_active_queues; i++) 577 adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i); 578 } 579 580 /** 581 * iavf_configure_rx - Configure Receive Unit after Reset 582 * @adapter: board private structure 583 * 584 * Configure the Rx unit of the MAC after a reset. 585 **/ 586 static void iavf_configure_rx(struct iavf_adapter *adapter) 587 { 588 unsigned int rx_buf_len = IAVF_RXBUFFER_2048; 589 struct iavf_hw *hw = &adapter->hw; 590 int i; 591 592 /* Legacy Rx will always default to a 2048 buffer size. */ 593 #if (PAGE_SIZE < 8192) 594 if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) { 595 struct net_device *netdev = adapter->netdev; 596 597 /* For jumbo frames on systems with 4K pages we have to use 598 * an order 1 page, so we might as well increase the size 599 * of our Rx buffer to make better use of the available space 600 */ 601 rx_buf_len = IAVF_RXBUFFER_3072; 602 603 /* We use a 1536 buffer size for configurations with 604 * standard Ethernet mtu. On x86 this gives us enough room 605 * for shared info and 192 bytes of padding. 606 */ 607 if (!IAVF_2K_TOO_SMALL_WITH_PADDING && 608 (netdev->mtu <= ETH_DATA_LEN)) 609 rx_buf_len = IAVF_RXBUFFER_1536 - NET_IP_ALIGN; 610 } 611 #endif 612 613 for (i = 0; i < adapter->num_active_queues; i++) { 614 adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i); 615 adapter->rx_rings[i].rx_buf_len = rx_buf_len; 616 617 if (adapter->flags & IAVF_FLAG_LEGACY_RX) 618 clear_ring_build_skb_enabled(&adapter->rx_rings[i]); 619 else 620 set_ring_build_skb_enabled(&adapter->rx_rings[i]); 621 } 622 } 623 624 /** 625 * iavf_find_vlan - Search filter list for specific vlan filter 626 * @adapter: board private structure 627 * @vlan: vlan tag 628 * 629 * Returns ptr to the filter object or NULL. Must be called while holding the 630 * mac_vlan_list_lock. 631 **/ 632 static struct 633 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter, u16 vlan) 634 { 635 struct iavf_vlan_filter *f; 636 637 list_for_each_entry(f, &adapter->vlan_filter_list, list) { 638 if (vlan == f->vlan) 639 return f; 640 } 641 return NULL; 642 } 643 644 /** 645 * iavf_add_vlan - Add a vlan filter to the list 646 * @adapter: board private structure 647 * @vlan: VLAN tag 648 * 649 * Returns ptr to the filter object or NULL when no memory available. 650 **/ 651 static struct 652 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, u16 vlan) 653 { 654 struct iavf_vlan_filter *f = NULL; 655 656 spin_lock_bh(&adapter->mac_vlan_list_lock); 657 658 f = iavf_find_vlan(adapter, vlan); 659 if (!f) { 660 f = kzalloc(sizeof(*f), GFP_KERNEL); 661 if (!f) 662 goto clearout; 663 664 f->vlan = vlan; 665 666 INIT_LIST_HEAD(&f->list); 667 list_add(&f->list, &adapter->vlan_filter_list); 668 f->add = true; 669 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER; 670 } 671 672 clearout: 673 spin_unlock_bh(&adapter->mac_vlan_list_lock); 674 return f; 675 } 676 677 /** 678 * iavf_del_vlan - Remove a vlan filter from the list 679 * @adapter: board private structure 680 * @vlan: VLAN tag 681 **/ 682 static void iavf_del_vlan(struct iavf_adapter *adapter, u16 vlan) 683 { 684 struct iavf_vlan_filter *f; 685 686 spin_lock_bh(&adapter->mac_vlan_list_lock); 687 688 f = iavf_find_vlan(adapter, vlan); 689 if (f) { 690 f->remove = true; 691 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER; 692 } 693 694 spin_unlock_bh(&adapter->mac_vlan_list_lock); 695 } 696 697 /** 698 * iavf_vlan_rx_add_vid - Add a VLAN filter to a device 699 * @netdev: network device struct 700 * @proto: unused protocol data 701 * @vid: VLAN tag 702 **/ 703 static int iavf_vlan_rx_add_vid(struct net_device *netdev, 704 __always_unused __be16 proto, u16 vid) 705 { 706 struct iavf_adapter *adapter = netdev_priv(netdev); 707 708 if (!VLAN_ALLOWED(adapter)) 709 return -EIO; 710 if (iavf_add_vlan(adapter, vid) == NULL) 711 return -ENOMEM; 712 return 0; 713 } 714 715 /** 716 * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device 717 * @netdev: network device struct 718 * @proto: unused protocol data 719 * @vid: VLAN tag 720 **/ 721 static int iavf_vlan_rx_kill_vid(struct net_device *netdev, 722 __always_unused __be16 proto, u16 vid) 723 { 724 struct iavf_adapter *adapter = netdev_priv(netdev); 725 726 if (VLAN_ALLOWED(adapter)) { 727 iavf_del_vlan(adapter, vid); 728 return 0; 729 } 730 return -EIO; 731 } 732 733 /** 734 * iavf_find_filter - Search filter list for specific mac filter 735 * @adapter: board private structure 736 * @macaddr: the MAC address 737 * 738 * Returns ptr to the filter object or NULL. Must be called while holding the 739 * mac_vlan_list_lock. 740 **/ 741 static struct 742 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter, 743 const u8 *macaddr) 744 { 745 struct iavf_mac_filter *f; 746 747 if (!macaddr) 748 return NULL; 749 750 list_for_each_entry(f, &adapter->mac_filter_list, list) { 751 if (ether_addr_equal(macaddr, f->macaddr)) 752 return f; 753 } 754 return NULL; 755 } 756 757 /** 758 * iavf_add_filter - Add a mac filter to the filter list 759 * @adapter: board private structure 760 * @macaddr: the MAC address 761 * 762 * Returns ptr to the filter object or NULL when no memory available. 763 **/ 764 static struct 765 iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter, 766 const u8 *macaddr) 767 { 768 struct iavf_mac_filter *f; 769 770 if (!macaddr) 771 return NULL; 772 773 f = iavf_find_filter(adapter, macaddr); 774 if (!f) { 775 f = kzalloc(sizeof(*f), GFP_ATOMIC); 776 if (!f) 777 return f; 778 779 ether_addr_copy(f->macaddr, macaddr); 780 781 list_add_tail(&f->list, &adapter->mac_filter_list); 782 f->add = true; 783 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; 784 } else { 785 f->remove = false; 786 } 787 788 return f; 789 } 790 791 /** 792 * iavf_set_mac - NDO callback to set port mac address 793 * @netdev: network interface device structure 794 * @p: pointer to an address structure 795 * 796 * Returns 0 on success, negative on failure 797 **/ 798 static int iavf_set_mac(struct net_device *netdev, void *p) 799 { 800 struct iavf_adapter *adapter = netdev_priv(netdev); 801 struct iavf_hw *hw = &adapter->hw; 802 struct iavf_mac_filter *f; 803 struct sockaddr *addr = p; 804 805 if (!is_valid_ether_addr(addr->sa_data)) 806 return -EADDRNOTAVAIL; 807 808 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) 809 return 0; 810 811 if (adapter->flags & IAVF_FLAG_ADDR_SET_BY_PF) 812 return -EPERM; 813 814 spin_lock_bh(&adapter->mac_vlan_list_lock); 815 816 f = iavf_find_filter(adapter, hw->mac.addr); 817 if (f) { 818 f->remove = true; 819 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; 820 } 821 822 f = iavf_add_filter(adapter, addr->sa_data); 823 824 spin_unlock_bh(&adapter->mac_vlan_list_lock); 825 826 if (f) { 827 ether_addr_copy(hw->mac.addr, addr->sa_data); 828 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr); 829 } 830 831 return (f == NULL) ? -ENOMEM : 0; 832 } 833 834 /** 835 * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address 836 * @netdev: the netdevice 837 * @addr: address to add 838 * 839 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 840 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 841 */ 842 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr) 843 { 844 struct iavf_adapter *adapter = netdev_priv(netdev); 845 846 if (iavf_add_filter(adapter, addr)) 847 return 0; 848 else 849 return -ENOMEM; 850 } 851 852 /** 853 * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 854 * @netdev: the netdevice 855 * @addr: address to add 856 * 857 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 858 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 859 */ 860 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr) 861 { 862 struct iavf_adapter *adapter = netdev_priv(netdev); 863 struct iavf_mac_filter *f; 864 865 /* Under some circumstances, we might receive a request to delete 866 * our own device address from our uc list. Because we store the 867 * device address in the VSI's MAC/VLAN filter list, we need to ignore 868 * such requests and not delete our device address from this list. 869 */ 870 if (ether_addr_equal(addr, netdev->dev_addr)) 871 return 0; 872 873 f = iavf_find_filter(adapter, addr); 874 if (f) { 875 f->remove = true; 876 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; 877 } 878 return 0; 879 } 880 881 /** 882 * iavf_set_rx_mode - NDO callback to set the netdev filters 883 * @netdev: network interface device structure 884 **/ 885 static void iavf_set_rx_mode(struct net_device *netdev) 886 { 887 struct iavf_adapter *adapter = netdev_priv(netdev); 888 889 spin_lock_bh(&adapter->mac_vlan_list_lock); 890 __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync); 891 __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync); 892 spin_unlock_bh(&adapter->mac_vlan_list_lock); 893 894 if (netdev->flags & IFF_PROMISC && 895 !(adapter->flags & IAVF_FLAG_PROMISC_ON)) 896 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_PROMISC; 897 else if (!(netdev->flags & IFF_PROMISC) && 898 adapter->flags & IAVF_FLAG_PROMISC_ON) 899 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_PROMISC; 900 901 if (netdev->flags & IFF_ALLMULTI && 902 !(adapter->flags & IAVF_FLAG_ALLMULTI_ON)) 903 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_ALLMULTI; 904 else if (!(netdev->flags & IFF_ALLMULTI) && 905 adapter->flags & IAVF_FLAG_ALLMULTI_ON) 906 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_ALLMULTI; 907 } 908 909 /** 910 * iavf_napi_enable_all - enable NAPI on all queue vectors 911 * @adapter: board private structure 912 **/ 913 static void iavf_napi_enable_all(struct iavf_adapter *adapter) 914 { 915 int q_idx; 916 struct iavf_q_vector *q_vector; 917 int q_vectors = adapter->num_msix_vectors - NONQ_VECS; 918 919 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 920 struct napi_struct *napi; 921 922 q_vector = &adapter->q_vectors[q_idx]; 923 napi = &q_vector->napi; 924 napi_enable(napi); 925 } 926 } 927 928 /** 929 * iavf_napi_disable_all - disable NAPI on all queue vectors 930 * @adapter: board private structure 931 **/ 932 static void iavf_napi_disable_all(struct iavf_adapter *adapter) 933 { 934 int q_idx; 935 struct iavf_q_vector *q_vector; 936 int q_vectors = adapter->num_msix_vectors - NONQ_VECS; 937 938 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 939 q_vector = &adapter->q_vectors[q_idx]; 940 napi_disable(&q_vector->napi); 941 } 942 } 943 944 /** 945 * iavf_configure - set up transmit and receive data structures 946 * @adapter: board private structure 947 **/ 948 static void iavf_configure(struct iavf_adapter *adapter) 949 { 950 struct net_device *netdev = adapter->netdev; 951 int i; 952 953 iavf_set_rx_mode(netdev); 954 955 iavf_configure_tx(adapter); 956 iavf_configure_rx(adapter); 957 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES; 958 959 for (i = 0; i < adapter->num_active_queues; i++) { 960 struct iavf_ring *ring = &adapter->rx_rings[i]; 961 962 iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring)); 963 } 964 } 965 966 /** 967 * iavf_up_complete - Finish the last steps of bringing up a connection 968 * @adapter: board private structure 969 * 970 * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock. 971 **/ 972 static void iavf_up_complete(struct iavf_adapter *adapter) 973 { 974 adapter->state = __IAVF_RUNNING; 975 clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 976 977 iavf_napi_enable_all(adapter); 978 979 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES; 980 if (CLIENT_ENABLED(adapter)) 981 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN; 982 mod_timer_pending(&adapter->watchdog_timer, jiffies + 1); 983 } 984 985 /** 986 * iavf_down - Shutdown the connection processing 987 * @adapter: board private structure 988 * 989 * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock. 990 **/ 991 void iavf_down(struct iavf_adapter *adapter) 992 { 993 struct net_device *netdev = adapter->netdev; 994 struct iavf_vlan_filter *vlf; 995 struct iavf_mac_filter *f; 996 struct iavf_cloud_filter *cf; 997 998 if (adapter->state <= __IAVF_DOWN_PENDING) 999 return; 1000 1001 netif_carrier_off(netdev); 1002 netif_tx_disable(netdev); 1003 adapter->link_up = false; 1004 iavf_napi_disable_all(adapter); 1005 iavf_irq_disable(adapter); 1006 1007 spin_lock_bh(&adapter->mac_vlan_list_lock); 1008 1009 /* clear the sync flag on all filters */ 1010 __dev_uc_unsync(adapter->netdev, NULL); 1011 __dev_mc_unsync(adapter->netdev, NULL); 1012 1013 /* remove all MAC filters */ 1014 list_for_each_entry(f, &adapter->mac_filter_list, list) { 1015 f->remove = true; 1016 } 1017 1018 /* remove all VLAN filters */ 1019 list_for_each_entry(vlf, &adapter->vlan_filter_list, list) { 1020 vlf->remove = true; 1021 } 1022 1023 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1024 1025 /* remove all cloud filters */ 1026 spin_lock_bh(&adapter->cloud_filter_list_lock); 1027 list_for_each_entry(cf, &adapter->cloud_filter_list, list) { 1028 cf->del = true; 1029 } 1030 spin_unlock_bh(&adapter->cloud_filter_list_lock); 1031 1032 if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) && 1033 adapter->state != __IAVF_RESETTING) { 1034 /* cancel any current operation */ 1035 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 1036 /* Schedule operations to close down the HW. Don't wait 1037 * here for this to complete. The watchdog is still running 1038 * and it will take care of this. 1039 */ 1040 adapter->aq_required = IAVF_FLAG_AQ_DEL_MAC_FILTER; 1041 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER; 1042 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; 1043 adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES; 1044 } 1045 1046 mod_timer_pending(&adapter->watchdog_timer, jiffies + 1); 1047 } 1048 1049 /** 1050 * iavf_acquire_msix_vectors - Setup the MSIX capability 1051 * @adapter: board private structure 1052 * @vectors: number of vectors to request 1053 * 1054 * Work with the OS to set up the MSIX vectors needed. 1055 * 1056 * Returns 0 on success, negative on failure 1057 **/ 1058 static int 1059 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors) 1060 { 1061 int err, vector_threshold; 1062 1063 /* We'll want at least 3 (vector_threshold): 1064 * 0) Other (Admin Queue and link, mostly) 1065 * 1) TxQ[0] Cleanup 1066 * 2) RxQ[0] Cleanup 1067 */ 1068 vector_threshold = MIN_MSIX_COUNT; 1069 1070 /* The more we get, the more we will assign to Tx/Rx Cleanup 1071 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 1072 * Right now, we simply care about how many we'll get; we'll 1073 * set them up later while requesting irq's. 1074 */ 1075 err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 1076 vector_threshold, vectors); 1077 if (err < 0) { 1078 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n"); 1079 kfree(adapter->msix_entries); 1080 adapter->msix_entries = NULL; 1081 return err; 1082 } 1083 1084 /* Adjust for only the vectors we'll use, which is minimum 1085 * of max_msix_q_vectors + NONQ_VECS, or the number of 1086 * vectors we were allocated. 1087 */ 1088 adapter->num_msix_vectors = err; 1089 return 0; 1090 } 1091 1092 /** 1093 * iavf_free_queues - Free memory for all rings 1094 * @adapter: board private structure to initialize 1095 * 1096 * Free all of the memory associated with queue pairs. 1097 **/ 1098 static void iavf_free_queues(struct iavf_adapter *adapter) 1099 { 1100 if (!adapter->vsi_res) 1101 return; 1102 adapter->num_active_queues = 0; 1103 kfree(adapter->tx_rings); 1104 adapter->tx_rings = NULL; 1105 kfree(adapter->rx_rings); 1106 adapter->rx_rings = NULL; 1107 } 1108 1109 /** 1110 * iavf_alloc_queues - Allocate memory for all rings 1111 * @adapter: board private structure to initialize 1112 * 1113 * We allocate one ring per queue at run-time since we don't know the 1114 * number of queues at compile-time. The polling_netdev array is 1115 * intended for Multiqueue, but should work fine with a single queue. 1116 **/ 1117 static int iavf_alloc_queues(struct iavf_adapter *adapter) 1118 { 1119 int i, num_active_queues; 1120 1121 /* If we're in reset reallocating queues we don't actually know yet for 1122 * certain the PF gave us the number of queues we asked for but we'll 1123 * assume it did. Once basic reset is finished we'll confirm once we 1124 * start negotiating config with PF. 1125 */ 1126 if (adapter->num_req_queues) 1127 num_active_queues = adapter->num_req_queues; 1128 else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 1129 adapter->num_tc) 1130 num_active_queues = adapter->ch_config.total_qps; 1131 else 1132 num_active_queues = min_t(int, 1133 adapter->vsi_res->num_queue_pairs, 1134 (int)(num_online_cpus())); 1135 1136 1137 adapter->tx_rings = kcalloc(num_active_queues, 1138 sizeof(struct iavf_ring), GFP_KERNEL); 1139 if (!adapter->tx_rings) 1140 goto err_out; 1141 adapter->rx_rings = kcalloc(num_active_queues, 1142 sizeof(struct iavf_ring), GFP_KERNEL); 1143 if (!adapter->rx_rings) 1144 goto err_out; 1145 1146 for (i = 0; i < num_active_queues; i++) { 1147 struct iavf_ring *tx_ring; 1148 struct iavf_ring *rx_ring; 1149 1150 tx_ring = &adapter->tx_rings[i]; 1151 1152 tx_ring->queue_index = i; 1153 tx_ring->netdev = adapter->netdev; 1154 tx_ring->dev = &adapter->pdev->dev; 1155 tx_ring->count = adapter->tx_desc_count; 1156 tx_ring->itr_setting = IAVF_ITR_TX_DEF; 1157 if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE) 1158 tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR; 1159 1160 rx_ring = &adapter->rx_rings[i]; 1161 rx_ring->queue_index = i; 1162 rx_ring->netdev = adapter->netdev; 1163 rx_ring->dev = &adapter->pdev->dev; 1164 rx_ring->count = adapter->rx_desc_count; 1165 rx_ring->itr_setting = IAVF_ITR_RX_DEF; 1166 } 1167 1168 adapter->num_active_queues = num_active_queues; 1169 1170 return 0; 1171 1172 err_out: 1173 iavf_free_queues(adapter); 1174 return -ENOMEM; 1175 } 1176 1177 /** 1178 * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported 1179 * @adapter: board private structure to initialize 1180 * 1181 * Attempt to configure the interrupts using the best available 1182 * capabilities of the hardware and the kernel. 1183 **/ 1184 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter) 1185 { 1186 int vector, v_budget; 1187 int pairs = 0; 1188 int err = 0; 1189 1190 if (!adapter->vsi_res) { 1191 err = -EIO; 1192 goto out; 1193 } 1194 pairs = adapter->num_active_queues; 1195 1196 /* It's easy to be greedy for MSI-X vectors, but it really doesn't do 1197 * us much good if we have more vectors than CPUs. However, we already 1198 * limit the total number of queues by the number of CPUs so we do not 1199 * need any further limiting here. 1200 */ 1201 v_budget = min_t(int, pairs + NONQ_VECS, 1202 (int)adapter->vf_res->max_vectors); 1203 1204 adapter->msix_entries = kcalloc(v_budget, 1205 sizeof(struct msix_entry), GFP_KERNEL); 1206 if (!adapter->msix_entries) { 1207 err = -ENOMEM; 1208 goto out; 1209 } 1210 1211 for (vector = 0; vector < v_budget; vector++) 1212 adapter->msix_entries[vector].entry = vector; 1213 1214 err = iavf_acquire_msix_vectors(adapter, v_budget); 1215 1216 out: 1217 netif_set_real_num_rx_queues(adapter->netdev, pairs); 1218 netif_set_real_num_tx_queues(adapter->netdev, pairs); 1219 return err; 1220 } 1221 1222 /** 1223 * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands 1224 * @adapter: board private structure 1225 * 1226 * Return 0 on success, negative on failure 1227 **/ 1228 static int iavf_config_rss_aq(struct iavf_adapter *adapter) 1229 { 1230 struct i40e_aqc_get_set_rss_key_data *rss_key = 1231 (struct i40e_aqc_get_set_rss_key_data *)adapter->rss_key; 1232 struct iavf_hw *hw = &adapter->hw; 1233 int ret = 0; 1234 1235 if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { 1236 /* bail because we already have a command pending */ 1237 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n", 1238 adapter->current_op); 1239 return -EBUSY; 1240 } 1241 1242 ret = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key); 1243 if (ret) { 1244 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n", 1245 iavf_stat_str(hw, ret), 1246 iavf_aq_str(hw, hw->aq.asq_last_status)); 1247 return ret; 1248 1249 } 1250 1251 ret = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false, 1252 adapter->rss_lut, adapter->rss_lut_size); 1253 if (ret) { 1254 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n", 1255 iavf_stat_str(hw, ret), 1256 iavf_aq_str(hw, hw->aq.asq_last_status)); 1257 } 1258 1259 return ret; 1260 1261 } 1262 1263 /** 1264 * iavf_config_rss_reg - Configure RSS keys and lut by writing registers 1265 * @adapter: board private structure 1266 * 1267 * Returns 0 on success, negative on failure 1268 **/ 1269 static int iavf_config_rss_reg(struct iavf_adapter *adapter) 1270 { 1271 struct iavf_hw *hw = &adapter->hw; 1272 u32 *dw; 1273 u16 i; 1274 1275 dw = (u32 *)adapter->rss_key; 1276 for (i = 0; i <= adapter->rss_key_size / 4; i++) 1277 wr32(hw, IAVF_VFQF_HKEY(i), dw[i]); 1278 1279 dw = (u32 *)adapter->rss_lut; 1280 for (i = 0; i <= adapter->rss_lut_size / 4; i++) 1281 wr32(hw, IAVF_VFQF_HLUT(i), dw[i]); 1282 1283 iavf_flush(hw); 1284 1285 return 0; 1286 } 1287 1288 /** 1289 * iavf_config_rss - Configure RSS keys and lut 1290 * @adapter: board private structure 1291 * 1292 * Returns 0 on success, negative on failure 1293 **/ 1294 int iavf_config_rss(struct iavf_adapter *adapter) 1295 { 1296 1297 if (RSS_PF(adapter)) { 1298 adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT | 1299 IAVF_FLAG_AQ_SET_RSS_KEY; 1300 return 0; 1301 } else if (RSS_AQ(adapter)) { 1302 return iavf_config_rss_aq(adapter); 1303 } else { 1304 return iavf_config_rss_reg(adapter); 1305 } 1306 } 1307 1308 /** 1309 * iavf_fill_rss_lut - Fill the lut with default values 1310 * @adapter: board private structure 1311 **/ 1312 static void iavf_fill_rss_lut(struct iavf_adapter *adapter) 1313 { 1314 u16 i; 1315 1316 for (i = 0; i < adapter->rss_lut_size; i++) 1317 adapter->rss_lut[i] = i % adapter->num_active_queues; 1318 } 1319 1320 /** 1321 * iavf_init_rss - Prepare for RSS 1322 * @adapter: board private structure 1323 * 1324 * Return 0 on success, negative on failure 1325 **/ 1326 static int iavf_init_rss(struct iavf_adapter *adapter) 1327 { 1328 struct iavf_hw *hw = &adapter->hw; 1329 int ret; 1330 1331 if (!RSS_PF(adapter)) { 1332 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */ 1333 if (adapter->vf_res->vf_cap_flags & 1334 VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2) 1335 adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED; 1336 else 1337 adapter->hena = IAVF_DEFAULT_RSS_HENA; 1338 1339 wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena); 1340 wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32)); 1341 } 1342 1343 iavf_fill_rss_lut(adapter); 1344 netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size); 1345 ret = iavf_config_rss(adapter); 1346 1347 return ret; 1348 } 1349 1350 /** 1351 * iavf_alloc_q_vectors - Allocate memory for interrupt vectors 1352 * @adapter: board private structure to initialize 1353 * 1354 * We allocate one q_vector per queue interrupt. If allocation fails we 1355 * return -ENOMEM. 1356 **/ 1357 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter) 1358 { 1359 int q_idx = 0, num_q_vectors; 1360 struct iavf_q_vector *q_vector; 1361 1362 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1363 adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector), 1364 GFP_KERNEL); 1365 if (!adapter->q_vectors) 1366 return -ENOMEM; 1367 1368 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1369 q_vector = &adapter->q_vectors[q_idx]; 1370 q_vector->adapter = adapter; 1371 q_vector->vsi = &adapter->vsi; 1372 q_vector->v_idx = q_idx; 1373 q_vector->reg_idx = q_idx; 1374 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 1375 netif_napi_add(adapter->netdev, &q_vector->napi, 1376 iavf_napi_poll, NAPI_POLL_WEIGHT); 1377 } 1378 1379 return 0; 1380 } 1381 1382 /** 1383 * iavf_free_q_vectors - Free memory allocated for interrupt vectors 1384 * @adapter: board private structure to initialize 1385 * 1386 * This function frees the memory allocated to the q_vectors. In addition if 1387 * NAPI is enabled it will delete any references to the NAPI struct prior 1388 * to freeing the q_vector. 1389 **/ 1390 static void iavf_free_q_vectors(struct iavf_adapter *adapter) 1391 { 1392 int q_idx, num_q_vectors; 1393 int napi_vectors; 1394 1395 if (!adapter->q_vectors) 1396 return; 1397 1398 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1399 napi_vectors = adapter->num_active_queues; 1400 1401 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1402 struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx]; 1403 1404 if (q_idx < napi_vectors) 1405 netif_napi_del(&q_vector->napi); 1406 } 1407 kfree(adapter->q_vectors); 1408 adapter->q_vectors = NULL; 1409 } 1410 1411 /** 1412 * iavf_reset_interrupt_capability - Reset MSIX setup 1413 * @adapter: board private structure 1414 * 1415 **/ 1416 void iavf_reset_interrupt_capability(struct iavf_adapter *adapter) 1417 { 1418 if (!adapter->msix_entries) 1419 return; 1420 1421 pci_disable_msix(adapter->pdev); 1422 kfree(adapter->msix_entries); 1423 adapter->msix_entries = NULL; 1424 } 1425 1426 /** 1427 * iavf_init_interrupt_scheme - Determine if MSIX is supported and init 1428 * @adapter: board private structure to initialize 1429 * 1430 **/ 1431 int iavf_init_interrupt_scheme(struct iavf_adapter *adapter) 1432 { 1433 int err; 1434 1435 err = iavf_alloc_queues(adapter); 1436 if (err) { 1437 dev_err(&adapter->pdev->dev, 1438 "Unable to allocate memory for queues\n"); 1439 goto err_alloc_queues; 1440 } 1441 1442 rtnl_lock(); 1443 err = iavf_set_interrupt_capability(adapter); 1444 rtnl_unlock(); 1445 if (err) { 1446 dev_err(&adapter->pdev->dev, 1447 "Unable to setup interrupt capabilities\n"); 1448 goto err_set_interrupt; 1449 } 1450 1451 err = iavf_alloc_q_vectors(adapter); 1452 if (err) { 1453 dev_err(&adapter->pdev->dev, 1454 "Unable to allocate memory for queue vectors\n"); 1455 goto err_alloc_q_vectors; 1456 } 1457 1458 /* If we've made it so far while ADq flag being ON, then we haven't 1459 * bailed out anywhere in middle. And ADq isn't just enabled but actual 1460 * resources have been allocated in the reset path. 1461 * Now we can truly claim that ADq is enabled. 1462 */ 1463 if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 1464 adapter->num_tc) 1465 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created", 1466 adapter->num_tc); 1467 1468 dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u", 1469 (adapter->num_active_queues > 1) ? "Enabled" : "Disabled", 1470 adapter->num_active_queues); 1471 1472 return 0; 1473 err_alloc_q_vectors: 1474 iavf_reset_interrupt_capability(adapter); 1475 err_set_interrupt: 1476 iavf_free_queues(adapter); 1477 err_alloc_queues: 1478 return err; 1479 } 1480 1481 /** 1482 * iavf_free_rss - Free memory used by RSS structs 1483 * @adapter: board private structure 1484 **/ 1485 static void iavf_free_rss(struct iavf_adapter *adapter) 1486 { 1487 kfree(adapter->rss_key); 1488 adapter->rss_key = NULL; 1489 1490 kfree(adapter->rss_lut); 1491 adapter->rss_lut = NULL; 1492 } 1493 1494 /** 1495 * iavf_reinit_interrupt_scheme - Reallocate queues and vectors 1496 * @adapter: board private structure 1497 * 1498 * Returns 0 on success, negative on failure 1499 **/ 1500 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter) 1501 { 1502 struct net_device *netdev = adapter->netdev; 1503 int err; 1504 1505 if (netif_running(netdev)) 1506 iavf_free_traffic_irqs(adapter); 1507 iavf_free_misc_irq(adapter); 1508 iavf_reset_interrupt_capability(adapter); 1509 iavf_free_q_vectors(adapter); 1510 iavf_free_queues(adapter); 1511 1512 err = iavf_init_interrupt_scheme(adapter); 1513 if (err) 1514 goto err; 1515 1516 netif_tx_stop_all_queues(netdev); 1517 1518 err = iavf_request_misc_irq(adapter); 1519 if (err) 1520 goto err; 1521 1522 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 1523 1524 iavf_map_rings_to_vectors(adapter); 1525 1526 if (RSS_AQ(adapter)) 1527 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS; 1528 else 1529 err = iavf_init_rss(adapter); 1530 err: 1531 return err; 1532 } 1533 1534 /** 1535 * iavf_watchdog_timer - Periodic call-back timer 1536 * @data: pointer to adapter disguised as unsigned long 1537 **/ 1538 static void iavf_watchdog_timer(struct timer_list *t) 1539 { 1540 struct iavf_adapter *adapter = from_timer(adapter, t, 1541 watchdog_timer); 1542 1543 schedule_work(&adapter->watchdog_task); 1544 /* timer will be rescheduled in watchdog task */ 1545 } 1546 1547 /** 1548 * iavf_watchdog_task - Periodic call-back task 1549 * @work: pointer to work_struct 1550 **/ 1551 static void iavf_watchdog_task(struct work_struct *work) 1552 { 1553 struct iavf_adapter *adapter = container_of(work, 1554 struct iavf_adapter, 1555 watchdog_task); 1556 struct iavf_hw *hw = &adapter->hw; 1557 u32 reg_val; 1558 1559 if (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section)) 1560 goto restart_watchdog; 1561 1562 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) { 1563 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) & 1564 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 1565 if ((reg_val == VIRTCHNL_VFR_VFACTIVE) || 1566 (reg_val == VIRTCHNL_VFR_COMPLETED)) { 1567 /* A chance for redemption! */ 1568 dev_err(&adapter->pdev->dev, "Hardware came out of reset. Attempting reinit.\n"); 1569 adapter->state = __IAVF_STARTUP; 1570 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED; 1571 schedule_delayed_work(&adapter->init_task, 10); 1572 clear_bit(__IAVF_IN_CRITICAL_TASK, 1573 &adapter->crit_section); 1574 /* Don't reschedule the watchdog, since we've restarted 1575 * the init task. When init_task contacts the PF and 1576 * gets everything set up again, it'll restart the 1577 * watchdog for us. Down, boy. Sit. Stay. Woof. 1578 */ 1579 return; 1580 } 1581 adapter->aq_required = 0; 1582 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 1583 goto watchdog_done; 1584 } 1585 1586 if ((adapter->state < __IAVF_DOWN) || 1587 (adapter->flags & IAVF_FLAG_RESET_PENDING)) 1588 goto watchdog_done; 1589 1590 /* check for reset */ 1591 reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK; 1592 if (!(adapter->flags & IAVF_FLAG_RESET_PENDING) && !reg_val) { 1593 adapter->state = __IAVF_RESETTING; 1594 adapter->flags |= IAVF_FLAG_RESET_PENDING; 1595 dev_err(&adapter->pdev->dev, "Hardware reset detected\n"); 1596 schedule_work(&adapter->reset_task); 1597 adapter->aq_required = 0; 1598 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 1599 goto watchdog_done; 1600 } 1601 1602 /* Process admin queue tasks. After init, everything gets done 1603 * here so we don't race on the admin queue. 1604 */ 1605 if (adapter->current_op) { 1606 if (!iavf_asq_done(hw)) { 1607 dev_dbg(&adapter->pdev->dev, "Admin queue timeout\n"); 1608 iavf_send_api_ver(adapter); 1609 } 1610 goto watchdog_done; 1611 } 1612 if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG) { 1613 iavf_send_vf_config_msg(adapter); 1614 goto watchdog_done; 1615 } 1616 1617 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) { 1618 iavf_disable_queues(adapter); 1619 goto watchdog_done; 1620 } 1621 1622 if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) { 1623 iavf_map_queues(adapter); 1624 goto watchdog_done; 1625 } 1626 1627 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) { 1628 iavf_add_ether_addrs(adapter); 1629 goto watchdog_done; 1630 } 1631 1632 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) { 1633 iavf_add_vlans(adapter); 1634 goto watchdog_done; 1635 } 1636 1637 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) { 1638 iavf_del_ether_addrs(adapter); 1639 goto watchdog_done; 1640 } 1641 1642 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) { 1643 iavf_del_vlans(adapter); 1644 goto watchdog_done; 1645 } 1646 1647 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) { 1648 iavf_enable_vlan_stripping(adapter); 1649 goto watchdog_done; 1650 } 1651 1652 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) { 1653 iavf_disable_vlan_stripping(adapter); 1654 goto watchdog_done; 1655 } 1656 1657 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) { 1658 iavf_configure_queues(adapter); 1659 goto watchdog_done; 1660 } 1661 1662 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) { 1663 iavf_enable_queues(adapter); 1664 goto watchdog_done; 1665 } 1666 1667 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) { 1668 /* This message goes straight to the firmware, not the 1669 * PF, so we don't have to set current_op as we will 1670 * not get a response through the ARQ. 1671 */ 1672 iavf_init_rss(adapter); 1673 adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS; 1674 goto watchdog_done; 1675 } 1676 if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) { 1677 iavf_get_hena(adapter); 1678 goto watchdog_done; 1679 } 1680 if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) { 1681 iavf_set_hena(adapter); 1682 goto watchdog_done; 1683 } 1684 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) { 1685 iavf_set_rss_key(adapter); 1686 goto watchdog_done; 1687 } 1688 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) { 1689 iavf_set_rss_lut(adapter); 1690 goto watchdog_done; 1691 } 1692 1693 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_PROMISC) { 1694 iavf_set_promiscuous(adapter, FLAG_VF_UNICAST_PROMISC | 1695 FLAG_VF_MULTICAST_PROMISC); 1696 goto watchdog_done; 1697 } 1698 1699 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_ALLMULTI) { 1700 iavf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC); 1701 goto watchdog_done; 1702 } 1703 1704 if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) && 1705 (adapter->aq_required & IAVF_FLAG_AQ_RELEASE_ALLMULTI)) { 1706 iavf_set_promiscuous(adapter, 0); 1707 goto watchdog_done; 1708 } 1709 1710 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) { 1711 iavf_enable_channels(adapter); 1712 goto watchdog_done; 1713 } 1714 1715 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) { 1716 iavf_disable_channels(adapter); 1717 goto watchdog_done; 1718 } 1719 1720 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) { 1721 iavf_add_cloud_filter(adapter); 1722 goto watchdog_done; 1723 } 1724 1725 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) { 1726 iavf_del_cloud_filter(adapter); 1727 goto watchdog_done; 1728 } 1729 1730 schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5)); 1731 1732 if (adapter->state == __IAVF_RUNNING) 1733 iavf_request_stats(adapter); 1734 watchdog_done: 1735 if (adapter->state == __IAVF_RUNNING) 1736 iavf_detect_recover_hung(&adapter->vsi); 1737 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 1738 restart_watchdog: 1739 if (adapter->state == __IAVF_REMOVE) 1740 return; 1741 if (adapter->aq_required) 1742 mod_timer(&adapter->watchdog_timer, 1743 jiffies + msecs_to_jiffies(20)); 1744 else 1745 mod_timer(&adapter->watchdog_timer, jiffies + (HZ * 2)); 1746 schedule_work(&adapter->adminq_task); 1747 } 1748 1749 static void iavf_disable_vf(struct iavf_adapter *adapter) 1750 { 1751 struct iavf_mac_filter *f, *ftmp; 1752 struct iavf_vlan_filter *fv, *fvtmp; 1753 struct iavf_cloud_filter *cf, *cftmp; 1754 1755 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED; 1756 1757 /* We don't use netif_running() because it may be true prior to 1758 * ndo_open() returning, so we can't assume it means all our open 1759 * tasks have finished, since we're not holding the rtnl_lock here. 1760 */ 1761 if (adapter->state == __IAVF_RUNNING) { 1762 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 1763 netif_carrier_off(adapter->netdev); 1764 netif_tx_disable(adapter->netdev); 1765 adapter->link_up = false; 1766 iavf_napi_disable_all(adapter); 1767 iavf_irq_disable(adapter); 1768 iavf_free_traffic_irqs(adapter); 1769 iavf_free_all_tx_resources(adapter); 1770 iavf_free_all_rx_resources(adapter); 1771 } 1772 1773 spin_lock_bh(&adapter->mac_vlan_list_lock); 1774 1775 /* Delete all of the filters */ 1776 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 1777 list_del(&f->list); 1778 kfree(f); 1779 } 1780 1781 list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) { 1782 list_del(&fv->list); 1783 kfree(fv); 1784 } 1785 1786 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1787 1788 spin_lock_bh(&adapter->cloud_filter_list_lock); 1789 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { 1790 list_del(&cf->list); 1791 kfree(cf); 1792 adapter->num_cloud_filters--; 1793 } 1794 spin_unlock_bh(&adapter->cloud_filter_list_lock); 1795 1796 iavf_free_misc_irq(adapter); 1797 iavf_reset_interrupt_capability(adapter); 1798 iavf_free_queues(adapter); 1799 iavf_free_q_vectors(adapter); 1800 kfree(adapter->vf_res); 1801 iavf_shutdown_adminq(&adapter->hw); 1802 adapter->netdev->flags &= ~IFF_UP; 1803 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 1804 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 1805 adapter->state = __IAVF_DOWN; 1806 wake_up(&adapter->down_waitqueue); 1807 dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n"); 1808 } 1809 1810 #define IAVF_RESET_WAIT_MS 10 1811 #define IAVF_RESET_WAIT_COUNT 500 1812 /** 1813 * iavf_reset_task - Call-back task to handle hardware reset 1814 * @work: pointer to work_struct 1815 * 1816 * During reset we need to shut down and reinitialize the admin queue 1817 * before we can use it to communicate with the PF again. We also clear 1818 * and reinit the rings because that context is lost as well. 1819 **/ 1820 static void iavf_reset_task(struct work_struct *work) 1821 { 1822 struct iavf_adapter *adapter = container_of(work, 1823 struct iavf_adapter, 1824 reset_task); 1825 struct virtchnl_vf_resource *vfres = adapter->vf_res; 1826 struct net_device *netdev = adapter->netdev; 1827 struct iavf_hw *hw = &adapter->hw; 1828 struct iavf_vlan_filter *vlf; 1829 struct iavf_cloud_filter *cf; 1830 struct iavf_mac_filter *f; 1831 u32 reg_val; 1832 int i = 0, err; 1833 bool running; 1834 1835 /* When device is being removed it doesn't make sense to run the reset 1836 * task, just return in such a case. 1837 */ 1838 if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) 1839 return; 1840 1841 while (test_and_set_bit(__IAVF_IN_CLIENT_TASK, 1842 &adapter->crit_section)) 1843 usleep_range(500, 1000); 1844 if (CLIENT_ENABLED(adapter)) { 1845 adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN | 1846 IAVF_FLAG_CLIENT_NEEDS_CLOSE | 1847 IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS | 1848 IAVF_FLAG_SERVICE_CLIENT_REQUESTED); 1849 cancel_delayed_work_sync(&adapter->client_task); 1850 iavf_notify_client_close(&adapter->vsi, true); 1851 } 1852 iavf_misc_irq_disable(adapter); 1853 if (adapter->flags & IAVF_FLAG_RESET_NEEDED) { 1854 adapter->flags &= ~IAVF_FLAG_RESET_NEEDED; 1855 /* Restart the AQ here. If we have been reset but didn't 1856 * detect it, or if the PF had to reinit, our AQ will be hosed. 1857 */ 1858 iavf_shutdown_adminq(hw); 1859 iavf_init_adminq(hw); 1860 iavf_request_reset(adapter); 1861 } 1862 adapter->flags |= IAVF_FLAG_RESET_PENDING; 1863 1864 /* poll until we see the reset actually happen */ 1865 for (i = 0; i < IAVF_RESET_WAIT_COUNT; i++) { 1866 reg_val = rd32(hw, IAVF_VF_ARQLEN1) & 1867 IAVF_VF_ARQLEN1_ARQENABLE_MASK; 1868 if (!reg_val) 1869 break; 1870 usleep_range(5000, 10000); 1871 } 1872 if (i == IAVF_RESET_WAIT_COUNT) { 1873 dev_info(&adapter->pdev->dev, "Never saw reset\n"); 1874 goto continue_reset; /* act like the reset happened */ 1875 } 1876 1877 /* wait until the reset is complete and the PF is responding to us */ 1878 for (i = 0; i < IAVF_RESET_WAIT_COUNT; i++) { 1879 /* sleep first to make sure a minimum wait time is met */ 1880 msleep(IAVF_RESET_WAIT_MS); 1881 1882 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) & 1883 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 1884 if (reg_val == VIRTCHNL_VFR_VFACTIVE) 1885 break; 1886 } 1887 1888 pci_set_master(adapter->pdev); 1889 1890 if (i == IAVF_RESET_WAIT_COUNT) { 1891 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n", 1892 reg_val); 1893 iavf_disable_vf(adapter); 1894 clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); 1895 return; /* Do not attempt to reinit. It's dead, Jim. */ 1896 } 1897 1898 continue_reset: 1899 /* We don't use netif_running() because it may be true prior to 1900 * ndo_open() returning, so we can't assume it means all our open 1901 * tasks have finished, since we're not holding the rtnl_lock here. 1902 */ 1903 running = ((adapter->state == __IAVF_RUNNING) || 1904 (adapter->state == __IAVF_RESETTING)); 1905 1906 if (running) { 1907 netif_carrier_off(netdev); 1908 netif_tx_stop_all_queues(netdev); 1909 adapter->link_up = false; 1910 iavf_napi_disable_all(adapter); 1911 } 1912 iavf_irq_disable(adapter); 1913 1914 adapter->state = __IAVF_RESETTING; 1915 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 1916 1917 /* free the Tx/Rx rings and descriptors, might be better to just 1918 * re-use them sometime in the future 1919 */ 1920 iavf_free_all_rx_resources(adapter); 1921 iavf_free_all_tx_resources(adapter); 1922 1923 adapter->flags |= IAVF_FLAG_QUEUES_DISABLED; 1924 /* kill and reinit the admin queue */ 1925 iavf_shutdown_adminq(hw); 1926 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 1927 err = iavf_init_adminq(hw); 1928 if (err) 1929 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n", 1930 err); 1931 adapter->aq_required = 0; 1932 1933 if (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED) { 1934 err = iavf_reinit_interrupt_scheme(adapter); 1935 if (err) 1936 goto reset_err; 1937 } 1938 1939 adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG; 1940 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS; 1941 1942 spin_lock_bh(&adapter->mac_vlan_list_lock); 1943 1944 /* re-add all MAC filters */ 1945 list_for_each_entry(f, &adapter->mac_filter_list, list) { 1946 f->add = true; 1947 } 1948 /* re-add all VLAN filters */ 1949 list_for_each_entry(vlf, &adapter->vlan_filter_list, list) { 1950 vlf->add = true; 1951 } 1952 1953 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1954 1955 /* check if TCs are running and re-add all cloud filters */ 1956 spin_lock_bh(&adapter->cloud_filter_list_lock); 1957 if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 1958 adapter->num_tc) { 1959 list_for_each_entry(cf, &adapter->cloud_filter_list, list) { 1960 cf->add = true; 1961 } 1962 } 1963 spin_unlock_bh(&adapter->cloud_filter_list_lock); 1964 1965 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; 1966 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER; 1967 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; 1968 iavf_misc_irq_enable(adapter); 1969 1970 mod_timer(&adapter->watchdog_timer, jiffies + 2); 1971 1972 /* We were running when the reset started, so we need to restore some 1973 * state here. 1974 */ 1975 if (running) { 1976 /* allocate transmit descriptors */ 1977 err = iavf_setup_all_tx_resources(adapter); 1978 if (err) 1979 goto reset_err; 1980 1981 /* allocate receive descriptors */ 1982 err = iavf_setup_all_rx_resources(adapter); 1983 if (err) 1984 goto reset_err; 1985 1986 if (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED) { 1987 err = iavf_request_traffic_irqs(adapter, netdev->name); 1988 if (err) 1989 goto reset_err; 1990 1991 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; 1992 } 1993 1994 iavf_configure(adapter); 1995 1996 iavf_up_complete(adapter); 1997 1998 iavf_irq_enable(adapter, true); 1999 } else { 2000 adapter->state = __IAVF_DOWN; 2001 wake_up(&adapter->down_waitqueue); 2002 } 2003 clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); 2004 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 2005 2006 return; 2007 reset_err: 2008 clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); 2009 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 2010 dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n"); 2011 iavf_close(netdev); 2012 } 2013 2014 /** 2015 * iavf_adminq_task - worker thread to clean the admin queue 2016 * @work: pointer to work_struct containing our data 2017 **/ 2018 static void iavf_adminq_task(struct work_struct *work) 2019 { 2020 struct iavf_adapter *adapter = 2021 container_of(work, struct iavf_adapter, adminq_task); 2022 struct iavf_hw *hw = &adapter->hw; 2023 struct i40e_arq_event_info event; 2024 enum virtchnl_ops v_op; 2025 iavf_status ret, v_ret; 2026 u32 val, oldval; 2027 u16 pending; 2028 2029 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) 2030 goto out; 2031 2032 event.buf_len = IAVF_MAX_AQ_BUF_SIZE; 2033 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 2034 if (!event.msg_buf) 2035 goto out; 2036 2037 do { 2038 ret = iavf_clean_arq_element(hw, &event, &pending); 2039 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high); 2040 v_ret = (iavf_status)le32_to_cpu(event.desc.cookie_low); 2041 2042 if (ret || !v_op) 2043 break; /* No event to process or error cleaning ARQ */ 2044 2045 iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf, 2046 event.msg_len); 2047 if (pending != 0) 2048 memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE); 2049 } while (pending); 2050 2051 if ((adapter->flags & 2052 (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED)) || 2053 adapter->state == __IAVF_RESETTING) 2054 goto freedom; 2055 2056 /* check for error indications */ 2057 val = rd32(hw, hw->aq.arq.len); 2058 if (val == 0xdeadbeef) /* indicates device in reset */ 2059 goto freedom; 2060 oldval = val; 2061 if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) { 2062 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n"); 2063 val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK; 2064 } 2065 if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) { 2066 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n"); 2067 val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK; 2068 } 2069 if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) { 2070 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n"); 2071 val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK; 2072 } 2073 if (oldval != val) 2074 wr32(hw, hw->aq.arq.len, val); 2075 2076 val = rd32(hw, hw->aq.asq.len); 2077 oldval = val; 2078 if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) { 2079 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n"); 2080 val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK; 2081 } 2082 if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) { 2083 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n"); 2084 val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK; 2085 } 2086 if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) { 2087 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n"); 2088 val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK; 2089 } 2090 if (oldval != val) 2091 wr32(hw, hw->aq.asq.len, val); 2092 2093 freedom: 2094 kfree(event.msg_buf); 2095 out: 2096 /* re-enable Admin queue interrupt cause */ 2097 iavf_misc_irq_enable(adapter); 2098 } 2099 2100 /** 2101 * iavf_client_task - worker thread to perform client work 2102 * @work: pointer to work_struct containing our data 2103 * 2104 * This task handles client interactions. Because client calls can be 2105 * reentrant, we can't handle them in the watchdog. 2106 **/ 2107 static void iavf_client_task(struct work_struct *work) 2108 { 2109 struct iavf_adapter *adapter = 2110 container_of(work, struct iavf_adapter, client_task.work); 2111 2112 /* If we can't get the client bit, just give up. We'll be rescheduled 2113 * later. 2114 */ 2115 2116 if (test_and_set_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section)) 2117 return; 2118 2119 if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) { 2120 iavf_client_subtask(adapter); 2121 adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED; 2122 goto out; 2123 } 2124 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) { 2125 iavf_notify_client_l2_params(&adapter->vsi); 2126 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS; 2127 goto out; 2128 } 2129 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) { 2130 iavf_notify_client_close(&adapter->vsi, false); 2131 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE; 2132 goto out; 2133 } 2134 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) { 2135 iavf_notify_client_open(&adapter->vsi); 2136 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN; 2137 } 2138 out: 2139 clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); 2140 } 2141 2142 /** 2143 * iavf_free_all_tx_resources - Free Tx Resources for All Queues 2144 * @adapter: board private structure 2145 * 2146 * Free all transmit software resources 2147 **/ 2148 void iavf_free_all_tx_resources(struct iavf_adapter *adapter) 2149 { 2150 int i; 2151 2152 if (!adapter->tx_rings) 2153 return; 2154 2155 for (i = 0; i < adapter->num_active_queues; i++) 2156 if (adapter->tx_rings[i].desc) 2157 iavf_free_tx_resources(&adapter->tx_rings[i]); 2158 } 2159 2160 /** 2161 * iavf_setup_all_tx_resources - allocate all queues Tx resources 2162 * @adapter: board private structure 2163 * 2164 * If this function returns with an error, then it's possible one or 2165 * more of the rings is populated (while the rest are not). It is the 2166 * callers duty to clean those orphaned rings. 2167 * 2168 * Return 0 on success, negative on failure 2169 **/ 2170 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter) 2171 { 2172 int i, err = 0; 2173 2174 for (i = 0; i < adapter->num_active_queues; i++) { 2175 adapter->tx_rings[i].count = adapter->tx_desc_count; 2176 err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]); 2177 if (!err) 2178 continue; 2179 dev_err(&adapter->pdev->dev, 2180 "Allocation for Tx Queue %u failed\n", i); 2181 break; 2182 } 2183 2184 return err; 2185 } 2186 2187 /** 2188 * iavf_setup_all_rx_resources - allocate all queues Rx resources 2189 * @adapter: board private structure 2190 * 2191 * If this function returns with an error, then it's possible one or 2192 * more of the rings is populated (while the rest are not). It is the 2193 * callers duty to clean those orphaned rings. 2194 * 2195 * Return 0 on success, negative on failure 2196 **/ 2197 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter) 2198 { 2199 int i, err = 0; 2200 2201 for (i = 0; i < adapter->num_active_queues; i++) { 2202 adapter->rx_rings[i].count = adapter->rx_desc_count; 2203 err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]); 2204 if (!err) 2205 continue; 2206 dev_err(&adapter->pdev->dev, 2207 "Allocation for Rx Queue %u failed\n", i); 2208 break; 2209 } 2210 return err; 2211 } 2212 2213 /** 2214 * iavf_free_all_rx_resources - Free Rx Resources for All Queues 2215 * @adapter: board private structure 2216 * 2217 * Free all receive software resources 2218 **/ 2219 void iavf_free_all_rx_resources(struct iavf_adapter *adapter) 2220 { 2221 int i; 2222 2223 if (!adapter->rx_rings) 2224 return; 2225 2226 for (i = 0; i < adapter->num_active_queues; i++) 2227 if (adapter->rx_rings[i].desc) 2228 iavf_free_rx_resources(&adapter->rx_rings[i]); 2229 } 2230 2231 /** 2232 * iavf_validate_tx_bandwidth - validate the max Tx bandwidth 2233 * @adapter: board private structure 2234 * @max_tx_rate: max Tx bw for a tc 2235 **/ 2236 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter, 2237 u64 max_tx_rate) 2238 { 2239 int speed = 0, ret = 0; 2240 2241 switch (adapter->link_speed) { 2242 case I40E_LINK_SPEED_40GB: 2243 speed = 40000; 2244 break; 2245 case I40E_LINK_SPEED_25GB: 2246 speed = 25000; 2247 break; 2248 case I40E_LINK_SPEED_20GB: 2249 speed = 20000; 2250 break; 2251 case I40E_LINK_SPEED_10GB: 2252 speed = 10000; 2253 break; 2254 case I40E_LINK_SPEED_1GB: 2255 speed = 1000; 2256 break; 2257 case I40E_LINK_SPEED_100MB: 2258 speed = 100; 2259 break; 2260 default: 2261 break; 2262 } 2263 2264 if (max_tx_rate > speed) { 2265 dev_err(&adapter->pdev->dev, 2266 "Invalid tx rate specified\n"); 2267 ret = -EINVAL; 2268 } 2269 2270 return ret; 2271 } 2272 2273 /** 2274 * iavf_validate_channel_config - validate queue mapping info 2275 * @adapter: board private structure 2276 * @mqprio_qopt: queue parameters 2277 * 2278 * This function validates if the config provided by the user to 2279 * configure queue channels is valid or not. Returns 0 on a valid 2280 * config. 2281 **/ 2282 static int iavf_validate_ch_config(struct iavf_adapter *adapter, 2283 struct tc_mqprio_qopt_offload *mqprio_qopt) 2284 { 2285 u64 total_max_rate = 0; 2286 int i, num_qps = 0; 2287 u64 tx_rate = 0; 2288 int ret = 0; 2289 2290 if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS || 2291 mqprio_qopt->qopt.num_tc < 1) 2292 return -EINVAL; 2293 2294 for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) { 2295 if (!mqprio_qopt->qopt.count[i] || 2296 mqprio_qopt->qopt.offset[i] != num_qps) 2297 return -EINVAL; 2298 if (mqprio_qopt->min_rate[i]) { 2299 dev_err(&adapter->pdev->dev, 2300 "Invalid min tx rate (greater than 0) specified\n"); 2301 return -EINVAL; 2302 } 2303 /*convert to Mbps */ 2304 tx_rate = div_u64(mqprio_qopt->max_rate[i], 2305 IAVF_MBPS_DIVISOR); 2306 total_max_rate += tx_rate; 2307 num_qps += mqprio_qopt->qopt.count[i]; 2308 } 2309 if (num_qps > IAVF_MAX_REQ_QUEUES) 2310 return -EINVAL; 2311 2312 ret = iavf_validate_tx_bandwidth(adapter, total_max_rate); 2313 return ret; 2314 } 2315 2316 /** 2317 * iavf_del_all_cloud_filters - delete all cloud filters 2318 * on the traffic classes 2319 **/ 2320 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter) 2321 { 2322 struct iavf_cloud_filter *cf, *cftmp; 2323 2324 spin_lock_bh(&adapter->cloud_filter_list_lock); 2325 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, 2326 list) { 2327 list_del(&cf->list); 2328 kfree(cf); 2329 adapter->num_cloud_filters--; 2330 } 2331 spin_unlock_bh(&adapter->cloud_filter_list_lock); 2332 } 2333 2334 /** 2335 * __iavf_setup_tc - configure multiple traffic classes 2336 * @netdev: network interface device structure 2337 * @type_date: tc offload data 2338 * 2339 * This function processes the config information provided by the 2340 * user to configure traffic classes/queue channels and packages the 2341 * information to request the PF to setup traffic classes. 2342 * 2343 * Returns 0 on success. 2344 **/ 2345 static int __iavf_setup_tc(struct net_device *netdev, void *type_data) 2346 { 2347 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 2348 struct iavf_adapter *adapter = netdev_priv(netdev); 2349 struct virtchnl_vf_resource *vfres = adapter->vf_res; 2350 u8 num_tc = 0, total_qps = 0; 2351 int ret = 0, netdev_tc = 0; 2352 u64 max_tx_rate; 2353 u16 mode; 2354 int i; 2355 2356 num_tc = mqprio_qopt->qopt.num_tc; 2357 mode = mqprio_qopt->mode; 2358 2359 /* delete queue_channel */ 2360 if (!mqprio_qopt->qopt.hw) { 2361 if (adapter->ch_config.state == __IAVF_TC_RUNNING) { 2362 /* reset the tc configuration */ 2363 netdev_reset_tc(netdev); 2364 adapter->num_tc = 0; 2365 netif_tx_stop_all_queues(netdev); 2366 netif_tx_disable(netdev); 2367 iavf_del_all_cloud_filters(adapter); 2368 adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS; 2369 goto exit; 2370 } else { 2371 return -EINVAL; 2372 } 2373 } 2374 2375 /* add queue channel */ 2376 if (mode == TC_MQPRIO_MODE_CHANNEL) { 2377 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) { 2378 dev_err(&adapter->pdev->dev, "ADq not supported\n"); 2379 return -EOPNOTSUPP; 2380 } 2381 if (adapter->ch_config.state != __IAVF_TC_INVALID) { 2382 dev_err(&adapter->pdev->dev, "TC configuration already exists\n"); 2383 return -EINVAL; 2384 } 2385 2386 ret = iavf_validate_ch_config(adapter, mqprio_qopt); 2387 if (ret) 2388 return ret; 2389 /* Return if same TC config is requested */ 2390 if (adapter->num_tc == num_tc) 2391 return 0; 2392 adapter->num_tc = num_tc; 2393 2394 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) { 2395 if (i < num_tc) { 2396 adapter->ch_config.ch_info[i].count = 2397 mqprio_qopt->qopt.count[i]; 2398 adapter->ch_config.ch_info[i].offset = 2399 mqprio_qopt->qopt.offset[i]; 2400 total_qps += mqprio_qopt->qopt.count[i]; 2401 max_tx_rate = mqprio_qopt->max_rate[i]; 2402 /* convert to Mbps */ 2403 max_tx_rate = div_u64(max_tx_rate, 2404 IAVF_MBPS_DIVISOR); 2405 adapter->ch_config.ch_info[i].max_tx_rate = 2406 max_tx_rate; 2407 } else { 2408 adapter->ch_config.ch_info[i].count = 1; 2409 adapter->ch_config.ch_info[i].offset = 0; 2410 } 2411 } 2412 adapter->ch_config.total_qps = total_qps; 2413 netif_tx_stop_all_queues(netdev); 2414 netif_tx_disable(netdev); 2415 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS; 2416 netdev_reset_tc(netdev); 2417 /* Report the tc mapping up the stack */ 2418 netdev_set_num_tc(adapter->netdev, num_tc); 2419 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) { 2420 u16 qcount = mqprio_qopt->qopt.count[i]; 2421 u16 qoffset = mqprio_qopt->qopt.offset[i]; 2422 2423 if (i < num_tc) 2424 netdev_set_tc_queue(netdev, netdev_tc++, qcount, 2425 qoffset); 2426 } 2427 } 2428 exit: 2429 return ret; 2430 } 2431 2432 /** 2433 * iavf_parse_cls_flower - Parse tc flower filters provided by kernel 2434 * @adapter: board private structure 2435 * @cls_flower: pointer to struct tc_cls_flower_offload 2436 * @filter: pointer to cloud filter structure 2437 */ 2438 static int iavf_parse_cls_flower(struct iavf_adapter *adapter, 2439 struct tc_cls_flower_offload *f, 2440 struct iavf_cloud_filter *filter) 2441 { 2442 struct flow_rule *rule = tc_cls_flower_offload_flow_rule(f); 2443 struct flow_dissector *dissector = rule->match.dissector; 2444 u16 n_proto_mask = 0; 2445 u16 n_proto_key = 0; 2446 u8 field_flags = 0; 2447 u16 addr_type = 0; 2448 u16 n_proto = 0; 2449 int i = 0; 2450 struct virtchnl_filter *vf = &filter->f; 2451 2452 if (dissector->used_keys & 2453 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 2454 BIT(FLOW_DISSECTOR_KEY_BASIC) | 2455 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 2456 BIT(FLOW_DISSECTOR_KEY_VLAN) | 2457 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 2458 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 2459 BIT(FLOW_DISSECTOR_KEY_PORTS) | 2460 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 2461 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%x\n", 2462 dissector->used_keys); 2463 return -EOPNOTSUPP; 2464 } 2465 2466 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 2467 struct flow_match_enc_keyid match; 2468 2469 flow_rule_match_enc_keyid(rule, &match); 2470 if (match.mask->keyid != 0) 2471 field_flags |= IAVF_CLOUD_FIELD_TEN_ID; 2472 } 2473 2474 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 2475 struct flow_match_basic match; 2476 2477 flow_rule_match_basic(rule, &match); 2478 n_proto_key = ntohs(match.key->n_proto); 2479 n_proto_mask = ntohs(match.mask->n_proto); 2480 2481 if (n_proto_key == ETH_P_ALL) { 2482 n_proto_key = 0; 2483 n_proto_mask = 0; 2484 } 2485 n_proto = n_proto_key & n_proto_mask; 2486 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6) 2487 return -EINVAL; 2488 if (n_proto == ETH_P_IPV6) { 2489 /* specify flow type as TCP IPv6 */ 2490 vf->flow_type = VIRTCHNL_TCP_V6_FLOW; 2491 } 2492 2493 if (match.key->ip_proto != IPPROTO_TCP) { 2494 dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n"); 2495 return -EINVAL; 2496 } 2497 } 2498 2499 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 2500 struct flow_match_eth_addrs match; 2501 2502 flow_rule_match_eth_addrs(rule, &match); 2503 2504 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 2505 if (!is_zero_ether_addr(match.mask->dst)) { 2506 if (is_broadcast_ether_addr(match.mask->dst)) { 2507 field_flags |= IAVF_CLOUD_FIELD_OMAC; 2508 } else { 2509 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n", 2510 match.mask->dst); 2511 return I40E_ERR_CONFIG; 2512 } 2513 } 2514 2515 if (!is_zero_ether_addr(match.mask->src)) { 2516 if (is_broadcast_ether_addr(match.mask->src)) { 2517 field_flags |= IAVF_CLOUD_FIELD_IMAC; 2518 } else { 2519 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n", 2520 match.mask->src); 2521 return I40E_ERR_CONFIG; 2522 } 2523 } 2524 2525 if (!is_zero_ether_addr(match.key->dst)) 2526 if (is_valid_ether_addr(match.key->dst) || 2527 is_multicast_ether_addr(match.key->dst)) { 2528 /* set the mask if a valid dst_mac address */ 2529 for (i = 0; i < ETH_ALEN; i++) 2530 vf->mask.tcp_spec.dst_mac[i] |= 0xff; 2531 ether_addr_copy(vf->data.tcp_spec.dst_mac, 2532 match.key->dst); 2533 } 2534 2535 if (!is_zero_ether_addr(match.key->src)) 2536 if (is_valid_ether_addr(match.key->src) || 2537 is_multicast_ether_addr(match.key->src)) { 2538 /* set the mask if a valid dst_mac address */ 2539 for (i = 0; i < ETH_ALEN; i++) 2540 vf->mask.tcp_spec.src_mac[i] |= 0xff; 2541 ether_addr_copy(vf->data.tcp_spec.src_mac, 2542 match.key->src); 2543 } 2544 } 2545 2546 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 2547 struct flow_match_vlan match; 2548 2549 flow_rule_match_vlan(rule, &match); 2550 if (match.mask->vlan_id) { 2551 if (match.mask->vlan_id == VLAN_VID_MASK) { 2552 field_flags |= IAVF_CLOUD_FIELD_IVLAN; 2553 } else { 2554 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n", 2555 match.mask->vlan_id); 2556 return I40E_ERR_CONFIG; 2557 } 2558 } 2559 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff); 2560 vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id); 2561 } 2562 2563 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 2564 struct flow_match_control match; 2565 2566 flow_rule_match_control(rule, &match); 2567 addr_type = match.key->addr_type; 2568 } 2569 2570 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 2571 struct flow_match_ipv4_addrs match; 2572 2573 flow_rule_match_ipv4_addrs(rule, &match); 2574 if (match.mask->dst) { 2575 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 2576 field_flags |= IAVF_CLOUD_FIELD_IIP; 2577 } else { 2578 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n", 2579 be32_to_cpu(match.mask->dst)); 2580 return I40E_ERR_CONFIG; 2581 } 2582 } 2583 2584 if (match.mask->src) { 2585 if (match.mask->src == cpu_to_be32(0xffffffff)) { 2586 field_flags |= IAVF_CLOUD_FIELD_IIP; 2587 } else { 2588 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n", 2589 be32_to_cpu(match.mask->dst)); 2590 return I40E_ERR_CONFIG; 2591 } 2592 } 2593 2594 if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) { 2595 dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n"); 2596 return I40E_ERR_CONFIG; 2597 } 2598 if (match.key->dst) { 2599 vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff); 2600 vf->data.tcp_spec.dst_ip[0] = match.key->dst; 2601 } 2602 if (match.key->src) { 2603 vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff); 2604 vf->data.tcp_spec.src_ip[0] = match.key->src; 2605 } 2606 } 2607 2608 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 2609 struct flow_match_ipv6_addrs match; 2610 2611 flow_rule_match_ipv6_addrs(rule, &match); 2612 2613 /* validate mask, make sure it is not IPV6_ADDR_ANY */ 2614 if (ipv6_addr_any(&match.mask->dst)) { 2615 dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n", 2616 IPV6_ADDR_ANY); 2617 return I40E_ERR_CONFIG; 2618 } 2619 2620 /* src and dest IPv6 address should not be LOOPBACK 2621 * (0:0:0:0:0:0:0:1) which can be represented as ::1 2622 */ 2623 if (ipv6_addr_loopback(&match.key->dst) || 2624 ipv6_addr_loopback(&match.key->src)) { 2625 dev_err(&adapter->pdev->dev, 2626 "ipv6 addr should not be loopback\n"); 2627 return I40E_ERR_CONFIG; 2628 } 2629 if (!ipv6_addr_any(&match.mask->dst) || 2630 !ipv6_addr_any(&match.mask->src)) 2631 field_flags |= IAVF_CLOUD_FIELD_IIP; 2632 2633 for (i = 0; i < 4; i++) 2634 vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff); 2635 memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32, 2636 sizeof(vf->data.tcp_spec.dst_ip)); 2637 for (i = 0; i < 4; i++) 2638 vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff); 2639 memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32, 2640 sizeof(vf->data.tcp_spec.src_ip)); 2641 } 2642 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 2643 struct flow_match_ports match; 2644 2645 flow_rule_match_ports(rule, &match); 2646 if (match.mask->src) { 2647 if (match.mask->src == cpu_to_be16(0xffff)) { 2648 field_flags |= IAVF_CLOUD_FIELD_IIP; 2649 } else { 2650 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n", 2651 be16_to_cpu(match.mask->src)); 2652 return I40E_ERR_CONFIG; 2653 } 2654 } 2655 2656 if (match.mask->dst) { 2657 if (match.mask->dst == cpu_to_be16(0xffff)) { 2658 field_flags |= IAVF_CLOUD_FIELD_IIP; 2659 } else { 2660 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n", 2661 be16_to_cpu(match.mask->dst)); 2662 return I40E_ERR_CONFIG; 2663 } 2664 } 2665 if (match.key->dst) { 2666 vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff); 2667 vf->data.tcp_spec.dst_port = match.key->dst; 2668 } 2669 2670 if (match.key->src) { 2671 vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff); 2672 vf->data.tcp_spec.src_port = match.key->src; 2673 } 2674 } 2675 vf->field_flags = field_flags; 2676 2677 return 0; 2678 } 2679 2680 /** 2681 * iavf_handle_tclass - Forward to a traffic class on the device 2682 * @adapter: board private structure 2683 * @tc: traffic class index on the device 2684 * @filter: pointer to cloud filter structure 2685 */ 2686 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc, 2687 struct iavf_cloud_filter *filter) 2688 { 2689 if (tc == 0) 2690 return 0; 2691 if (tc < adapter->num_tc) { 2692 if (!filter->f.data.tcp_spec.dst_port) { 2693 dev_err(&adapter->pdev->dev, 2694 "Specify destination port to redirect to traffic class other than TC0\n"); 2695 return -EINVAL; 2696 } 2697 } 2698 /* redirect to a traffic class on the same device */ 2699 filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT; 2700 filter->f.action_meta = tc; 2701 return 0; 2702 } 2703 2704 /** 2705 * iavf_configure_clsflower - Add tc flower filters 2706 * @adapter: board private structure 2707 * @cls_flower: Pointer to struct tc_cls_flower_offload 2708 */ 2709 static int iavf_configure_clsflower(struct iavf_adapter *adapter, 2710 struct tc_cls_flower_offload *cls_flower) 2711 { 2712 int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid); 2713 struct iavf_cloud_filter *filter = NULL; 2714 int err = -EINVAL, count = 50; 2715 2716 if (tc < 0) { 2717 dev_err(&adapter->pdev->dev, "Invalid traffic class\n"); 2718 return -EINVAL; 2719 } 2720 2721 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 2722 if (!filter) 2723 return -ENOMEM; 2724 2725 while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, 2726 &adapter->crit_section)) { 2727 if (--count == 0) 2728 goto err; 2729 udelay(1); 2730 } 2731 2732 filter->cookie = cls_flower->cookie; 2733 2734 /* set the mask to all zeroes to begin with */ 2735 memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec)); 2736 /* start out with flow type and eth type IPv4 to begin with */ 2737 filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW; 2738 err = iavf_parse_cls_flower(adapter, cls_flower, filter); 2739 if (err < 0) 2740 goto err; 2741 2742 err = iavf_handle_tclass(adapter, tc, filter); 2743 if (err < 0) 2744 goto err; 2745 2746 /* add filter to the list */ 2747 spin_lock_bh(&adapter->cloud_filter_list_lock); 2748 list_add_tail(&filter->list, &adapter->cloud_filter_list); 2749 adapter->num_cloud_filters++; 2750 filter->add = true; 2751 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; 2752 spin_unlock_bh(&adapter->cloud_filter_list_lock); 2753 err: 2754 if (err) 2755 kfree(filter); 2756 2757 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 2758 return err; 2759 } 2760 2761 /* iavf_find_cf - Find the cloud filter in the list 2762 * @adapter: Board private structure 2763 * @cookie: filter specific cookie 2764 * 2765 * Returns ptr to the filter object or NULL. Must be called while holding the 2766 * cloud_filter_list_lock. 2767 */ 2768 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter, 2769 unsigned long *cookie) 2770 { 2771 struct iavf_cloud_filter *filter = NULL; 2772 2773 if (!cookie) 2774 return NULL; 2775 2776 list_for_each_entry(filter, &adapter->cloud_filter_list, list) { 2777 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 2778 return filter; 2779 } 2780 return NULL; 2781 } 2782 2783 /** 2784 * iavf_delete_clsflower - Remove tc flower filters 2785 * @adapter: board private structure 2786 * @cls_flower: Pointer to struct tc_cls_flower_offload 2787 */ 2788 static int iavf_delete_clsflower(struct iavf_adapter *adapter, 2789 struct tc_cls_flower_offload *cls_flower) 2790 { 2791 struct iavf_cloud_filter *filter = NULL; 2792 int err = 0; 2793 2794 spin_lock_bh(&adapter->cloud_filter_list_lock); 2795 filter = iavf_find_cf(adapter, &cls_flower->cookie); 2796 if (filter) { 2797 filter->del = true; 2798 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; 2799 } else { 2800 err = -EINVAL; 2801 } 2802 spin_unlock_bh(&adapter->cloud_filter_list_lock); 2803 2804 return err; 2805 } 2806 2807 /** 2808 * iavf_setup_tc_cls_flower - flower classifier offloads 2809 * @netdev: net device to configure 2810 * @type_data: offload data 2811 */ 2812 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter, 2813 struct tc_cls_flower_offload *cls_flower) 2814 { 2815 if (cls_flower->common.chain_index) 2816 return -EOPNOTSUPP; 2817 2818 switch (cls_flower->command) { 2819 case TC_CLSFLOWER_REPLACE: 2820 return iavf_configure_clsflower(adapter, cls_flower); 2821 case TC_CLSFLOWER_DESTROY: 2822 return iavf_delete_clsflower(adapter, cls_flower); 2823 case TC_CLSFLOWER_STATS: 2824 return -EOPNOTSUPP; 2825 default: 2826 return -EOPNOTSUPP; 2827 } 2828 } 2829 2830 /** 2831 * iavf_setup_tc_block_cb - block callback for tc 2832 * @type: type of offload 2833 * @type_data: offload data 2834 * @cb_priv: 2835 * 2836 * This function is the block callback for traffic classes 2837 **/ 2838 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 2839 void *cb_priv) 2840 { 2841 switch (type) { 2842 case TC_SETUP_CLSFLOWER: 2843 return iavf_setup_tc_cls_flower(cb_priv, type_data); 2844 default: 2845 return -EOPNOTSUPP; 2846 } 2847 } 2848 2849 /** 2850 * iavf_setup_tc_block - register callbacks for tc 2851 * @netdev: network interface device structure 2852 * @f: tc offload data 2853 * 2854 * This function registers block callbacks for tc 2855 * offloads 2856 **/ 2857 static int iavf_setup_tc_block(struct net_device *dev, 2858 struct tc_block_offload *f) 2859 { 2860 struct iavf_adapter *adapter = netdev_priv(dev); 2861 2862 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 2863 return -EOPNOTSUPP; 2864 2865 switch (f->command) { 2866 case TC_BLOCK_BIND: 2867 return tcf_block_cb_register(f->block, iavf_setup_tc_block_cb, 2868 adapter, adapter, f->extack); 2869 case TC_BLOCK_UNBIND: 2870 tcf_block_cb_unregister(f->block, iavf_setup_tc_block_cb, 2871 adapter); 2872 return 0; 2873 default: 2874 return -EOPNOTSUPP; 2875 } 2876 } 2877 2878 /** 2879 * iavf_setup_tc - configure multiple traffic classes 2880 * @netdev: network interface device structure 2881 * @type: type of offload 2882 * @type_date: tc offload data 2883 * 2884 * This function is the callback to ndo_setup_tc in the 2885 * netdev_ops. 2886 * 2887 * Returns 0 on success 2888 **/ 2889 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type, 2890 void *type_data) 2891 { 2892 switch (type) { 2893 case TC_SETUP_QDISC_MQPRIO: 2894 return __iavf_setup_tc(netdev, type_data); 2895 case TC_SETUP_BLOCK: 2896 return iavf_setup_tc_block(netdev, type_data); 2897 default: 2898 return -EOPNOTSUPP; 2899 } 2900 } 2901 2902 /** 2903 * iavf_open - Called when a network interface is made active 2904 * @netdev: network interface device structure 2905 * 2906 * Returns 0 on success, negative value on failure 2907 * 2908 * The open entry point is called when a network interface is made 2909 * active by the system (IFF_UP). At this point all resources needed 2910 * for transmit and receive operations are allocated, the interrupt 2911 * handler is registered with the OS, the watchdog timer is started, 2912 * and the stack is notified that the interface is ready. 2913 **/ 2914 static int iavf_open(struct net_device *netdev) 2915 { 2916 struct iavf_adapter *adapter = netdev_priv(netdev); 2917 int err; 2918 2919 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) { 2920 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n"); 2921 return -EIO; 2922 } 2923 2924 while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, 2925 &adapter->crit_section)) 2926 usleep_range(500, 1000); 2927 2928 if (adapter->state != __IAVF_DOWN) { 2929 err = -EBUSY; 2930 goto err_unlock; 2931 } 2932 2933 /* allocate transmit descriptors */ 2934 err = iavf_setup_all_tx_resources(adapter); 2935 if (err) 2936 goto err_setup_tx; 2937 2938 /* allocate receive descriptors */ 2939 err = iavf_setup_all_rx_resources(adapter); 2940 if (err) 2941 goto err_setup_rx; 2942 2943 /* clear any pending interrupts, may auto mask */ 2944 err = iavf_request_traffic_irqs(adapter, netdev->name); 2945 if (err) 2946 goto err_req_irq; 2947 2948 spin_lock_bh(&adapter->mac_vlan_list_lock); 2949 2950 iavf_add_filter(adapter, adapter->hw.mac.addr); 2951 2952 spin_unlock_bh(&adapter->mac_vlan_list_lock); 2953 2954 iavf_configure(adapter); 2955 2956 iavf_up_complete(adapter); 2957 2958 iavf_irq_enable(adapter, true); 2959 2960 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 2961 2962 return 0; 2963 2964 err_req_irq: 2965 iavf_down(adapter); 2966 iavf_free_traffic_irqs(adapter); 2967 err_setup_rx: 2968 iavf_free_all_rx_resources(adapter); 2969 err_setup_tx: 2970 iavf_free_all_tx_resources(adapter); 2971 err_unlock: 2972 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 2973 2974 return err; 2975 } 2976 2977 /** 2978 * iavf_close - Disables a network interface 2979 * @netdev: network interface device structure 2980 * 2981 * Returns 0, this is not allowed to fail 2982 * 2983 * The close entry point is called when an interface is de-activated 2984 * by the OS. The hardware is still under the drivers control, but 2985 * needs to be disabled. All IRQs except vector 0 (reserved for admin queue) 2986 * are freed, along with all transmit and receive resources. 2987 **/ 2988 static int iavf_close(struct net_device *netdev) 2989 { 2990 struct iavf_adapter *adapter = netdev_priv(netdev); 2991 int status; 2992 2993 if (adapter->state <= __IAVF_DOWN_PENDING) 2994 return 0; 2995 2996 while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, 2997 &adapter->crit_section)) 2998 usleep_range(500, 1000); 2999 3000 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 3001 if (CLIENT_ENABLED(adapter)) 3002 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE; 3003 3004 iavf_down(adapter); 3005 adapter->state = __IAVF_DOWN_PENDING; 3006 iavf_free_traffic_irqs(adapter); 3007 3008 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 3009 3010 /* We explicitly don't free resources here because the hardware is 3011 * still active and can DMA into memory. Resources are cleared in 3012 * iavf_virtchnl_completion() after we get confirmation from the PF 3013 * driver that the rings have been stopped. 3014 * 3015 * Also, we wait for state to transition to __IAVF_DOWN before 3016 * returning. State change occurs in iavf_virtchnl_completion() after 3017 * VF resources are released (which occurs after PF driver processes and 3018 * responds to admin queue commands). 3019 */ 3020 3021 status = wait_event_timeout(adapter->down_waitqueue, 3022 adapter->state == __IAVF_DOWN, 3023 msecs_to_jiffies(200)); 3024 if (!status) 3025 netdev_warn(netdev, "Device resources not yet released\n"); 3026 return 0; 3027 } 3028 3029 /** 3030 * iavf_change_mtu - Change the Maximum Transfer Unit 3031 * @netdev: network interface device structure 3032 * @new_mtu: new value for maximum frame size 3033 * 3034 * Returns 0 on success, negative on failure 3035 **/ 3036 static int iavf_change_mtu(struct net_device *netdev, int new_mtu) 3037 { 3038 struct iavf_adapter *adapter = netdev_priv(netdev); 3039 3040 netdev->mtu = new_mtu; 3041 if (CLIENT_ENABLED(adapter)) { 3042 iavf_notify_client_l2_params(&adapter->vsi); 3043 adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED; 3044 } 3045 adapter->flags |= IAVF_FLAG_RESET_NEEDED; 3046 schedule_work(&adapter->reset_task); 3047 3048 return 0; 3049 } 3050 3051 /** 3052 * iavf_set_features - set the netdev feature flags 3053 * @netdev: ptr to the netdev being adjusted 3054 * @features: the feature set that the stack is suggesting 3055 * Note: expects to be called while under rtnl_lock() 3056 **/ 3057 static int iavf_set_features(struct net_device *netdev, 3058 netdev_features_t features) 3059 { 3060 struct iavf_adapter *adapter = netdev_priv(netdev); 3061 3062 /* Don't allow changing VLAN_RX flag when adapter is not capable 3063 * of VLAN offload 3064 */ 3065 if (!VLAN_ALLOWED(adapter)) { 3066 if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX) 3067 return -EINVAL; 3068 } else if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX) { 3069 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3070 adapter->aq_required |= 3071 IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING; 3072 else 3073 adapter->aq_required |= 3074 IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING; 3075 } 3076 3077 return 0; 3078 } 3079 3080 /** 3081 * iavf_features_check - Validate encapsulated packet conforms to limits 3082 * @skb: skb buff 3083 * @dev: This physical port's netdev 3084 * @features: Offload features that the stack believes apply 3085 **/ 3086 static netdev_features_t iavf_features_check(struct sk_buff *skb, 3087 struct net_device *dev, 3088 netdev_features_t features) 3089 { 3090 size_t len; 3091 3092 /* No point in doing any of this if neither checksum nor GSO are 3093 * being requested for this frame. We can rule out both by just 3094 * checking for CHECKSUM_PARTIAL 3095 */ 3096 if (skb->ip_summed != CHECKSUM_PARTIAL) 3097 return features; 3098 3099 /* We cannot support GSO if the MSS is going to be less than 3100 * 64 bytes. If it is then we need to drop support for GSO. 3101 */ 3102 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 3103 features &= ~NETIF_F_GSO_MASK; 3104 3105 /* MACLEN can support at most 63 words */ 3106 len = skb_network_header(skb) - skb->data; 3107 if (len & ~(63 * 2)) 3108 goto out_err; 3109 3110 /* IPLEN and EIPLEN can support at most 127 dwords */ 3111 len = skb_transport_header(skb) - skb_network_header(skb); 3112 if (len & ~(127 * 4)) 3113 goto out_err; 3114 3115 if (skb->encapsulation) { 3116 /* L4TUNLEN can support 127 words */ 3117 len = skb_inner_network_header(skb) - skb_transport_header(skb); 3118 if (len & ~(127 * 2)) 3119 goto out_err; 3120 3121 /* IPLEN can support at most 127 dwords */ 3122 len = skb_inner_transport_header(skb) - 3123 skb_inner_network_header(skb); 3124 if (len & ~(127 * 4)) 3125 goto out_err; 3126 } 3127 3128 /* No need to validate L4LEN as TCP is the only protocol with a 3129 * a flexible value and we support all possible values supported 3130 * by TCP, which is at most 15 dwords 3131 */ 3132 3133 return features; 3134 out_err: 3135 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 3136 } 3137 3138 /** 3139 * iavf_fix_features - fix up the netdev feature bits 3140 * @netdev: our net device 3141 * @features: desired feature bits 3142 * 3143 * Returns fixed-up features bits 3144 **/ 3145 static netdev_features_t iavf_fix_features(struct net_device *netdev, 3146 netdev_features_t features) 3147 { 3148 struct iavf_adapter *adapter = netdev_priv(netdev); 3149 3150 if (!(adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 3151 features &= ~(NETIF_F_HW_VLAN_CTAG_TX | 3152 NETIF_F_HW_VLAN_CTAG_RX | 3153 NETIF_F_HW_VLAN_CTAG_FILTER); 3154 3155 return features; 3156 } 3157 3158 static const struct net_device_ops iavf_netdev_ops = { 3159 .ndo_open = iavf_open, 3160 .ndo_stop = iavf_close, 3161 .ndo_start_xmit = iavf_xmit_frame, 3162 .ndo_set_rx_mode = iavf_set_rx_mode, 3163 .ndo_validate_addr = eth_validate_addr, 3164 .ndo_set_mac_address = iavf_set_mac, 3165 .ndo_change_mtu = iavf_change_mtu, 3166 .ndo_tx_timeout = iavf_tx_timeout, 3167 .ndo_vlan_rx_add_vid = iavf_vlan_rx_add_vid, 3168 .ndo_vlan_rx_kill_vid = iavf_vlan_rx_kill_vid, 3169 .ndo_features_check = iavf_features_check, 3170 .ndo_fix_features = iavf_fix_features, 3171 .ndo_set_features = iavf_set_features, 3172 .ndo_setup_tc = iavf_setup_tc, 3173 }; 3174 3175 /** 3176 * iavf_check_reset_complete - check that VF reset is complete 3177 * @hw: pointer to hw struct 3178 * 3179 * Returns 0 if device is ready to use, or -EBUSY if it's in reset. 3180 **/ 3181 static int iavf_check_reset_complete(struct iavf_hw *hw) 3182 { 3183 u32 rstat; 3184 int i; 3185 3186 for (i = 0; i < 100; i++) { 3187 rstat = rd32(hw, IAVF_VFGEN_RSTAT) & 3188 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 3189 if ((rstat == VIRTCHNL_VFR_VFACTIVE) || 3190 (rstat == VIRTCHNL_VFR_COMPLETED)) 3191 return 0; 3192 usleep_range(10, 20); 3193 } 3194 return -EBUSY; 3195 } 3196 3197 /** 3198 * iavf_process_config - Process the config information we got from the PF 3199 * @adapter: board private structure 3200 * 3201 * Verify that we have a valid config struct, and set up our netdev features 3202 * and our VSI struct. 3203 **/ 3204 int iavf_process_config(struct iavf_adapter *adapter) 3205 { 3206 struct virtchnl_vf_resource *vfres = adapter->vf_res; 3207 int i, num_req_queues = adapter->num_req_queues; 3208 struct net_device *netdev = adapter->netdev; 3209 struct iavf_vsi *vsi = &adapter->vsi; 3210 netdev_features_t hw_enc_features; 3211 netdev_features_t hw_features; 3212 3213 /* got VF config message back from PF, now we can parse it */ 3214 for (i = 0; i < vfres->num_vsis; i++) { 3215 if (vfres->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) 3216 adapter->vsi_res = &vfres->vsi_res[i]; 3217 } 3218 if (!adapter->vsi_res) { 3219 dev_err(&adapter->pdev->dev, "No LAN VSI found\n"); 3220 return -ENODEV; 3221 } 3222 3223 if (num_req_queues && 3224 num_req_queues != adapter->vsi_res->num_queue_pairs) { 3225 /* Problem. The PF gave us fewer queues than what we had 3226 * negotiated in our request. Need a reset to see if we can't 3227 * get back to a working state. 3228 */ 3229 dev_err(&adapter->pdev->dev, 3230 "Requested %d queues, but PF only gave us %d.\n", 3231 num_req_queues, 3232 adapter->vsi_res->num_queue_pairs); 3233 adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED; 3234 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs; 3235 iavf_schedule_reset(adapter); 3236 return -ENODEV; 3237 } 3238 adapter->num_req_queues = 0; 3239 3240 hw_enc_features = NETIF_F_SG | 3241 NETIF_F_IP_CSUM | 3242 NETIF_F_IPV6_CSUM | 3243 NETIF_F_HIGHDMA | 3244 NETIF_F_SOFT_FEATURES | 3245 NETIF_F_TSO | 3246 NETIF_F_TSO_ECN | 3247 NETIF_F_TSO6 | 3248 NETIF_F_SCTP_CRC | 3249 NETIF_F_RXHASH | 3250 NETIF_F_RXCSUM | 3251 0; 3252 3253 /* advertise to stack only if offloads for encapsulated packets is 3254 * supported 3255 */ 3256 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) { 3257 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 3258 NETIF_F_GSO_GRE | 3259 NETIF_F_GSO_GRE_CSUM | 3260 NETIF_F_GSO_IPXIP4 | 3261 NETIF_F_GSO_IPXIP6 | 3262 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3263 NETIF_F_GSO_PARTIAL | 3264 0; 3265 3266 if (!(vfres->vf_cap_flags & 3267 VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM)) 3268 netdev->gso_partial_features |= 3269 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3270 3271 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 3272 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 3273 netdev->hw_enc_features |= hw_enc_features; 3274 } 3275 /* record features VLANs can make use of */ 3276 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 3277 3278 /* Write features and hw_features separately to avoid polluting 3279 * with, or dropping, features that are set when we registered. 3280 */ 3281 hw_features = hw_enc_features; 3282 3283 /* Enable VLAN features if supported */ 3284 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) 3285 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX | 3286 NETIF_F_HW_VLAN_CTAG_RX); 3287 /* Enable cloud filter if ADQ is supported */ 3288 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) 3289 hw_features |= NETIF_F_HW_TC; 3290 3291 netdev->hw_features |= hw_features; 3292 3293 netdev->features |= hw_features; 3294 3295 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) 3296 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 3297 3298 netdev->priv_flags |= IFF_UNICAST_FLT; 3299 3300 /* Do not turn on offloads when they are requested to be turned off. 3301 * TSO needs minimum 576 bytes to work correctly. 3302 */ 3303 if (netdev->wanted_features) { 3304 if (!(netdev->wanted_features & NETIF_F_TSO) || 3305 netdev->mtu < 576) 3306 netdev->features &= ~NETIF_F_TSO; 3307 if (!(netdev->wanted_features & NETIF_F_TSO6) || 3308 netdev->mtu < 576) 3309 netdev->features &= ~NETIF_F_TSO6; 3310 if (!(netdev->wanted_features & NETIF_F_TSO_ECN)) 3311 netdev->features &= ~NETIF_F_TSO_ECN; 3312 if (!(netdev->wanted_features & NETIF_F_GRO)) 3313 netdev->features &= ~NETIF_F_GRO; 3314 if (!(netdev->wanted_features & NETIF_F_GSO)) 3315 netdev->features &= ~NETIF_F_GSO; 3316 } 3317 3318 adapter->vsi.id = adapter->vsi_res->vsi_id; 3319 3320 adapter->vsi.back = adapter; 3321 adapter->vsi.base_vector = 1; 3322 adapter->vsi.work_limit = IAVF_DEFAULT_IRQ_WORK; 3323 vsi->netdev = adapter->netdev; 3324 vsi->qs_handle = adapter->vsi_res->qset_handle; 3325 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 3326 adapter->rss_key_size = vfres->rss_key_size; 3327 adapter->rss_lut_size = vfres->rss_lut_size; 3328 } else { 3329 adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE; 3330 adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE; 3331 } 3332 3333 return 0; 3334 } 3335 3336 /** 3337 * iavf_init_task - worker thread to perform delayed initialization 3338 * @work: pointer to work_struct containing our data 3339 * 3340 * This task completes the work that was begun in probe. Due to the nature 3341 * of VF-PF communications, we may need to wait tens of milliseconds to get 3342 * responses back from the PF. Rather than busy-wait in probe and bog down the 3343 * whole system, we'll do it in a task so we can sleep. 3344 * This task only runs during driver init. Once we've established 3345 * communications with the PF driver and set up our netdev, the watchdog 3346 * takes over. 3347 **/ 3348 static void iavf_init_task(struct work_struct *work) 3349 { 3350 struct iavf_adapter *adapter = container_of(work, 3351 struct iavf_adapter, 3352 init_task.work); 3353 struct net_device *netdev = adapter->netdev; 3354 struct iavf_hw *hw = &adapter->hw; 3355 struct pci_dev *pdev = adapter->pdev; 3356 int err, bufsz; 3357 3358 switch (adapter->state) { 3359 case __IAVF_STARTUP: 3360 /* driver loaded, probe complete */ 3361 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED; 3362 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 3363 err = iavf_set_mac_type(hw); 3364 if (err) { 3365 dev_err(&pdev->dev, "Failed to set MAC type (%d)\n", 3366 err); 3367 goto err; 3368 } 3369 err = iavf_check_reset_complete(hw); 3370 if (err) { 3371 dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n", 3372 err); 3373 goto err; 3374 } 3375 hw->aq.num_arq_entries = IAVF_AQ_LEN; 3376 hw->aq.num_asq_entries = IAVF_AQ_LEN; 3377 hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE; 3378 hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE; 3379 3380 err = iavf_init_adminq(hw); 3381 if (err) { 3382 dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n", 3383 err); 3384 goto err; 3385 } 3386 err = iavf_send_api_ver(adapter); 3387 if (err) { 3388 dev_err(&pdev->dev, "Unable to send to PF (%d)\n", err); 3389 iavf_shutdown_adminq(hw); 3390 goto err; 3391 } 3392 adapter->state = __IAVF_INIT_VERSION_CHECK; 3393 goto restart; 3394 case __IAVF_INIT_VERSION_CHECK: 3395 if (!iavf_asq_done(hw)) { 3396 dev_err(&pdev->dev, "Admin queue command never completed\n"); 3397 iavf_shutdown_adminq(hw); 3398 adapter->state = __IAVF_STARTUP; 3399 goto err; 3400 } 3401 3402 /* aq msg sent, awaiting reply */ 3403 err = iavf_verify_api_ver(adapter); 3404 if (err) { 3405 if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK) 3406 err = iavf_send_api_ver(adapter); 3407 else 3408 dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n", 3409 adapter->pf_version.major, 3410 adapter->pf_version.minor, 3411 VIRTCHNL_VERSION_MAJOR, 3412 VIRTCHNL_VERSION_MINOR); 3413 goto err; 3414 } 3415 err = iavf_send_vf_config_msg(adapter); 3416 if (err) { 3417 dev_err(&pdev->dev, "Unable to send config request (%d)\n", 3418 err); 3419 goto err; 3420 } 3421 adapter->state = __IAVF_INIT_GET_RESOURCES; 3422 goto restart; 3423 case __IAVF_INIT_GET_RESOURCES: 3424 /* aq msg sent, awaiting reply */ 3425 if (!adapter->vf_res) { 3426 bufsz = sizeof(struct virtchnl_vf_resource) + 3427 (IAVF_MAX_VF_VSI * 3428 sizeof(struct virtchnl_vsi_resource)); 3429 adapter->vf_res = kzalloc(bufsz, GFP_KERNEL); 3430 if (!adapter->vf_res) 3431 goto err; 3432 } 3433 err = iavf_get_vf_config(adapter); 3434 if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK) { 3435 err = iavf_send_vf_config_msg(adapter); 3436 goto err; 3437 } else if (err == I40E_ERR_PARAM) { 3438 /* We only get ERR_PARAM if the device is in a very bad 3439 * state or if we've been disabled for previous bad 3440 * behavior. Either way, we're done now. 3441 */ 3442 iavf_shutdown_adminq(hw); 3443 dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n"); 3444 return; 3445 } 3446 if (err) { 3447 dev_err(&pdev->dev, "Unable to get VF config (%d)\n", 3448 err); 3449 goto err_alloc; 3450 } 3451 adapter->state = __IAVF_INIT_SW; 3452 break; 3453 default: 3454 goto err_alloc; 3455 } 3456 3457 if (iavf_process_config(adapter)) 3458 goto err_alloc; 3459 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 3460 3461 adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED; 3462 3463 netdev->netdev_ops = &iavf_netdev_ops; 3464 iavf_set_ethtool_ops(netdev); 3465 netdev->watchdog_timeo = 5 * HZ; 3466 3467 /* MTU range: 68 - 9710 */ 3468 netdev->min_mtu = ETH_MIN_MTU; 3469 netdev->max_mtu = IAVF_MAX_RXBUFFER - IAVF_PACKET_HDR_PAD; 3470 3471 if (!is_valid_ether_addr(adapter->hw.mac.addr)) { 3472 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n", 3473 adapter->hw.mac.addr); 3474 eth_hw_addr_random(netdev); 3475 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr); 3476 } else { 3477 adapter->flags |= IAVF_FLAG_ADDR_SET_BY_PF; 3478 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr); 3479 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr); 3480 } 3481 3482 timer_setup(&adapter->watchdog_timer, iavf_watchdog_timer, 0); 3483 mod_timer(&adapter->watchdog_timer, jiffies + 1); 3484 3485 adapter->tx_desc_count = IAVF_DEFAULT_TXD; 3486 adapter->rx_desc_count = IAVF_DEFAULT_RXD; 3487 err = iavf_init_interrupt_scheme(adapter); 3488 if (err) 3489 goto err_sw_init; 3490 iavf_map_rings_to_vectors(adapter); 3491 if (adapter->vf_res->vf_cap_flags & 3492 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) 3493 adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE; 3494 3495 err = iavf_request_misc_irq(adapter); 3496 if (err) 3497 goto err_sw_init; 3498 3499 netif_carrier_off(netdev); 3500 adapter->link_up = false; 3501 3502 if (!adapter->netdev_registered) { 3503 err = register_netdev(netdev); 3504 if (err) 3505 goto err_register; 3506 } 3507 3508 adapter->netdev_registered = true; 3509 3510 netif_tx_stop_all_queues(netdev); 3511 if (CLIENT_ALLOWED(adapter)) { 3512 err = iavf_lan_add_device(adapter); 3513 if (err) 3514 dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n", 3515 err); 3516 } 3517 3518 dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr); 3519 if (netdev->features & NETIF_F_GRO) 3520 dev_info(&pdev->dev, "GRO is enabled\n"); 3521 3522 adapter->state = __IAVF_DOWN; 3523 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 3524 iavf_misc_irq_enable(adapter); 3525 wake_up(&adapter->down_waitqueue); 3526 3527 adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL); 3528 adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL); 3529 if (!adapter->rss_key || !adapter->rss_lut) 3530 goto err_mem; 3531 3532 if (RSS_AQ(adapter)) { 3533 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS; 3534 mod_timer_pending(&adapter->watchdog_timer, jiffies + 1); 3535 } else { 3536 iavf_init_rss(adapter); 3537 } 3538 return; 3539 restart: 3540 schedule_delayed_work(&adapter->init_task, msecs_to_jiffies(30)); 3541 return; 3542 err_mem: 3543 iavf_free_rss(adapter); 3544 err_register: 3545 iavf_free_misc_irq(adapter); 3546 err_sw_init: 3547 iavf_reset_interrupt_capability(adapter); 3548 err_alloc: 3549 kfree(adapter->vf_res); 3550 adapter->vf_res = NULL; 3551 err: 3552 /* Things went into the weeds, so try again later */ 3553 if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) { 3554 dev_err(&pdev->dev, "Failed to communicate with PF; waiting before retry\n"); 3555 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED; 3556 iavf_shutdown_adminq(hw); 3557 adapter->state = __IAVF_STARTUP; 3558 schedule_delayed_work(&adapter->init_task, HZ * 5); 3559 return; 3560 } 3561 schedule_delayed_work(&adapter->init_task, HZ); 3562 } 3563 3564 /** 3565 * iavf_shutdown - Shutdown the device in preparation for a reboot 3566 * @pdev: pci device structure 3567 **/ 3568 static void iavf_shutdown(struct pci_dev *pdev) 3569 { 3570 struct net_device *netdev = pci_get_drvdata(pdev); 3571 struct iavf_adapter *adapter = netdev_priv(netdev); 3572 3573 netif_device_detach(netdev); 3574 3575 if (netif_running(netdev)) 3576 iavf_close(netdev); 3577 3578 /* Prevent the watchdog from running. */ 3579 adapter->state = __IAVF_REMOVE; 3580 adapter->aq_required = 0; 3581 3582 #ifdef CONFIG_PM 3583 pci_save_state(pdev); 3584 3585 #endif 3586 pci_disable_device(pdev); 3587 } 3588 3589 /** 3590 * iavf_probe - Device Initialization Routine 3591 * @pdev: PCI device information struct 3592 * @ent: entry in iavf_pci_tbl 3593 * 3594 * Returns 0 on success, negative on failure 3595 * 3596 * iavf_probe initializes an adapter identified by a pci_dev structure. 3597 * The OS initialization, configuring of the adapter private structure, 3598 * and a hardware reset occur. 3599 **/ 3600 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3601 { 3602 struct net_device *netdev; 3603 struct iavf_adapter *adapter = NULL; 3604 struct iavf_hw *hw = NULL; 3605 int err; 3606 3607 err = pci_enable_device(pdev); 3608 if (err) 3609 return err; 3610 3611 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3612 if (err) { 3613 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 3614 if (err) { 3615 dev_err(&pdev->dev, 3616 "DMA configuration failed: 0x%x\n", err); 3617 goto err_dma; 3618 } 3619 } 3620 3621 err = pci_request_regions(pdev, iavf_driver_name); 3622 if (err) { 3623 dev_err(&pdev->dev, 3624 "pci_request_regions failed 0x%x\n", err); 3625 goto err_pci_reg; 3626 } 3627 3628 pci_enable_pcie_error_reporting(pdev); 3629 3630 pci_set_master(pdev); 3631 3632 netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter), 3633 IAVF_MAX_REQ_QUEUES); 3634 if (!netdev) { 3635 err = -ENOMEM; 3636 goto err_alloc_etherdev; 3637 } 3638 3639 SET_NETDEV_DEV(netdev, &pdev->dev); 3640 3641 pci_set_drvdata(pdev, netdev); 3642 adapter = netdev_priv(netdev); 3643 3644 adapter->netdev = netdev; 3645 adapter->pdev = pdev; 3646 3647 hw = &adapter->hw; 3648 hw->back = adapter; 3649 3650 adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1; 3651 adapter->state = __IAVF_STARTUP; 3652 3653 /* Call save state here because it relies on the adapter struct. */ 3654 pci_save_state(pdev); 3655 3656 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3657 pci_resource_len(pdev, 0)); 3658 if (!hw->hw_addr) { 3659 err = -EIO; 3660 goto err_ioremap; 3661 } 3662 hw->vendor_id = pdev->vendor; 3663 hw->device_id = pdev->device; 3664 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 3665 hw->subsystem_vendor_id = pdev->subsystem_vendor; 3666 hw->subsystem_device_id = pdev->subsystem_device; 3667 hw->bus.device = PCI_SLOT(pdev->devfn); 3668 hw->bus.func = PCI_FUNC(pdev->devfn); 3669 hw->bus.bus_id = pdev->bus->number; 3670 3671 /* set up the locks for the AQ, do this only once in probe 3672 * and destroy them only once in remove 3673 */ 3674 mutex_init(&hw->aq.asq_mutex); 3675 mutex_init(&hw->aq.arq_mutex); 3676 3677 spin_lock_init(&adapter->mac_vlan_list_lock); 3678 spin_lock_init(&adapter->cloud_filter_list_lock); 3679 3680 INIT_LIST_HEAD(&adapter->mac_filter_list); 3681 INIT_LIST_HEAD(&adapter->vlan_filter_list); 3682 INIT_LIST_HEAD(&adapter->cloud_filter_list); 3683 3684 INIT_WORK(&adapter->reset_task, iavf_reset_task); 3685 INIT_WORK(&adapter->adminq_task, iavf_adminq_task); 3686 INIT_WORK(&adapter->watchdog_task, iavf_watchdog_task); 3687 INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task); 3688 INIT_DELAYED_WORK(&adapter->init_task, iavf_init_task); 3689 schedule_delayed_work(&adapter->init_task, 3690 msecs_to_jiffies(5 * (pdev->devfn & 0x07))); 3691 3692 /* Setup the wait queue for indicating transition to down status */ 3693 init_waitqueue_head(&adapter->down_waitqueue); 3694 3695 return 0; 3696 3697 err_ioremap: 3698 free_netdev(netdev); 3699 err_alloc_etherdev: 3700 pci_release_regions(pdev); 3701 err_pci_reg: 3702 err_dma: 3703 pci_disable_device(pdev); 3704 return err; 3705 } 3706 3707 #ifdef CONFIG_PM 3708 /** 3709 * iavf_suspend - Power management suspend routine 3710 * @pdev: PCI device information struct 3711 * @state: unused 3712 * 3713 * Called when the system (VM) is entering sleep/suspend. 3714 **/ 3715 static int iavf_suspend(struct pci_dev *pdev, pm_message_t state) 3716 { 3717 struct net_device *netdev = pci_get_drvdata(pdev); 3718 struct iavf_adapter *adapter = netdev_priv(netdev); 3719 int retval = 0; 3720 3721 netif_device_detach(netdev); 3722 3723 while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, 3724 &adapter->crit_section)) 3725 usleep_range(500, 1000); 3726 3727 if (netif_running(netdev)) { 3728 rtnl_lock(); 3729 iavf_down(adapter); 3730 rtnl_unlock(); 3731 } 3732 iavf_free_misc_irq(adapter); 3733 iavf_reset_interrupt_capability(adapter); 3734 3735 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 3736 3737 retval = pci_save_state(pdev); 3738 if (retval) 3739 return retval; 3740 3741 pci_disable_device(pdev); 3742 3743 return 0; 3744 } 3745 3746 /** 3747 * iavf_resume - Power management resume routine 3748 * @pdev: PCI device information struct 3749 * 3750 * Called when the system (VM) is resumed from sleep/suspend. 3751 **/ 3752 static int iavf_resume(struct pci_dev *pdev) 3753 { 3754 struct iavf_adapter *adapter = pci_get_drvdata(pdev); 3755 struct net_device *netdev = adapter->netdev; 3756 u32 err; 3757 3758 pci_set_power_state(pdev, PCI_D0); 3759 pci_restore_state(pdev); 3760 /* pci_restore_state clears dev->state_saved so call 3761 * pci_save_state to restore it. 3762 */ 3763 pci_save_state(pdev); 3764 3765 err = pci_enable_device_mem(pdev); 3766 if (err) { 3767 dev_err(&pdev->dev, "Cannot enable PCI device from suspend.\n"); 3768 return err; 3769 } 3770 pci_set_master(pdev); 3771 3772 rtnl_lock(); 3773 err = iavf_set_interrupt_capability(adapter); 3774 if (err) { 3775 rtnl_unlock(); 3776 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n"); 3777 return err; 3778 } 3779 err = iavf_request_misc_irq(adapter); 3780 rtnl_unlock(); 3781 if (err) { 3782 dev_err(&pdev->dev, "Cannot get interrupt vector.\n"); 3783 return err; 3784 } 3785 3786 schedule_work(&adapter->reset_task); 3787 3788 netif_device_attach(netdev); 3789 3790 return err; 3791 } 3792 3793 #endif /* CONFIG_PM */ 3794 /** 3795 * iavf_remove - Device Removal Routine 3796 * @pdev: PCI device information struct 3797 * 3798 * iavf_remove is called by the PCI subsystem to alert the driver 3799 * that it should release a PCI device. The could be caused by a 3800 * Hot-Plug event, or because the driver is going to be removed from 3801 * memory. 3802 **/ 3803 static void iavf_remove(struct pci_dev *pdev) 3804 { 3805 struct net_device *netdev = pci_get_drvdata(pdev); 3806 struct iavf_adapter *adapter = netdev_priv(netdev); 3807 struct iavf_vlan_filter *vlf, *vlftmp; 3808 struct iavf_mac_filter *f, *ftmp; 3809 struct iavf_cloud_filter *cf, *cftmp; 3810 struct iavf_hw *hw = &adapter->hw; 3811 int err; 3812 /* Indicate we are in remove and not to run reset_task */ 3813 set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section); 3814 cancel_delayed_work_sync(&adapter->init_task); 3815 cancel_work_sync(&adapter->reset_task); 3816 cancel_delayed_work_sync(&adapter->client_task); 3817 if (adapter->netdev_registered) { 3818 unregister_netdev(netdev); 3819 adapter->netdev_registered = false; 3820 } 3821 if (CLIENT_ALLOWED(adapter)) { 3822 err = iavf_lan_del_device(adapter); 3823 if (err) 3824 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 3825 err); 3826 } 3827 3828 /* Shut down all the garbage mashers on the detention level */ 3829 adapter->state = __IAVF_REMOVE; 3830 adapter->aq_required = 0; 3831 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; 3832 iavf_request_reset(adapter); 3833 msleep(50); 3834 /* If the FW isn't responding, kick it once, but only once. */ 3835 if (!iavf_asq_done(hw)) { 3836 iavf_request_reset(adapter); 3837 msleep(50); 3838 } 3839 iavf_free_all_tx_resources(adapter); 3840 iavf_free_all_rx_resources(adapter); 3841 iavf_misc_irq_disable(adapter); 3842 iavf_free_misc_irq(adapter); 3843 iavf_reset_interrupt_capability(adapter); 3844 iavf_free_q_vectors(adapter); 3845 3846 if (adapter->watchdog_timer.function) 3847 del_timer_sync(&adapter->watchdog_timer); 3848 3849 cancel_work_sync(&adapter->adminq_task); 3850 3851 iavf_free_rss(adapter); 3852 3853 if (hw->aq.asq.count) 3854 iavf_shutdown_adminq(hw); 3855 3856 /* destroy the locks only once, here */ 3857 mutex_destroy(&hw->aq.arq_mutex); 3858 mutex_destroy(&hw->aq.asq_mutex); 3859 3860 iounmap(hw->hw_addr); 3861 pci_release_regions(pdev); 3862 iavf_free_all_tx_resources(adapter); 3863 iavf_free_all_rx_resources(adapter); 3864 iavf_free_queues(adapter); 3865 kfree(adapter->vf_res); 3866 spin_lock_bh(&adapter->mac_vlan_list_lock); 3867 /* If we got removed before an up/down sequence, we've got a filter 3868 * hanging out there that we need to get rid of. 3869 */ 3870 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 3871 list_del(&f->list); 3872 kfree(f); 3873 } 3874 list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list, 3875 list) { 3876 list_del(&vlf->list); 3877 kfree(vlf); 3878 } 3879 3880 spin_unlock_bh(&adapter->mac_vlan_list_lock); 3881 3882 spin_lock_bh(&adapter->cloud_filter_list_lock); 3883 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { 3884 list_del(&cf->list); 3885 kfree(cf); 3886 } 3887 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3888 3889 free_netdev(netdev); 3890 3891 pci_disable_pcie_error_reporting(pdev); 3892 3893 pci_disable_device(pdev); 3894 } 3895 3896 static struct pci_driver iavf_driver = { 3897 .name = iavf_driver_name, 3898 .id_table = iavf_pci_tbl, 3899 .probe = iavf_probe, 3900 .remove = iavf_remove, 3901 #ifdef CONFIG_PM 3902 .suspend = iavf_suspend, 3903 .resume = iavf_resume, 3904 #endif 3905 .shutdown = iavf_shutdown, 3906 }; 3907 3908 /** 3909 * iavf_init_module - Driver Registration Routine 3910 * 3911 * iavf_init_module is the first routine called when the driver is 3912 * loaded. All it does is register with the PCI subsystem. 3913 **/ 3914 static int __init iavf_init_module(void) 3915 { 3916 int ret; 3917 3918 pr_info("iavf: %s - version %s\n", iavf_driver_string, 3919 iavf_driver_version); 3920 3921 pr_info("%s\n", iavf_copyright); 3922 3923 iavf_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1, 3924 iavf_driver_name); 3925 if (!iavf_wq) { 3926 pr_err("%s: Failed to create workqueue\n", iavf_driver_name); 3927 return -ENOMEM; 3928 } 3929 ret = pci_register_driver(&iavf_driver); 3930 return ret; 3931 } 3932 3933 module_init(iavf_init_module); 3934 3935 /** 3936 * iavf_exit_module - Driver Exit Cleanup Routine 3937 * 3938 * iavf_exit_module is called just before the driver is removed 3939 * from memory. 3940 **/ 3941 static void __exit iavf_exit_module(void) 3942 { 3943 pci_unregister_driver(&iavf_driver); 3944 destroy_workqueue(iavf_wq); 3945 } 3946 3947 module_exit(iavf_exit_module); 3948 3949 /* iavf_main.c */ 3950