1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2018 Intel Corporation. */ 3 4 #include "iavf.h" 5 #include "iavf_prototype.h" 6 #include "iavf_client.h" 7 /* All iavf tracepoints are defined by the include below, which must 8 * be included exactly once across the whole kernel with 9 * CREATE_TRACE_POINTS defined 10 */ 11 #define CREATE_TRACE_POINTS 12 #include "iavf_trace.h" 13 14 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter); 15 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter); 16 static int iavf_close(struct net_device *netdev); 17 static int iavf_init_get_resources(struct iavf_adapter *adapter); 18 static int iavf_check_reset_complete(struct iavf_hw *hw); 19 20 char iavf_driver_name[] = "iavf"; 21 static const char iavf_driver_string[] = 22 "Intel(R) Ethernet Adaptive Virtual Function Network Driver"; 23 24 static const char iavf_copyright[] = 25 "Copyright (c) 2013 - 2018 Intel Corporation."; 26 27 /* iavf_pci_tbl - PCI Device ID Table 28 * 29 * Wildcard entries (PCI_ANY_ID) should come last 30 * Last entry must be all 0s 31 * 32 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 33 * Class, Class Mask, private data (not used) } 34 */ 35 static const struct pci_device_id iavf_pci_tbl[] = { 36 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0}, 37 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0}, 38 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0}, 39 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0}, 40 /* required last entry */ 41 {0, } 42 }; 43 44 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl); 45 46 MODULE_ALIAS("i40evf"); 47 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 48 MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver"); 49 MODULE_LICENSE("GPL v2"); 50 51 static const struct net_device_ops iavf_netdev_ops; 52 struct workqueue_struct *iavf_wq; 53 54 /** 55 * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code 56 * @hw: pointer to the HW structure 57 * @mem: ptr to mem struct to fill out 58 * @size: size of memory requested 59 * @alignment: what to align the allocation to 60 **/ 61 enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw, 62 struct iavf_dma_mem *mem, 63 u64 size, u32 alignment) 64 { 65 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back; 66 67 if (!mem) 68 return IAVF_ERR_PARAM; 69 70 mem->size = ALIGN(size, alignment); 71 mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size, 72 (dma_addr_t *)&mem->pa, GFP_KERNEL); 73 if (mem->va) 74 return 0; 75 else 76 return IAVF_ERR_NO_MEMORY; 77 } 78 79 /** 80 * iavf_free_dma_mem_d - OS specific memory free for shared code 81 * @hw: pointer to the HW structure 82 * @mem: ptr to mem struct to free 83 **/ 84 enum iavf_status iavf_free_dma_mem_d(struct iavf_hw *hw, 85 struct iavf_dma_mem *mem) 86 { 87 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back; 88 89 if (!mem || !mem->va) 90 return IAVF_ERR_PARAM; 91 dma_free_coherent(&adapter->pdev->dev, mem->size, 92 mem->va, (dma_addr_t)mem->pa); 93 return 0; 94 } 95 96 /** 97 * iavf_allocate_virt_mem_d - OS specific memory alloc for shared code 98 * @hw: pointer to the HW structure 99 * @mem: ptr to mem struct to fill out 100 * @size: size of memory requested 101 **/ 102 enum iavf_status iavf_allocate_virt_mem_d(struct iavf_hw *hw, 103 struct iavf_virt_mem *mem, u32 size) 104 { 105 if (!mem) 106 return IAVF_ERR_PARAM; 107 108 mem->size = size; 109 mem->va = kzalloc(size, GFP_KERNEL); 110 111 if (mem->va) 112 return 0; 113 else 114 return IAVF_ERR_NO_MEMORY; 115 } 116 117 /** 118 * iavf_free_virt_mem_d - OS specific memory free for shared code 119 * @hw: pointer to the HW structure 120 * @mem: ptr to mem struct to free 121 **/ 122 enum iavf_status iavf_free_virt_mem_d(struct iavf_hw *hw, 123 struct iavf_virt_mem *mem) 124 { 125 if (!mem) 126 return IAVF_ERR_PARAM; 127 128 /* it's ok to kfree a NULL pointer */ 129 kfree(mem->va); 130 131 return 0; 132 } 133 134 /** 135 * iavf_schedule_reset - Set the flags and schedule a reset event 136 * @adapter: board private structure 137 **/ 138 void iavf_schedule_reset(struct iavf_adapter *adapter) 139 { 140 if (!(adapter->flags & 141 (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) { 142 adapter->flags |= IAVF_FLAG_RESET_NEEDED; 143 queue_work(iavf_wq, &adapter->reset_task); 144 } 145 } 146 147 /** 148 * iavf_tx_timeout - Respond to a Tx Hang 149 * @netdev: network interface device structure 150 * @txqueue: queue number that is timing out 151 **/ 152 static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue) 153 { 154 struct iavf_adapter *adapter = netdev_priv(netdev); 155 156 adapter->tx_timeout_count++; 157 iavf_schedule_reset(adapter); 158 } 159 160 /** 161 * iavf_misc_irq_disable - Mask off interrupt generation on the NIC 162 * @adapter: board private structure 163 **/ 164 static void iavf_misc_irq_disable(struct iavf_adapter *adapter) 165 { 166 struct iavf_hw *hw = &adapter->hw; 167 168 if (!adapter->msix_entries) 169 return; 170 171 wr32(hw, IAVF_VFINT_DYN_CTL01, 0); 172 173 iavf_flush(hw); 174 175 synchronize_irq(adapter->msix_entries[0].vector); 176 } 177 178 /** 179 * iavf_misc_irq_enable - Enable default interrupt generation settings 180 * @adapter: board private structure 181 **/ 182 static void iavf_misc_irq_enable(struct iavf_adapter *adapter) 183 { 184 struct iavf_hw *hw = &adapter->hw; 185 186 wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK | 187 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 188 wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 189 190 iavf_flush(hw); 191 } 192 193 /** 194 * iavf_irq_disable - Mask off interrupt generation on the NIC 195 * @adapter: board private structure 196 **/ 197 static void iavf_irq_disable(struct iavf_adapter *adapter) 198 { 199 int i; 200 struct iavf_hw *hw = &adapter->hw; 201 202 if (!adapter->msix_entries) 203 return; 204 205 for (i = 1; i < adapter->num_msix_vectors; i++) { 206 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0); 207 synchronize_irq(adapter->msix_entries[i].vector); 208 } 209 iavf_flush(hw); 210 } 211 212 /** 213 * iavf_irq_enable_queues - Enable interrupt for specified queues 214 * @adapter: board private structure 215 * @mask: bitmap of queues to enable 216 **/ 217 void iavf_irq_enable_queues(struct iavf_adapter *adapter, u32 mask) 218 { 219 struct iavf_hw *hw = &adapter->hw; 220 int i; 221 222 for (i = 1; i < adapter->num_msix_vectors; i++) { 223 if (mask & BIT(i - 1)) { 224 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 225 IAVF_VFINT_DYN_CTLN1_INTENA_MASK | 226 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK); 227 } 228 } 229 } 230 231 /** 232 * iavf_irq_enable - Enable default interrupt generation settings 233 * @adapter: board private structure 234 * @flush: boolean value whether to run rd32() 235 **/ 236 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush) 237 { 238 struct iavf_hw *hw = &adapter->hw; 239 240 iavf_misc_irq_enable(adapter); 241 iavf_irq_enable_queues(adapter, ~0); 242 243 if (flush) 244 iavf_flush(hw); 245 } 246 247 /** 248 * iavf_msix_aq - Interrupt handler for vector 0 249 * @irq: interrupt number 250 * @data: pointer to netdev 251 **/ 252 static irqreturn_t iavf_msix_aq(int irq, void *data) 253 { 254 struct net_device *netdev = data; 255 struct iavf_adapter *adapter = netdev_priv(netdev); 256 struct iavf_hw *hw = &adapter->hw; 257 258 /* handle non-queue interrupts, these reads clear the registers */ 259 rd32(hw, IAVF_VFINT_ICR01); 260 rd32(hw, IAVF_VFINT_ICR0_ENA1); 261 262 /* schedule work on the private workqueue */ 263 queue_work(iavf_wq, &adapter->adminq_task); 264 265 return IRQ_HANDLED; 266 } 267 268 /** 269 * iavf_msix_clean_rings - MSIX mode Interrupt Handler 270 * @irq: interrupt number 271 * @data: pointer to a q_vector 272 **/ 273 static irqreturn_t iavf_msix_clean_rings(int irq, void *data) 274 { 275 struct iavf_q_vector *q_vector = data; 276 277 if (!q_vector->tx.ring && !q_vector->rx.ring) 278 return IRQ_HANDLED; 279 280 napi_schedule_irqoff(&q_vector->napi); 281 282 return IRQ_HANDLED; 283 } 284 285 /** 286 * iavf_map_vector_to_rxq - associate irqs with rx queues 287 * @adapter: board private structure 288 * @v_idx: interrupt number 289 * @r_idx: queue number 290 **/ 291 static void 292 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx) 293 { 294 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx]; 295 struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx]; 296 struct iavf_hw *hw = &adapter->hw; 297 298 rx_ring->q_vector = q_vector; 299 rx_ring->next = q_vector->rx.ring; 300 rx_ring->vsi = &adapter->vsi; 301 q_vector->rx.ring = rx_ring; 302 q_vector->rx.count++; 303 q_vector->rx.next_update = jiffies + 1; 304 q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting); 305 q_vector->ring_mask |= BIT(r_idx); 306 wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx), 307 q_vector->rx.current_itr >> 1); 308 q_vector->rx.current_itr = q_vector->rx.target_itr; 309 } 310 311 /** 312 * iavf_map_vector_to_txq - associate irqs with tx queues 313 * @adapter: board private structure 314 * @v_idx: interrupt number 315 * @t_idx: queue number 316 **/ 317 static void 318 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx) 319 { 320 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx]; 321 struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx]; 322 struct iavf_hw *hw = &adapter->hw; 323 324 tx_ring->q_vector = q_vector; 325 tx_ring->next = q_vector->tx.ring; 326 tx_ring->vsi = &adapter->vsi; 327 q_vector->tx.ring = tx_ring; 328 q_vector->tx.count++; 329 q_vector->tx.next_update = jiffies + 1; 330 q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting); 331 q_vector->num_ringpairs++; 332 wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx), 333 q_vector->tx.target_itr >> 1); 334 q_vector->tx.current_itr = q_vector->tx.target_itr; 335 } 336 337 /** 338 * iavf_map_rings_to_vectors - Maps descriptor rings to vectors 339 * @adapter: board private structure to initialize 340 * 341 * This function maps descriptor rings to the queue-specific vectors 342 * we were allotted through the MSI-X enabling code. Ideally, we'd have 343 * one vector per ring/queue, but on a constrained vector budget, we 344 * group the rings as "efficiently" as possible. You would add new 345 * mapping configurations in here. 346 **/ 347 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter) 348 { 349 int rings_remaining = adapter->num_active_queues; 350 int ridx = 0, vidx = 0; 351 int q_vectors; 352 353 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 354 355 for (; ridx < rings_remaining; ridx++) { 356 iavf_map_vector_to_rxq(adapter, vidx, ridx); 357 iavf_map_vector_to_txq(adapter, vidx, ridx); 358 359 /* In the case where we have more queues than vectors, continue 360 * round-robin on vectors until all queues are mapped. 361 */ 362 if (++vidx >= q_vectors) 363 vidx = 0; 364 } 365 366 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS; 367 } 368 369 /** 370 * iavf_irq_affinity_notify - Callback for affinity changes 371 * @notify: context as to what irq was changed 372 * @mask: the new affinity mask 373 * 374 * This is a callback function used by the irq_set_affinity_notifier function 375 * so that we may register to receive changes to the irq affinity masks. 376 **/ 377 static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify, 378 const cpumask_t *mask) 379 { 380 struct iavf_q_vector *q_vector = 381 container_of(notify, struct iavf_q_vector, affinity_notify); 382 383 cpumask_copy(&q_vector->affinity_mask, mask); 384 } 385 386 /** 387 * iavf_irq_affinity_release - Callback for affinity notifier release 388 * @ref: internal core kernel usage 389 * 390 * This is a callback function used by the irq_set_affinity_notifier function 391 * to inform the current notification subscriber that they will no longer 392 * receive notifications. 393 **/ 394 static void iavf_irq_affinity_release(struct kref *ref) {} 395 396 /** 397 * iavf_request_traffic_irqs - Initialize MSI-X interrupts 398 * @adapter: board private structure 399 * @basename: device basename 400 * 401 * Allocates MSI-X vectors for tx and rx handling, and requests 402 * interrupts from the kernel. 403 **/ 404 static int 405 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename) 406 { 407 unsigned int vector, q_vectors; 408 unsigned int rx_int_idx = 0, tx_int_idx = 0; 409 int irq_num, err; 410 int cpu; 411 412 iavf_irq_disable(adapter); 413 /* Decrement for Other and TCP Timer vectors */ 414 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 415 416 for (vector = 0; vector < q_vectors; vector++) { 417 struct iavf_q_vector *q_vector = &adapter->q_vectors[vector]; 418 419 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 420 421 if (q_vector->tx.ring && q_vector->rx.ring) { 422 snprintf(q_vector->name, sizeof(q_vector->name), 423 "iavf-%s-TxRx-%d", basename, rx_int_idx++); 424 tx_int_idx++; 425 } else if (q_vector->rx.ring) { 426 snprintf(q_vector->name, sizeof(q_vector->name), 427 "iavf-%s-rx-%d", basename, rx_int_idx++); 428 } else if (q_vector->tx.ring) { 429 snprintf(q_vector->name, sizeof(q_vector->name), 430 "iavf-%s-tx-%d", basename, tx_int_idx++); 431 } else { 432 /* skip this unused q_vector */ 433 continue; 434 } 435 err = request_irq(irq_num, 436 iavf_msix_clean_rings, 437 0, 438 q_vector->name, 439 q_vector); 440 if (err) { 441 dev_info(&adapter->pdev->dev, 442 "Request_irq failed, error: %d\n", err); 443 goto free_queue_irqs; 444 } 445 /* register for affinity change notifications */ 446 q_vector->affinity_notify.notify = iavf_irq_affinity_notify; 447 q_vector->affinity_notify.release = 448 iavf_irq_affinity_release; 449 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 450 /* Spread the IRQ affinity hints across online CPUs. Note that 451 * get_cpu_mask returns a mask with a permanent lifetime so 452 * it's safe to use as a hint for irq_set_affinity_hint. 453 */ 454 cpu = cpumask_local_spread(q_vector->v_idx, -1); 455 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu)); 456 } 457 458 return 0; 459 460 free_queue_irqs: 461 while (vector) { 462 vector--; 463 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 464 irq_set_affinity_notifier(irq_num, NULL); 465 irq_set_affinity_hint(irq_num, NULL); 466 free_irq(irq_num, &adapter->q_vectors[vector]); 467 } 468 return err; 469 } 470 471 /** 472 * iavf_request_misc_irq - Initialize MSI-X interrupts 473 * @adapter: board private structure 474 * 475 * Allocates MSI-X vector 0 and requests interrupts from the kernel. This 476 * vector is only for the admin queue, and stays active even when the netdev 477 * is closed. 478 **/ 479 static int iavf_request_misc_irq(struct iavf_adapter *adapter) 480 { 481 struct net_device *netdev = adapter->netdev; 482 int err; 483 484 snprintf(adapter->misc_vector_name, 485 sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx", 486 dev_name(&adapter->pdev->dev)); 487 err = request_irq(adapter->msix_entries[0].vector, 488 &iavf_msix_aq, 0, 489 adapter->misc_vector_name, netdev); 490 if (err) { 491 dev_err(&adapter->pdev->dev, 492 "request_irq for %s failed: %d\n", 493 adapter->misc_vector_name, err); 494 free_irq(adapter->msix_entries[0].vector, netdev); 495 } 496 return err; 497 } 498 499 /** 500 * iavf_free_traffic_irqs - Free MSI-X interrupts 501 * @adapter: board private structure 502 * 503 * Frees all MSI-X vectors other than 0. 504 **/ 505 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter) 506 { 507 int vector, irq_num, q_vectors; 508 509 if (!adapter->msix_entries) 510 return; 511 512 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 513 514 for (vector = 0; vector < q_vectors; vector++) { 515 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 516 irq_set_affinity_notifier(irq_num, NULL); 517 irq_set_affinity_hint(irq_num, NULL); 518 free_irq(irq_num, &adapter->q_vectors[vector]); 519 } 520 } 521 522 /** 523 * iavf_free_misc_irq - Free MSI-X miscellaneous vector 524 * @adapter: board private structure 525 * 526 * Frees MSI-X vector 0. 527 **/ 528 static void iavf_free_misc_irq(struct iavf_adapter *adapter) 529 { 530 struct net_device *netdev = adapter->netdev; 531 532 if (!adapter->msix_entries) 533 return; 534 535 free_irq(adapter->msix_entries[0].vector, netdev); 536 } 537 538 /** 539 * iavf_configure_tx - Configure Transmit Unit after Reset 540 * @adapter: board private structure 541 * 542 * Configure the Tx unit of the MAC after a reset. 543 **/ 544 static void iavf_configure_tx(struct iavf_adapter *adapter) 545 { 546 struct iavf_hw *hw = &adapter->hw; 547 int i; 548 549 for (i = 0; i < adapter->num_active_queues; i++) 550 adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i); 551 } 552 553 /** 554 * iavf_configure_rx - Configure Receive Unit after Reset 555 * @adapter: board private structure 556 * 557 * Configure the Rx unit of the MAC after a reset. 558 **/ 559 static void iavf_configure_rx(struct iavf_adapter *adapter) 560 { 561 unsigned int rx_buf_len = IAVF_RXBUFFER_2048; 562 struct iavf_hw *hw = &adapter->hw; 563 int i; 564 565 /* Legacy Rx will always default to a 2048 buffer size. */ 566 #if (PAGE_SIZE < 8192) 567 if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) { 568 struct net_device *netdev = adapter->netdev; 569 570 /* For jumbo frames on systems with 4K pages we have to use 571 * an order 1 page, so we might as well increase the size 572 * of our Rx buffer to make better use of the available space 573 */ 574 rx_buf_len = IAVF_RXBUFFER_3072; 575 576 /* We use a 1536 buffer size for configurations with 577 * standard Ethernet mtu. On x86 this gives us enough room 578 * for shared info and 192 bytes of padding. 579 */ 580 if (!IAVF_2K_TOO_SMALL_WITH_PADDING && 581 (netdev->mtu <= ETH_DATA_LEN)) 582 rx_buf_len = IAVF_RXBUFFER_1536 - NET_IP_ALIGN; 583 } 584 #endif 585 586 for (i = 0; i < adapter->num_active_queues; i++) { 587 adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i); 588 adapter->rx_rings[i].rx_buf_len = rx_buf_len; 589 590 if (adapter->flags & IAVF_FLAG_LEGACY_RX) 591 clear_ring_build_skb_enabled(&adapter->rx_rings[i]); 592 else 593 set_ring_build_skb_enabled(&adapter->rx_rings[i]); 594 } 595 } 596 597 /** 598 * iavf_find_vlan - Search filter list for specific vlan filter 599 * @adapter: board private structure 600 * @vlan: vlan tag 601 * 602 * Returns ptr to the filter object or NULL. Must be called while holding the 603 * mac_vlan_list_lock. 604 **/ 605 static struct 606 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter, u16 vlan) 607 { 608 struct iavf_vlan_filter *f; 609 610 list_for_each_entry(f, &adapter->vlan_filter_list, list) { 611 if (vlan == f->vlan) 612 return f; 613 } 614 return NULL; 615 } 616 617 /** 618 * iavf_add_vlan - Add a vlan filter to the list 619 * @adapter: board private structure 620 * @vlan: VLAN tag 621 * 622 * Returns ptr to the filter object or NULL when no memory available. 623 **/ 624 static struct 625 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, u16 vlan) 626 { 627 struct iavf_vlan_filter *f = NULL; 628 629 spin_lock_bh(&adapter->mac_vlan_list_lock); 630 631 f = iavf_find_vlan(adapter, vlan); 632 if (!f) { 633 f = kzalloc(sizeof(*f), GFP_ATOMIC); 634 if (!f) 635 goto clearout; 636 637 f->vlan = vlan; 638 639 list_add_tail(&f->list, &adapter->vlan_filter_list); 640 f->add = true; 641 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER; 642 } 643 644 clearout: 645 spin_unlock_bh(&adapter->mac_vlan_list_lock); 646 return f; 647 } 648 649 /** 650 * iavf_del_vlan - Remove a vlan filter from the list 651 * @adapter: board private structure 652 * @vlan: VLAN tag 653 **/ 654 static void iavf_del_vlan(struct iavf_adapter *adapter, u16 vlan) 655 { 656 struct iavf_vlan_filter *f; 657 658 spin_lock_bh(&adapter->mac_vlan_list_lock); 659 660 f = iavf_find_vlan(adapter, vlan); 661 if (f) { 662 f->remove = true; 663 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER; 664 } 665 666 spin_unlock_bh(&adapter->mac_vlan_list_lock); 667 } 668 669 /** 670 * iavf_vlan_rx_add_vid - Add a VLAN filter to a device 671 * @netdev: network device struct 672 * @proto: unused protocol data 673 * @vid: VLAN tag 674 **/ 675 static int iavf_vlan_rx_add_vid(struct net_device *netdev, 676 __always_unused __be16 proto, u16 vid) 677 { 678 struct iavf_adapter *adapter = netdev_priv(netdev); 679 680 if (!VLAN_ALLOWED(adapter)) 681 return -EIO; 682 if (iavf_add_vlan(adapter, vid) == NULL) 683 return -ENOMEM; 684 return 0; 685 } 686 687 /** 688 * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device 689 * @netdev: network device struct 690 * @proto: unused protocol data 691 * @vid: VLAN tag 692 **/ 693 static int iavf_vlan_rx_kill_vid(struct net_device *netdev, 694 __always_unused __be16 proto, u16 vid) 695 { 696 struct iavf_adapter *adapter = netdev_priv(netdev); 697 698 if (VLAN_ALLOWED(adapter)) { 699 iavf_del_vlan(adapter, vid); 700 return 0; 701 } 702 return -EIO; 703 } 704 705 /** 706 * iavf_find_filter - Search filter list for specific mac filter 707 * @adapter: board private structure 708 * @macaddr: the MAC address 709 * 710 * Returns ptr to the filter object or NULL. Must be called while holding the 711 * mac_vlan_list_lock. 712 **/ 713 static struct 714 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter, 715 const u8 *macaddr) 716 { 717 struct iavf_mac_filter *f; 718 719 if (!macaddr) 720 return NULL; 721 722 list_for_each_entry(f, &adapter->mac_filter_list, list) { 723 if (ether_addr_equal(macaddr, f->macaddr)) 724 return f; 725 } 726 return NULL; 727 } 728 729 /** 730 * iavf_add_filter - Add a mac filter to the filter list 731 * @adapter: board private structure 732 * @macaddr: the MAC address 733 * 734 * Returns ptr to the filter object or NULL when no memory available. 735 **/ 736 struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter, 737 const u8 *macaddr) 738 { 739 struct iavf_mac_filter *f; 740 741 if (!macaddr) 742 return NULL; 743 744 f = iavf_find_filter(adapter, macaddr); 745 if (!f) { 746 f = kzalloc(sizeof(*f), GFP_ATOMIC); 747 if (!f) 748 return f; 749 750 ether_addr_copy(f->macaddr, macaddr); 751 752 list_add_tail(&f->list, &adapter->mac_filter_list); 753 f->add = true; 754 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; 755 } else { 756 f->remove = false; 757 } 758 759 return f; 760 } 761 762 /** 763 * iavf_set_mac - NDO callback to set port mac address 764 * @netdev: network interface device structure 765 * @p: pointer to an address structure 766 * 767 * Returns 0 on success, negative on failure 768 **/ 769 static int iavf_set_mac(struct net_device *netdev, void *p) 770 { 771 struct iavf_adapter *adapter = netdev_priv(netdev); 772 struct iavf_hw *hw = &adapter->hw; 773 struct iavf_mac_filter *f; 774 struct sockaddr *addr = p; 775 776 if (!is_valid_ether_addr(addr->sa_data)) 777 return -EADDRNOTAVAIL; 778 779 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) 780 return 0; 781 782 spin_lock_bh(&adapter->mac_vlan_list_lock); 783 784 f = iavf_find_filter(adapter, hw->mac.addr); 785 if (f) { 786 f->remove = true; 787 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; 788 } 789 790 f = iavf_add_filter(adapter, addr->sa_data); 791 792 spin_unlock_bh(&adapter->mac_vlan_list_lock); 793 794 if (f) { 795 ether_addr_copy(hw->mac.addr, addr->sa_data); 796 } 797 798 return (f == NULL) ? -ENOMEM : 0; 799 } 800 801 /** 802 * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address 803 * @netdev: the netdevice 804 * @addr: address to add 805 * 806 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 807 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 808 */ 809 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr) 810 { 811 struct iavf_adapter *adapter = netdev_priv(netdev); 812 813 if (iavf_add_filter(adapter, addr)) 814 return 0; 815 else 816 return -ENOMEM; 817 } 818 819 /** 820 * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 821 * @netdev: the netdevice 822 * @addr: address to add 823 * 824 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 825 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 826 */ 827 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr) 828 { 829 struct iavf_adapter *adapter = netdev_priv(netdev); 830 struct iavf_mac_filter *f; 831 832 /* Under some circumstances, we might receive a request to delete 833 * our own device address from our uc list. Because we store the 834 * device address in the VSI's MAC/VLAN filter list, we need to ignore 835 * such requests and not delete our device address from this list. 836 */ 837 if (ether_addr_equal(addr, netdev->dev_addr)) 838 return 0; 839 840 f = iavf_find_filter(adapter, addr); 841 if (f) { 842 f->remove = true; 843 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; 844 } 845 return 0; 846 } 847 848 /** 849 * iavf_set_rx_mode - NDO callback to set the netdev filters 850 * @netdev: network interface device structure 851 **/ 852 static void iavf_set_rx_mode(struct net_device *netdev) 853 { 854 struct iavf_adapter *adapter = netdev_priv(netdev); 855 856 spin_lock_bh(&adapter->mac_vlan_list_lock); 857 __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync); 858 __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync); 859 spin_unlock_bh(&adapter->mac_vlan_list_lock); 860 861 if (netdev->flags & IFF_PROMISC && 862 !(adapter->flags & IAVF_FLAG_PROMISC_ON)) 863 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_PROMISC; 864 else if (!(netdev->flags & IFF_PROMISC) && 865 adapter->flags & IAVF_FLAG_PROMISC_ON) 866 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_PROMISC; 867 868 if (netdev->flags & IFF_ALLMULTI && 869 !(adapter->flags & IAVF_FLAG_ALLMULTI_ON)) 870 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_ALLMULTI; 871 else if (!(netdev->flags & IFF_ALLMULTI) && 872 adapter->flags & IAVF_FLAG_ALLMULTI_ON) 873 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_ALLMULTI; 874 } 875 876 /** 877 * iavf_napi_enable_all - enable NAPI on all queue vectors 878 * @adapter: board private structure 879 **/ 880 static void iavf_napi_enable_all(struct iavf_adapter *adapter) 881 { 882 int q_idx; 883 struct iavf_q_vector *q_vector; 884 int q_vectors = adapter->num_msix_vectors - NONQ_VECS; 885 886 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 887 struct napi_struct *napi; 888 889 q_vector = &adapter->q_vectors[q_idx]; 890 napi = &q_vector->napi; 891 napi_enable(napi); 892 } 893 } 894 895 /** 896 * iavf_napi_disable_all - disable NAPI on all queue vectors 897 * @adapter: board private structure 898 **/ 899 static void iavf_napi_disable_all(struct iavf_adapter *adapter) 900 { 901 int q_idx; 902 struct iavf_q_vector *q_vector; 903 int q_vectors = adapter->num_msix_vectors - NONQ_VECS; 904 905 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 906 q_vector = &adapter->q_vectors[q_idx]; 907 napi_disable(&q_vector->napi); 908 } 909 } 910 911 /** 912 * iavf_configure - set up transmit and receive data structures 913 * @adapter: board private structure 914 **/ 915 static void iavf_configure(struct iavf_adapter *adapter) 916 { 917 struct net_device *netdev = adapter->netdev; 918 int i; 919 920 iavf_set_rx_mode(netdev); 921 922 iavf_configure_tx(adapter); 923 iavf_configure_rx(adapter); 924 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES; 925 926 for (i = 0; i < adapter->num_active_queues; i++) { 927 struct iavf_ring *ring = &adapter->rx_rings[i]; 928 929 iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring)); 930 } 931 } 932 933 /** 934 * iavf_up_complete - Finish the last steps of bringing up a connection 935 * @adapter: board private structure 936 * 937 * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock. 938 **/ 939 static void iavf_up_complete(struct iavf_adapter *adapter) 940 { 941 adapter->state = __IAVF_RUNNING; 942 clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 943 944 iavf_napi_enable_all(adapter); 945 946 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES; 947 if (CLIENT_ENABLED(adapter)) 948 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN; 949 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); 950 } 951 952 /** 953 * iavf_down - Shutdown the connection processing 954 * @adapter: board private structure 955 * 956 * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock. 957 **/ 958 void iavf_down(struct iavf_adapter *adapter) 959 { 960 struct net_device *netdev = adapter->netdev; 961 struct iavf_vlan_filter *vlf; 962 struct iavf_mac_filter *f; 963 struct iavf_cloud_filter *cf; 964 965 if (adapter->state <= __IAVF_DOWN_PENDING) 966 return; 967 968 netif_carrier_off(netdev); 969 netif_tx_disable(netdev); 970 adapter->link_up = false; 971 iavf_napi_disable_all(adapter); 972 iavf_irq_disable(adapter); 973 974 spin_lock_bh(&adapter->mac_vlan_list_lock); 975 976 /* clear the sync flag on all filters */ 977 __dev_uc_unsync(adapter->netdev, NULL); 978 __dev_mc_unsync(adapter->netdev, NULL); 979 980 /* remove all MAC filters */ 981 list_for_each_entry(f, &adapter->mac_filter_list, list) { 982 f->remove = true; 983 } 984 985 /* remove all VLAN filters */ 986 list_for_each_entry(vlf, &adapter->vlan_filter_list, list) { 987 vlf->remove = true; 988 } 989 990 spin_unlock_bh(&adapter->mac_vlan_list_lock); 991 992 /* remove all cloud filters */ 993 spin_lock_bh(&adapter->cloud_filter_list_lock); 994 list_for_each_entry(cf, &adapter->cloud_filter_list, list) { 995 cf->del = true; 996 } 997 spin_unlock_bh(&adapter->cloud_filter_list_lock); 998 999 if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) && 1000 adapter->state != __IAVF_RESETTING) { 1001 /* cancel any current operation */ 1002 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 1003 /* Schedule operations to close down the HW. Don't wait 1004 * here for this to complete. The watchdog is still running 1005 * and it will take care of this. 1006 */ 1007 adapter->aq_required = IAVF_FLAG_AQ_DEL_MAC_FILTER; 1008 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER; 1009 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; 1010 adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES; 1011 } 1012 1013 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); 1014 } 1015 1016 /** 1017 * iavf_acquire_msix_vectors - Setup the MSIX capability 1018 * @adapter: board private structure 1019 * @vectors: number of vectors to request 1020 * 1021 * Work with the OS to set up the MSIX vectors needed. 1022 * 1023 * Returns 0 on success, negative on failure 1024 **/ 1025 static int 1026 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors) 1027 { 1028 int err, vector_threshold; 1029 1030 /* We'll want at least 3 (vector_threshold): 1031 * 0) Other (Admin Queue and link, mostly) 1032 * 1) TxQ[0] Cleanup 1033 * 2) RxQ[0] Cleanup 1034 */ 1035 vector_threshold = MIN_MSIX_COUNT; 1036 1037 /* The more we get, the more we will assign to Tx/Rx Cleanup 1038 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 1039 * Right now, we simply care about how many we'll get; we'll 1040 * set them up later while requesting irq's. 1041 */ 1042 err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 1043 vector_threshold, vectors); 1044 if (err < 0) { 1045 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n"); 1046 kfree(adapter->msix_entries); 1047 adapter->msix_entries = NULL; 1048 return err; 1049 } 1050 1051 /* Adjust for only the vectors we'll use, which is minimum 1052 * of max_msix_q_vectors + NONQ_VECS, or the number of 1053 * vectors we were allocated. 1054 */ 1055 adapter->num_msix_vectors = err; 1056 return 0; 1057 } 1058 1059 /** 1060 * iavf_free_queues - Free memory for all rings 1061 * @adapter: board private structure to initialize 1062 * 1063 * Free all of the memory associated with queue pairs. 1064 **/ 1065 static void iavf_free_queues(struct iavf_adapter *adapter) 1066 { 1067 if (!adapter->vsi_res) 1068 return; 1069 adapter->num_active_queues = 0; 1070 kfree(adapter->tx_rings); 1071 adapter->tx_rings = NULL; 1072 kfree(adapter->rx_rings); 1073 adapter->rx_rings = NULL; 1074 } 1075 1076 /** 1077 * iavf_alloc_queues - Allocate memory for all rings 1078 * @adapter: board private structure to initialize 1079 * 1080 * We allocate one ring per queue at run-time since we don't know the 1081 * number of queues at compile-time. The polling_netdev array is 1082 * intended for Multiqueue, but should work fine with a single queue. 1083 **/ 1084 static int iavf_alloc_queues(struct iavf_adapter *adapter) 1085 { 1086 int i, num_active_queues; 1087 1088 /* If we're in reset reallocating queues we don't actually know yet for 1089 * certain the PF gave us the number of queues we asked for but we'll 1090 * assume it did. Once basic reset is finished we'll confirm once we 1091 * start negotiating config with PF. 1092 */ 1093 if (adapter->num_req_queues) 1094 num_active_queues = adapter->num_req_queues; 1095 else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 1096 adapter->num_tc) 1097 num_active_queues = adapter->ch_config.total_qps; 1098 else 1099 num_active_queues = min_t(int, 1100 adapter->vsi_res->num_queue_pairs, 1101 (int)(num_online_cpus())); 1102 1103 1104 adapter->tx_rings = kcalloc(num_active_queues, 1105 sizeof(struct iavf_ring), GFP_KERNEL); 1106 if (!adapter->tx_rings) 1107 goto err_out; 1108 adapter->rx_rings = kcalloc(num_active_queues, 1109 sizeof(struct iavf_ring), GFP_KERNEL); 1110 if (!adapter->rx_rings) 1111 goto err_out; 1112 1113 for (i = 0; i < num_active_queues; i++) { 1114 struct iavf_ring *tx_ring; 1115 struct iavf_ring *rx_ring; 1116 1117 tx_ring = &adapter->tx_rings[i]; 1118 1119 tx_ring->queue_index = i; 1120 tx_ring->netdev = adapter->netdev; 1121 tx_ring->dev = &adapter->pdev->dev; 1122 tx_ring->count = adapter->tx_desc_count; 1123 tx_ring->itr_setting = IAVF_ITR_TX_DEF; 1124 if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE) 1125 tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR; 1126 1127 rx_ring = &adapter->rx_rings[i]; 1128 rx_ring->queue_index = i; 1129 rx_ring->netdev = adapter->netdev; 1130 rx_ring->dev = &adapter->pdev->dev; 1131 rx_ring->count = adapter->rx_desc_count; 1132 rx_ring->itr_setting = IAVF_ITR_RX_DEF; 1133 } 1134 1135 adapter->num_active_queues = num_active_queues; 1136 1137 return 0; 1138 1139 err_out: 1140 iavf_free_queues(adapter); 1141 return -ENOMEM; 1142 } 1143 1144 /** 1145 * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported 1146 * @adapter: board private structure to initialize 1147 * 1148 * Attempt to configure the interrupts using the best available 1149 * capabilities of the hardware and the kernel. 1150 **/ 1151 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter) 1152 { 1153 int vector, v_budget; 1154 int pairs = 0; 1155 int err = 0; 1156 1157 if (!adapter->vsi_res) { 1158 err = -EIO; 1159 goto out; 1160 } 1161 pairs = adapter->num_active_queues; 1162 1163 /* It's easy to be greedy for MSI-X vectors, but it really doesn't do 1164 * us much good if we have more vectors than CPUs. However, we already 1165 * limit the total number of queues by the number of CPUs so we do not 1166 * need any further limiting here. 1167 */ 1168 v_budget = min_t(int, pairs + NONQ_VECS, 1169 (int)adapter->vf_res->max_vectors); 1170 1171 adapter->msix_entries = kcalloc(v_budget, 1172 sizeof(struct msix_entry), GFP_KERNEL); 1173 if (!adapter->msix_entries) { 1174 err = -ENOMEM; 1175 goto out; 1176 } 1177 1178 for (vector = 0; vector < v_budget; vector++) 1179 adapter->msix_entries[vector].entry = vector; 1180 1181 err = iavf_acquire_msix_vectors(adapter, v_budget); 1182 1183 out: 1184 netif_set_real_num_rx_queues(adapter->netdev, pairs); 1185 netif_set_real_num_tx_queues(adapter->netdev, pairs); 1186 return err; 1187 } 1188 1189 /** 1190 * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands 1191 * @adapter: board private structure 1192 * 1193 * Return 0 on success, negative on failure 1194 **/ 1195 static int iavf_config_rss_aq(struct iavf_adapter *adapter) 1196 { 1197 struct iavf_aqc_get_set_rss_key_data *rss_key = 1198 (struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key; 1199 struct iavf_hw *hw = &adapter->hw; 1200 int ret = 0; 1201 1202 if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { 1203 /* bail because we already have a command pending */ 1204 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n", 1205 adapter->current_op); 1206 return -EBUSY; 1207 } 1208 1209 ret = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key); 1210 if (ret) { 1211 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n", 1212 iavf_stat_str(hw, ret), 1213 iavf_aq_str(hw, hw->aq.asq_last_status)); 1214 return ret; 1215 1216 } 1217 1218 ret = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false, 1219 adapter->rss_lut, adapter->rss_lut_size); 1220 if (ret) { 1221 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n", 1222 iavf_stat_str(hw, ret), 1223 iavf_aq_str(hw, hw->aq.asq_last_status)); 1224 } 1225 1226 return ret; 1227 1228 } 1229 1230 /** 1231 * iavf_config_rss_reg - Configure RSS keys and lut by writing registers 1232 * @adapter: board private structure 1233 * 1234 * Returns 0 on success, negative on failure 1235 **/ 1236 static int iavf_config_rss_reg(struct iavf_adapter *adapter) 1237 { 1238 struct iavf_hw *hw = &adapter->hw; 1239 u32 *dw; 1240 u16 i; 1241 1242 dw = (u32 *)adapter->rss_key; 1243 for (i = 0; i <= adapter->rss_key_size / 4; i++) 1244 wr32(hw, IAVF_VFQF_HKEY(i), dw[i]); 1245 1246 dw = (u32 *)adapter->rss_lut; 1247 for (i = 0; i <= adapter->rss_lut_size / 4; i++) 1248 wr32(hw, IAVF_VFQF_HLUT(i), dw[i]); 1249 1250 iavf_flush(hw); 1251 1252 return 0; 1253 } 1254 1255 /** 1256 * iavf_config_rss - Configure RSS keys and lut 1257 * @adapter: board private structure 1258 * 1259 * Returns 0 on success, negative on failure 1260 **/ 1261 int iavf_config_rss(struct iavf_adapter *adapter) 1262 { 1263 1264 if (RSS_PF(adapter)) { 1265 adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT | 1266 IAVF_FLAG_AQ_SET_RSS_KEY; 1267 return 0; 1268 } else if (RSS_AQ(adapter)) { 1269 return iavf_config_rss_aq(adapter); 1270 } else { 1271 return iavf_config_rss_reg(adapter); 1272 } 1273 } 1274 1275 /** 1276 * iavf_fill_rss_lut - Fill the lut with default values 1277 * @adapter: board private structure 1278 **/ 1279 static void iavf_fill_rss_lut(struct iavf_adapter *adapter) 1280 { 1281 u16 i; 1282 1283 for (i = 0; i < adapter->rss_lut_size; i++) 1284 adapter->rss_lut[i] = i % adapter->num_active_queues; 1285 } 1286 1287 /** 1288 * iavf_init_rss - Prepare for RSS 1289 * @adapter: board private structure 1290 * 1291 * Return 0 on success, negative on failure 1292 **/ 1293 static int iavf_init_rss(struct iavf_adapter *adapter) 1294 { 1295 struct iavf_hw *hw = &adapter->hw; 1296 int ret; 1297 1298 if (!RSS_PF(adapter)) { 1299 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */ 1300 if (adapter->vf_res->vf_cap_flags & 1301 VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2) 1302 adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED; 1303 else 1304 adapter->hena = IAVF_DEFAULT_RSS_HENA; 1305 1306 wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena); 1307 wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32)); 1308 } 1309 1310 iavf_fill_rss_lut(adapter); 1311 netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size); 1312 ret = iavf_config_rss(adapter); 1313 1314 return ret; 1315 } 1316 1317 /** 1318 * iavf_alloc_q_vectors - Allocate memory for interrupt vectors 1319 * @adapter: board private structure to initialize 1320 * 1321 * We allocate one q_vector per queue interrupt. If allocation fails we 1322 * return -ENOMEM. 1323 **/ 1324 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter) 1325 { 1326 int q_idx = 0, num_q_vectors; 1327 struct iavf_q_vector *q_vector; 1328 1329 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1330 adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector), 1331 GFP_KERNEL); 1332 if (!adapter->q_vectors) 1333 return -ENOMEM; 1334 1335 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1336 q_vector = &adapter->q_vectors[q_idx]; 1337 q_vector->adapter = adapter; 1338 q_vector->vsi = &adapter->vsi; 1339 q_vector->v_idx = q_idx; 1340 q_vector->reg_idx = q_idx; 1341 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 1342 netif_napi_add(adapter->netdev, &q_vector->napi, 1343 iavf_napi_poll, NAPI_POLL_WEIGHT); 1344 } 1345 1346 return 0; 1347 } 1348 1349 /** 1350 * iavf_free_q_vectors - Free memory allocated for interrupt vectors 1351 * @adapter: board private structure to initialize 1352 * 1353 * This function frees the memory allocated to the q_vectors. In addition if 1354 * NAPI is enabled it will delete any references to the NAPI struct prior 1355 * to freeing the q_vector. 1356 **/ 1357 static void iavf_free_q_vectors(struct iavf_adapter *adapter) 1358 { 1359 int q_idx, num_q_vectors; 1360 int napi_vectors; 1361 1362 if (!adapter->q_vectors) 1363 return; 1364 1365 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1366 napi_vectors = adapter->num_active_queues; 1367 1368 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1369 struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx]; 1370 1371 if (q_idx < napi_vectors) 1372 netif_napi_del(&q_vector->napi); 1373 } 1374 kfree(adapter->q_vectors); 1375 adapter->q_vectors = NULL; 1376 } 1377 1378 /** 1379 * iavf_reset_interrupt_capability - Reset MSIX setup 1380 * @adapter: board private structure 1381 * 1382 **/ 1383 void iavf_reset_interrupt_capability(struct iavf_adapter *adapter) 1384 { 1385 if (!adapter->msix_entries) 1386 return; 1387 1388 pci_disable_msix(adapter->pdev); 1389 kfree(adapter->msix_entries); 1390 adapter->msix_entries = NULL; 1391 } 1392 1393 /** 1394 * iavf_init_interrupt_scheme - Determine if MSIX is supported and init 1395 * @adapter: board private structure to initialize 1396 * 1397 **/ 1398 int iavf_init_interrupt_scheme(struct iavf_adapter *adapter) 1399 { 1400 int err; 1401 1402 err = iavf_alloc_queues(adapter); 1403 if (err) { 1404 dev_err(&adapter->pdev->dev, 1405 "Unable to allocate memory for queues\n"); 1406 goto err_alloc_queues; 1407 } 1408 1409 rtnl_lock(); 1410 err = iavf_set_interrupt_capability(adapter); 1411 rtnl_unlock(); 1412 if (err) { 1413 dev_err(&adapter->pdev->dev, 1414 "Unable to setup interrupt capabilities\n"); 1415 goto err_set_interrupt; 1416 } 1417 1418 err = iavf_alloc_q_vectors(adapter); 1419 if (err) { 1420 dev_err(&adapter->pdev->dev, 1421 "Unable to allocate memory for queue vectors\n"); 1422 goto err_alloc_q_vectors; 1423 } 1424 1425 /* If we've made it so far while ADq flag being ON, then we haven't 1426 * bailed out anywhere in middle. And ADq isn't just enabled but actual 1427 * resources have been allocated in the reset path. 1428 * Now we can truly claim that ADq is enabled. 1429 */ 1430 if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 1431 adapter->num_tc) 1432 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created", 1433 adapter->num_tc); 1434 1435 dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u", 1436 (adapter->num_active_queues > 1) ? "Enabled" : "Disabled", 1437 adapter->num_active_queues); 1438 1439 return 0; 1440 err_alloc_q_vectors: 1441 iavf_reset_interrupt_capability(adapter); 1442 err_set_interrupt: 1443 iavf_free_queues(adapter); 1444 err_alloc_queues: 1445 return err; 1446 } 1447 1448 /** 1449 * iavf_free_rss - Free memory used by RSS structs 1450 * @adapter: board private structure 1451 **/ 1452 static void iavf_free_rss(struct iavf_adapter *adapter) 1453 { 1454 kfree(adapter->rss_key); 1455 adapter->rss_key = NULL; 1456 1457 kfree(adapter->rss_lut); 1458 adapter->rss_lut = NULL; 1459 } 1460 1461 /** 1462 * iavf_reinit_interrupt_scheme - Reallocate queues and vectors 1463 * @adapter: board private structure 1464 * 1465 * Returns 0 on success, negative on failure 1466 **/ 1467 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter) 1468 { 1469 struct net_device *netdev = adapter->netdev; 1470 int err; 1471 1472 if (netif_running(netdev)) 1473 iavf_free_traffic_irqs(adapter); 1474 iavf_free_misc_irq(adapter); 1475 iavf_reset_interrupt_capability(adapter); 1476 iavf_free_q_vectors(adapter); 1477 iavf_free_queues(adapter); 1478 1479 err = iavf_init_interrupt_scheme(adapter); 1480 if (err) 1481 goto err; 1482 1483 netif_tx_stop_all_queues(netdev); 1484 1485 err = iavf_request_misc_irq(adapter); 1486 if (err) 1487 goto err; 1488 1489 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 1490 1491 iavf_map_rings_to_vectors(adapter); 1492 1493 if (RSS_AQ(adapter)) 1494 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS; 1495 else 1496 err = iavf_init_rss(adapter); 1497 err: 1498 return err; 1499 } 1500 1501 /** 1502 * iavf_process_aq_command - process aq_required flags 1503 * and sends aq command 1504 * @adapter: pointer to iavf adapter structure 1505 * 1506 * Returns 0 on success 1507 * Returns error code if no command was sent 1508 * or error code if the command failed. 1509 **/ 1510 static int iavf_process_aq_command(struct iavf_adapter *adapter) 1511 { 1512 if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG) 1513 return iavf_send_vf_config_msg(adapter); 1514 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) { 1515 iavf_disable_queues(adapter); 1516 return 0; 1517 } 1518 1519 if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) { 1520 iavf_map_queues(adapter); 1521 return 0; 1522 } 1523 1524 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) { 1525 iavf_add_ether_addrs(adapter); 1526 return 0; 1527 } 1528 1529 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) { 1530 iavf_add_vlans(adapter); 1531 return 0; 1532 } 1533 1534 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) { 1535 iavf_del_ether_addrs(adapter); 1536 return 0; 1537 } 1538 1539 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) { 1540 iavf_del_vlans(adapter); 1541 return 0; 1542 } 1543 1544 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) { 1545 iavf_enable_vlan_stripping(adapter); 1546 return 0; 1547 } 1548 1549 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) { 1550 iavf_disable_vlan_stripping(adapter); 1551 return 0; 1552 } 1553 1554 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) { 1555 iavf_configure_queues(adapter); 1556 return 0; 1557 } 1558 1559 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) { 1560 iavf_enable_queues(adapter); 1561 return 0; 1562 } 1563 1564 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) { 1565 /* This message goes straight to the firmware, not the 1566 * PF, so we don't have to set current_op as we will 1567 * not get a response through the ARQ. 1568 */ 1569 adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS; 1570 return 0; 1571 } 1572 if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) { 1573 iavf_get_hena(adapter); 1574 return 0; 1575 } 1576 if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) { 1577 iavf_set_hena(adapter); 1578 return 0; 1579 } 1580 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) { 1581 iavf_set_rss_key(adapter); 1582 return 0; 1583 } 1584 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) { 1585 iavf_set_rss_lut(adapter); 1586 return 0; 1587 } 1588 1589 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_PROMISC) { 1590 iavf_set_promiscuous(adapter, FLAG_VF_UNICAST_PROMISC | 1591 FLAG_VF_MULTICAST_PROMISC); 1592 return 0; 1593 } 1594 1595 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_ALLMULTI) { 1596 iavf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC); 1597 return 0; 1598 } 1599 1600 if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) && 1601 (adapter->aq_required & IAVF_FLAG_AQ_RELEASE_ALLMULTI)) { 1602 iavf_set_promiscuous(adapter, 0); 1603 return 0; 1604 } 1605 1606 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) { 1607 iavf_enable_channels(adapter); 1608 return 0; 1609 } 1610 1611 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) { 1612 iavf_disable_channels(adapter); 1613 return 0; 1614 } 1615 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) { 1616 iavf_add_cloud_filter(adapter); 1617 return 0; 1618 } 1619 1620 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) { 1621 iavf_del_cloud_filter(adapter); 1622 return 0; 1623 } 1624 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) { 1625 iavf_del_cloud_filter(adapter); 1626 return 0; 1627 } 1628 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) { 1629 iavf_add_cloud_filter(adapter); 1630 return 0; 1631 } 1632 return -EAGAIN; 1633 } 1634 1635 /** 1636 * iavf_startup - first step of driver startup 1637 * @adapter: board private structure 1638 * 1639 * Function process __IAVF_STARTUP driver state. 1640 * When success the state is changed to __IAVF_INIT_VERSION_CHECK 1641 * when fails it returns -EAGAIN 1642 **/ 1643 static int iavf_startup(struct iavf_adapter *adapter) 1644 { 1645 struct pci_dev *pdev = adapter->pdev; 1646 struct iavf_hw *hw = &adapter->hw; 1647 int err; 1648 1649 WARN_ON(adapter->state != __IAVF_STARTUP); 1650 1651 /* driver loaded, probe complete */ 1652 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED; 1653 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 1654 err = iavf_set_mac_type(hw); 1655 if (err) { 1656 dev_err(&pdev->dev, "Failed to set MAC type (%d)\n", err); 1657 goto err; 1658 } 1659 1660 err = iavf_check_reset_complete(hw); 1661 if (err) { 1662 dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n", 1663 err); 1664 goto err; 1665 } 1666 hw->aq.num_arq_entries = IAVF_AQ_LEN; 1667 hw->aq.num_asq_entries = IAVF_AQ_LEN; 1668 hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE; 1669 hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE; 1670 1671 err = iavf_init_adminq(hw); 1672 if (err) { 1673 dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n", err); 1674 goto err; 1675 } 1676 err = iavf_send_api_ver(adapter); 1677 if (err) { 1678 dev_err(&pdev->dev, "Unable to send to PF (%d)\n", err); 1679 iavf_shutdown_adminq(hw); 1680 goto err; 1681 } 1682 adapter->state = __IAVF_INIT_VERSION_CHECK; 1683 err: 1684 return err; 1685 } 1686 1687 /** 1688 * iavf_init_version_check - second step of driver startup 1689 * @adapter: board private structure 1690 * 1691 * Function process __IAVF_INIT_VERSION_CHECK driver state. 1692 * When success the state is changed to __IAVF_INIT_GET_RESOURCES 1693 * when fails it returns -EAGAIN 1694 **/ 1695 static int iavf_init_version_check(struct iavf_adapter *adapter) 1696 { 1697 struct pci_dev *pdev = adapter->pdev; 1698 struct iavf_hw *hw = &adapter->hw; 1699 int err = -EAGAIN; 1700 1701 WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK); 1702 1703 if (!iavf_asq_done(hw)) { 1704 dev_err(&pdev->dev, "Admin queue command never completed\n"); 1705 iavf_shutdown_adminq(hw); 1706 adapter->state = __IAVF_STARTUP; 1707 goto err; 1708 } 1709 1710 /* aq msg sent, awaiting reply */ 1711 err = iavf_verify_api_ver(adapter); 1712 if (err) { 1713 if (err == IAVF_ERR_ADMIN_QUEUE_NO_WORK) 1714 err = iavf_send_api_ver(adapter); 1715 else 1716 dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n", 1717 adapter->pf_version.major, 1718 adapter->pf_version.minor, 1719 VIRTCHNL_VERSION_MAJOR, 1720 VIRTCHNL_VERSION_MINOR); 1721 goto err; 1722 } 1723 err = iavf_send_vf_config_msg(adapter); 1724 if (err) { 1725 dev_err(&pdev->dev, "Unable to send config request (%d)\n", 1726 err); 1727 goto err; 1728 } 1729 adapter->state = __IAVF_INIT_GET_RESOURCES; 1730 1731 err: 1732 return err; 1733 } 1734 1735 /** 1736 * iavf_init_get_resources - third step of driver startup 1737 * @adapter: board private structure 1738 * 1739 * Function process __IAVF_INIT_GET_RESOURCES driver state and 1740 * finishes driver initialization procedure. 1741 * When success the state is changed to __IAVF_DOWN 1742 * when fails it returns -EAGAIN 1743 **/ 1744 static int iavf_init_get_resources(struct iavf_adapter *adapter) 1745 { 1746 struct net_device *netdev = adapter->netdev; 1747 struct pci_dev *pdev = adapter->pdev; 1748 struct iavf_hw *hw = &adapter->hw; 1749 int err; 1750 1751 WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES); 1752 /* aq msg sent, awaiting reply */ 1753 if (!adapter->vf_res) { 1754 adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE, 1755 GFP_KERNEL); 1756 if (!adapter->vf_res) { 1757 err = -ENOMEM; 1758 goto err; 1759 } 1760 } 1761 err = iavf_get_vf_config(adapter); 1762 if (err == IAVF_ERR_ADMIN_QUEUE_NO_WORK) { 1763 err = iavf_send_vf_config_msg(adapter); 1764 goto err; 1765 } else if (err == IAVF_ERR_PARAM) { 1766 /* We only get ERR_PARAM if the device is in a very bad 1767 * state or if we've been disabled for previous bad 1768 * behavior. Either way, we're done now. 1769 */ 1770 iavf_shutdown_adminq(hw); 1771 dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n"); 1772 return 0; 1773 } 1774 if (err) { 1775 dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err); 1776 goto err_alloc; 1777 } 1778 1779 if (iavf_process_config(adapter)) 1780 goto err_alloc; 1781 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 1782 1783 adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED; 1784 1785 netdev->netdev_ops = &iavf_netdev_ops; 1786 iavf_set_ethtool_ops(netdev); 1787 netdev->watchdog_timeo = 5 * HZ; 1788 1789 /* MTU range: 68 - 9710 */ 1790 netdev->min_mtu = ETH_MIN_MTU; 1791 netdev->max_mtu = IAVF_MAX_RXBUFFER - IAVF_PACKET_HDR_PAD; 1792 1793 if (!is_valid_ether_addr(adapter->hw.mac.addr)) { 1794 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n", 1795 adapter->hw.mac.addr); 1796 eth_hw_addr_random(netdev); 1797 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr); 1798 } else { 1799 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr); 1800 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr); 1801 } 1802 1803 adapter->tx_desc_count = IAVF_DEFAULT_TXD; 1804 adapter->rx_desc_count = IAVF_DEFAULT_RXD; 1805 err = iavf_init_interrupt_scheme(adapter); 1806 if (err) 1807 goto err_sw_init; 1808 iavf_map_rings_to_vectors(adapter); 1809 if (adapter->vf_res->vf_cap_flags & 1810 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) 1811 adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE; 1812 1813 err = iavf_request_misc_irq(adapter); 1814 if (err) 1815 goto err_sw_init; 1816 1817 netif_carrier_off(netdev); 1818 adapter->link_up = false; 1819 1820 /* set the semaphore to prevent any callbacks after device registration 1821 * up to time when state of driver will be set to __IAVF_DOWN 1822 */ 1823 rtnl_lock(); 1824 if (!adapter->netdev_registered) { 1825 err = register_netdevice(netdev); 1826 if (err) { 1827 rtnl_unlock(); 1828 goto err_register; 1829 } 1830 } 1831 1832 adapter->netdev_registered = true; 1833 1834 netif_tx_stop_all_queues(netdev); 1835 if (CLIENT_ALLOWED(adapter)) { 1836 err = iavf_lan_add_device(adapter); 1837 if (err) 1838 dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n", 1839 err); 1840 } 1841 dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr); 1842 if (netdev->features & NETIF_F_GRO) 1843 dev_info(&pdev->dev, "GRO is enabled\n"); 1844 1845 adapter->state = __IAVF_DOWN; 1846 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 1847 rtnl_unlock(); 1848 1849 iavf_misc_irq_enable(adapter); 1850 wake_up(&adapter->down_waitqueue); 1851 1852 adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL); 1853 adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL); 1854 if (!adapter->rss_key || !adapter->rss_lut) { 1855 err = -ENOMEM; 1856 goto err_mem; 1857 } 1858 if (RSS_AQ(adapter)) 1859 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS; 1860 else 1861 iavf_init_rss(adapter); 1862 1863 return err; 1864 err_mem: 1865 iavf_free_rss(adapter); 1866 err_register: 1867 iavf_free_misc_irq(adapter); 1868 err_sw_init: 1869 iavf_reset_interrupt_capability(adapter); 1870 err_alloc: 1871 kfree(adapter->vf_res); 1872 adapter->vf_res = NULL; 1873 err: 1874 return err; 1875 } 1876 1877 /** 1878 * iavf_watchdog_task - Periodic call-back task 1879 * @work: pointer to work_struct 1880 **/ 1881 static void iavf_watchdog_task(struct work_struct *work) 1882 { 1883 struct iavf_adapter *adapter = container_of(work, 1884 struct iavf_adapter, 1885 watchdog_task.work); 1886 struct iavf_hw *hw = &adapter->hw; 1887 u32 reg_val; 1888 1889 if (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section)) 1890 goto restart_watchdog; 1891 1892 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) 1893 adapter->state = __IAVF_COMM_FAILED; 1894 1895 switch (adapter->state) { 1896 case __IAVF_COMM_FAILED: 1897 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) & 1898 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 1899 if (reg_val == VIRTCHNL_VFR_VFACTIVE || 1900 reg_val == VIRTCHNL_VFR_COMPLETED) { 1901 /* A chance for redemption! */ 1902 dev_err(&adapter->pdev->dev, 1903 "Hardware came out of reset. Attempting reinit.\n"); 1904 adapter->state = __IAVF_STARTUP; 1905 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED; 1906 queue_delayed_work(iavf_wq, &adapter->init_task, 10); 1907 clear_bit(__IAVF_IN_CRITICAL_TASK, 1908 &adapter->crit_section); 1909 /* Don't reschedule the watchdog, since we've restarted 1910 * the init task. When init_task contacts the PF and 1911 * gets everything set up again, it'll restart the 1912 * watchdog for us. Down, boy. Sit. Stay. Woof. 1913 */ 1914 return; 1915 } 1916 adapter->aq_required = 0; 1917 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 1918 clear_bit(__IAVF_IN_CRITICAL_TASK, 1919 &adapter->crit_section); 1920 queue_delayed_work(iavf_wq, 1921 &adapter->watchdog_task, 1922 msecs_to_jiffies(10)); 1923 goto watchdog_done; 1924 case __IAVF_RESETTING: 1925 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 1926 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2); 1927 return; 1928 case __IAVF_DOWN: 1929 case __IAVF_DOWN_PENDING: 1930 case __IAVF_TESTING: 1931 case __IAVF_RUNNING: 1932 if (adapter->current_op) { 1933 if (!iavf_asq_done(hw)) { 1934 dev_dbg(&adapter->pdev->dev, 1935 "Admin queue timeout\n"); 1936 iavf_send_api_ver(adapter); 1937 } 1938 } else { 1939 /* An error will be returned if no commands were 1940 * processed; use this opportunity to update stats 1941 */ 1942 if (iavf_process_aq_command(adapter) && 1943 adapter->state == __IAVF_RUNNING) 1944 iavf_request_stats(adapter); 1945 } 1946 break; 1947 case __IAVF_REMOVE: 1948 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 1949 return; 1950 default: 1951 goto restart_watchdog; 1952 } 1953 1954 /* check for hw reset */ 1955 reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK; 1956 if (!reg_val) { 1957 adapter->state = __IAVF_RESETTING; 1958 adapter->flags |= IAVF_FLAG_RESET_PENDING; 1959 adapter->aq_required = 0; 1960 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 1961 dev_err(&adapter->pdev->dev, "Hardware reset detected\n"); 1962 queue_work(iavf_wq, &adapter->reset_task); 1963 goto watchdog_done; 1964 } 1965 1966 schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5)); 1967 watchdog_done: 1968 if (adapter->state == __IAVF_RUNNING || 1969 adapter->state == __IAVF_COMM_FAILED) 1970 iavf_detect_recover_hung(&adapter->vsi); 1971 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 1972 restart_watchdog: 1973 if (adapter->aq_required) 1974 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 1975 msecs_to_jiffies(20)); 1976 else 1977 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2); 1978 queue_work(iavf_wq, &adapter->adminq_task); 1979 } 1980 1981 static void iavf_disable_vf(struct iavf_adapter *adapter) 1982 { 1983 struct iavf_mac_filter *f, *ftmp; 1984 struct iavf_vlan_filter *fv, *fvtmp; 1985 struct iavf_cloud_filter *cf, *cftmp; 1986 1987 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED; 1988 1989 /* We don't use netif_running() because it may be true prior to 1990 * ndo_open() returning, so we can't assume it means all our open 1991 * tasks have finished, since we're not holding the rtnl_lock here. 1992 */ 1993 if (adapter->state == __IAVF_RUNNING) { 1994 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 1995 netif_carrier_off(adapter->netdev); 1996 netif_tx_disable(adapter->netdev); 1997 adapter->link_up = false; 1998 iavf_napi_disable_all(adapter); 1999 iavf_irq_disable(adapter); 2000 iavf_free_traffic_irqs(adapter); 2001 iavf_free_all_tx_resources(adapter); 2002 iavf_free_all_rx_resources(adapter); 2003 } 2004 2005 spin_lock_bh(&adapter->mac_vlan_list_lock); 2006 2007 /* Delete all of the filters */ 2008 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 2009 list_del(&f->list); 2010 kfree(f); 2011 } 2012 2013 list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) { 2014 list_del(&fv->list); 2015 kfree(fv); 2016 } 2017 2018 spin_unlock_bh(&adapter->mac_vlan_list_lock); 2019 2020 spin_lock_bh(&adapter->cloud_filter_list_lock); 2021 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { 2022 list_del(&cf->list); 2023 kfree(cf); 2024 adapter->num_cloud_filters--; 2025 } 2026 spin_unlock_bh(&adapter->cloud_filter_list_lock); 2027 2028 iavf_free_misc_irq(adapter); 2029 iavf_reset_interrupt_capability(adapter); 2030 iavf_free_queues(adapter); 2031 iavf_free_q_vectors(adapter); 2032 memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE); 2033 iavf_shutdown_adminq(&adapter->hw); 2034 adapter->netdev->flags &= ~IFF_UP; 2035 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 2036 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 2037 adapter->state = __IAVF_DOWN; 2038 wake_up(&adapter->down_waitqueue); 2039 dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n"); 2040 } 2041 2042 /** 2043 * iavf_reset_task - Call-back task to handle hardware reset 2044 * @work: pointer to work_struct 2045 * 2046 * During reset we need to shut down and reinitialize the admin queue 2047 * before we can use it to communicate with the PF again. We also clear 2048 * and reinit the rings because that context is lost as well. 2049 **/ 2050 static void iavf_reset_task(struct work_struct *work) 2051 { 2052 struct iavf_adapter *adapter = container_of(work, 2053 struct iavf_adapter, 2054 reset_task); 2055 struct virtchnl_vf_resource *vfres = adapter->vf_res; 2056 struct net_device *netdev = adapter->netdev; 2057 struct iavf_hw *hw = &adapter->hw; 2058 struct iavf_mac_filter *f, *ftmp; 2059 struct iavf_vlan_filter *vlf; 2060 struct iavf_cloud_filter *cf; 2061 u32 reg_val; 2062 int i = 0, err; 2063 bool running; 2064 2065 /* When device is being removed it doesn't make sense to run the reset 2066 * task, just return in such a case. 2067 */ 2068 if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) 2069 return; 2070 2071 while (test_and_set_bit(__IAVF_IN_CLIENT_TASK, 2072 &adapter->crit_section)) 2073 usleep_range(500, 1000); 2074 if (CLIENT_ENABLED(adapter)) { 2075 adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN | 2076 IAVF_FLAG_CLIENT_NEEDS_CLOSE | 2077 IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS | 2078 IAVF_FLAG_SERVICE_CLIENT_REQUESTED); 2079 cancel_delayed_work_sync(&adapter->client_task); 2080 iavf_notify_client_close(&adapter->vsi, true); 2081 } 2082 iavf_misc_irq_disable(adapter); 2083 if (adapter->flags & IAVF_FLAG_RESET_NEEDED) { 2084 adapter->flags &= ~IAVF_FLAG_RESET_NEEDED; 2085 /* Restart the AQ here. If we have been reset but didn't 2086 * detect it, or if the PF had to reinit, our AQ will be hosed. 2087 */ 2088 iavf_shutdown_adminq(hw); 2089 iavf_init_adminq(hw); 2090 iavf_request_reset(adapter); 2091 } 2092 adapter->flags |= IAVF_FLAG_RESET_PENDING; 2093 2094 /* poll until we see the reset actually happen */ 2095 for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) { 2096 reg_val = rd32(hw, IAVF_VF_ARQLEN1) & 2097 IAVF_VF_ARQLEN1_ARQENABLE_MASK; 2098 if (!reg_val) 2099 break; 2100 usleep_range(5000, 10000); 2101 } 2102 if (i == IAVF_RESET_WAIT_DETECTED_COUNT) { 2103 dev_info(&adapter->pdev->dev, "Never saw reset\n"); 2104 goto continue_reset; /* act like the reset happened */ 2105 } 2106 2107 /* wait until the reset is complete and the PF is responding to us */ 2108 for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) { 2109 /* sleep first to make sure a minimum wait time is met */ 2110 msleep(IAVF_RESET_WAIT_MS); 2111 2112 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) & 2113 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 2114 if (reg_val == VIRTCHNL_VFR_VFACTIVE) 2115 break; 2116 } 2117 2118 pci_set_master(adapter->pdev); 2119 2120 if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) { 2121 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n", 2122 reg_val); 2123 iavf_disable_vf(adapter); 2124 clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); 2125 return; /* Do not attempt to reinit. It's dead, Jim. */ 2126 } 2127 2128 continue_reset: 2129 /* We don't use netif_running() because it may be true prior to 2130 * ndo_open() returning, so we can't assume it means all our open 2131 * tasks have finished, since we're not holding the rtnl_lock here. 2132 */ 2133 running = ((adapter->state == __IAVF_RUNNING) || 2134 (adapter->state == __IAVF_RESETTING)); 2135 2136 if (running) { 2137 netif_carrier_off(netdev); 2138 netif_tx_stop_all_queues(netdev); 2139 adapter->link_up = false; 2140 iavf_napi_disable_all(adapter); 2141 } 2142 iavf_irq_disable(adapter); 2143 2144 adapter->state = __IAVF_RESETTING; 2145 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 2146 2147 /* free the Tx/Rx rings and descriptors, might be better to just 2148 * re-use them sometime in the future 2149 */ 2150 iavf_free_all_rx_resources(adapter); 2151 iavf_free_all_tx_resources(adapter); 2152 2153 adapter->flags |= IAVF_FLAG_QUEUES_DISABLED; 2154 /* kill and reinit the admin queue */ 2155 iavf_shutdown_adminq(hw); 2156 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2157 err = iavf_init_adminq(hw); 2158 if (err) 2159 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n", 2160 err); 2161 adapter->aq_required = 0; 2162 2163 if (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED) { 2164 err = iavf_reinit_interrupt_scheme(adapter); 2165 if (err) 2166 goto reset_err; 2167 } 2168 2169 adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG; 2170 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS; 2171 2172 spin_lock_bh(&adapter->mac_vlan_list_lock); 2173 2174 /* Delete filter for the current MAC address, it could have 2175 * been changed by the PF via administratively set MAC. 2176 * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES. 2177 */ 2178 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 2179 if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) { 2180 list_del(&f->list); 2181 kfree(f); 2182 } 2183 } 2184 /* re-add all MAC filters */ 2185 list_for_each_entry(f, &adapter->mac_filter_list, list) { 2186 f->add = true; 2187 } 2188 /* re-add all VLAN filters */ 2189 list_for_each_entry(vlf, &adapter->vlan_filter_list, list) { 2190 vlf->add = true; 2191 } 2192 2193 spin_unlock_bh(&adapter->mac_vlan_list_lock); 2194 2195 /* check if TCs are running and re-add all cloud filters */ 2196 spin_lock_bh(&adapter->cloud_filter_list_lock); 2197 if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 2198 adapter->num_tc) { 2199 list_for_each_entry(cf, &adapter->cloud_filter_list, list) { 2200 cf->add = true; 2201 } 2202 } 2203 spin_unlock_bh(&adapter->cloud_filter_list_lock); 2204 2205 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; 2206 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER; 2207 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; 2208 iavf_misc_irq_enable(adapter); 2209 2210 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 2); 2211 2212 /* We were running when the reset started, so we need to restore some 2213 * state here. 2214 */ 2215 if (running) { 2216 /* allocate transmit descriptors */ 2217 err = iavf_setup_all_tx_resources(adapter); 2218 if (err) 2219 goto reset_err; 2220 2221 /* allocate receive descriptors */ 2222 err = iavf_setup_all_rx_resources(adapter); 2223 if (err) 2224 goto reset_err; 2225 2226 if (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED) { 2227 err = iavf_request_traffic_irqs(adapter, netdev->name); 2228 if (err) 2229 goto reset_err; 2230 2231 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; 2232 } 2233 2234 iavf_configure(adapter); 2235 2236 iavf_up_complete(adapter); 2237 2238 iavf_irq_enable(adapter, true); 2239 } else { 2240 adapter->state = __IAVF_DOWN; 2241 wake_up(&adapter->down_waitqueue); 2242 } 2243 clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); 2244 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 2245 2246 return; 2247 reset_err: 2248 clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); 2249 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 2250 dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n"); 2251 iavf_close(netdev); 2252 } 2253 2254 /** 2255 * iavf_adminq_task - worker thread to clean the admin queue 2256 * @work: pointer to work_struct containing our data 2257 **/ 2258 static void iavf_adminq_task(struct work_struct *work) 2259 { 2260 struct iavf_adapter *adapter = 2261 container_of(work, struct iavf_adapter, adminq_task); 2262 struct iavf_hw *hw = &adapter->hw; 2263 struct iavf_arq_event_info event; 2264 enum virtchnl_ops v_op; 2265 enum iavf_status ret, v_ret; 2266 u32 val, oldval; 2267 u16 pending; 2268 2269 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) 2270 goto out; 2271 2272 event.buf_len = IAVF_MAX_AQ_BUF_SIZE; 2273 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 2274 if (!event.msg_buf) 2275 goto out; 2276 2277 do { 2278 ret = iavf_clean_arq_element(hw, &event, &pending); 2279 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high); 2280 v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low); 2281 2282 if (ret || !v_op) 2283 break; /* No event to process or error cleaning ARQ */ 2284 2285 iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf, 2286 event.msg_len); 2287 if (pending != 0) 2288 memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE); 2289 } while (pending); 2290 2291 if ((adapter->flags & 2292 (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED)) || 2293 adapter->state == __IAVF_RESETTING) 2294 goto freedom; 2295 2296 /* check for error indications */ 2297 val = rd32(hw, hw->aq.arq.len); 2298 if (val == 0xdeadbeef) /* indicates device in reset */ 2299 goto freedom; 2300 oldval = val; 2301 if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) { 2302 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n"); 2303 val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK; 2304 } 2305 if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) { 2306 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n"); 2307 val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK; 2308 } 2309 if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) { 2310 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n"); 2311 val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK; 2312 } 2313 if (oldval != val) 2314 wr32(hw, hw->aq.arq.len, val); 2315 2316 val = rd32(hw, hw->aq.asq.len); 2317 oldval = val; 2318 if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) { 2319 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n"); 2320 val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK; 2321 } 2322 if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) { 2323 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n"); 2324 val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK; 2325 } 2326 if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) { 2327 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n"); 2328 val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK; 2329 } 2330 if (oldval != val) 2331 wr32(hw, hw->aq.asq.len, val); 2332 2333 freedom: 2334 kfree(event.msg_buf); 2335 out: 2336 /* re-enable Admin queue interrupt cause */ 2337 iavf_misc_irq_enable(adapter); 2338 } 2339 2340 /** 2341 * iavf_client_task - worker thread to perform client work 2342 * @work: pointer to work_struct containing our data 2343 * 2344 * This task handles client interactions. Because client calls can be 2345 * reentrant, we can't handle them in the watchdog. 2346 **/ 2347 static void iavf_client_task(struct work_struct *work) 2348 { 2349 struct iavf_adapter *adapter = 2350 container_of(work, struct iavf_adapter, client_task.work); 2351 2352 /* If we can't get the client bit, just give up. We'll be rescheduled 2353 * later. 2354 */ 2355 2356 if (test_and_set_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section)) 2357 return; 2358 2359 if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) { 2360 iavf_client_subtask(adapter); 2361 adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED; 2362 goto out; 2363 } 2364 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) { 2365 iavf_notify_client_l2_params(&adapter->vsi); 2366 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS; 2367 goto out; 2368 } 2369 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) { 2370 iavf_notify_client_close(&adapter->vsi, false); 2371 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE; 2372 goto out; 2373 } 2374 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) { 2375 iavf_notify_client_open(&adapter->vsi); 2376 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN; 2377 } 2378 out: 2379 clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); 2380 } 2381 2382 /** 2383 * iavf_free_all_tx_resources - Free Tx Resources for All Queues 2384 * @adapter: board private structure 2385 * 2386 * Free all transmit software resources 2387 **/ 2388 void iavf_free_all_tx_resources(struct iavf_adapter *adapter) 2389 { 2390 int i; 2391 2392 if (!adapter->tx_rings) 2393 return; 2394 2395 for (i = 0; i < adapter->num_active_queues; i++) 2396 if (adapter->tx_rings[i].desc) 2397 iavf_free_tx_resources(&adapter->tx_rings[i]); 2398 } 2399 2400 /** 2401 * iavf_setup_all_tx_resources - allocate all queues Tx resources 2402 * @adapter: board private structure 2403 * 2404 * If this function returns with an error, then it's possible one or 2405 * more of the rings is populated (while the rest are not). It is the 2406 * callers duty to clean those orphaned rings. 2407 * 2408 * Return 0 on success, negative on failure 2409 **/ 2410 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter) 2411 { 2412 int i, err = 0; 2413 2414 for (i = 0; i < adapter->num_active_queues; i++) { 2415 adapter->tx_rings[i].count = adapter->tx_desc_count; 2416 err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]); 2417 if (!err) 2418 continue; 2419 dev_err(&adapter->pdev->dev, 2420 "Allocation for Tx Queue %u failed\n", i); 2421 break; 2422 } 2423 2424 return err; 2425 } 2426 2427 /** 2428 * iavf_setup_all_rx_resources - allocate all queues Rx resources 2429 * @adapter: board private structure 2430 * 2431 * If this function returns with an error, then it's possible one or 2432 * more of the rings is populated (while the rest are not). It is the 2433 * callers duty to clean those orphaned rings. 2434 * 2435 * Return 0 on success, negative on failure 2436 **/ 2437 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter) 2438 { 2439 int i, err = 0; 2440 2441 for (i = 0; i < adapter->num_active_queues; i++) { 2442 adapter->rx_rings[i].count = adapter->rx_desc_count; 2443 err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]); 2444 if (!err) 2445 continue; 2446 dev_err(&adapter->pdev->dev, 2447 "Allocation for Rx Queue %u failed\n", i); 2448 break; 2449 } 2450 return err; 2451 } 2452 2453 /** 2454 * iavf_free_all_rx_resources - Free Rx Resources for All Queues 2455 * @adapter: board private structure 2456 * 2457 * Free all receive software resources 2458 **/ 2459 void iavf_free_all_rx_resources(struct iavf_adapter *adapter) 2460 { 2461 int i; 2462 2463 if (!adapter->rx_rings) 2464 return; 2465 2466 for (i = 0; i < adapter->num_active_queues; i++) 2467 if (adapter->rx_rings[i].desc) 2468 iavf_free_rx_resources(&adapter->rx_rings[i]); 2469 } 2470 2471 /** 2472 * iavf_validate_tx_bandwidth - validate the max Tx bandwidth 2473 * @adapter: board private structure 2474 * @max_tx_rate: max Tx bw for a tc 2475 **/ 2476 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter, 2477 u64 max_tx_rate) 2478 { 2479 int speed = 0, ret = 0; 2480 2481 if (ADV_LINK_SUPPORT(adapter)) { 2482 if (adapter->link_speed_mbps < U32_MAX) { 2483 speed = adapter->link_speed_mbps; 2484 goto validate_bw; 2485 } else { 2486 dev_err(&adapter->pdev->dev, "Unknown link speed\n"); 2487 return -EINVAL; 2488 } 2489 } 2490 2491 switch (adapter->link_speed) { 2492 case VIRTCHNL_LINK_SPEED_40GB: 2493 speed = SPEED_40000; 2494 break; 2495 case VIRTCHNL_LINK_SPEED_25GB: 2496 speed = SPEED_25000; 2497 break; 2498 case VIRTCHNL_LINK_SPEED_20GB: 2499 speed = SPEED_20000; 2500 break; 2501 case VIRTCHNL_LINK_SPEED_10GB: 2502 speed = SPEED_10000; 2503 break; 2504 case VIRTCHNL_LINK_SPEED_5GB: 2505 speed = SPEED_5000; 2506 break; 2507 case VIRTCHNL_LINK_SPEED_2_5GB: 2508 speed = SPEED_2500; 2509 break; 2510 case VIRTCHNL_LINK_SPEED_1GB: 2511 speed = SPEED_1000; 2512 break; 2513 case VIRTCHNL_LINK_SPEED_100MB: 2514 speed = SPEED_100; 2515 break; 2516 default: 2517 break; 2518 } 2519 2520 validate_bw: 2521 if (max_tx_rate > speed) { 2522 dev_err(&adapter->pdev->dev, 2523 "Invalid tx rate specified\n"); 2524 ret = -EINVAL; 2525 } 2526 2527 return ret; 2528 } 2529 2530 /** 2531 * iavf_validate_channel_config - validate queue mapping info 2532 * @adapter: board private structure 2533 * @mqprio_qopt: queue parameters 2534 * 2535 * This function validates if the config provided by the user to 2536 * configure queue channels is valid or not. Returns 0 on a valid 2537 * config. 2538 **/ 2539 static int iavf_validate_ch_config(struct iavf_adapter *adapter, 2540 struct tc_mqprio_qopt_offload *mqprio_qopt) 2541 { 2542 u64 total_max_rate = 0; 2543 int i, num_qps = 0; 2544 u64 tx_rate = 0; 2545 int ret = 0; 2546 2547 if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS || 2548 mqprio_qopt->qopt.num_tc < 1) 2549 return -EINVAL; 2550 2551 for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) { 2552 if (!mqprio_qopt->qopt.count[i] || 2553 mqprio_qopt->qopt.offset[i] != num_qps) 2554 return -EINVAL; 2555 if (mqprio_qopt->min_rate[i]) { 2556 dev_err(&adapter->pdev->dev, 2557 "Invalid min tx rate (greater than 0) specified\n"); 2558 return -EINVAL; 2559 } 2560 /*convert to Mbps */ 2561 tx_rate = div_u64(mqprio_qopt->max_rate[i], 2562 IAVF_MBPS_DIVISOR); 2563 total_max_rate += tx_rate; 2564 num_qps += mqprio_qopt->qopt.count[i]; 2565 } 2566 if (num_qps > IAVF_MAX_REQ_QUEUES) 2567 return -EINVAL; 2568 2569 ret = iavf_validate_tx_bandwidth(adapter, total_max_rate); 2570 return ret; 2571 } 2572 2573 /** 2574 * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes 2575 * @adapter: board private structure 2576 **/ 2577 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter) 2578 { 2579 struct iavf_cloud_filter *cf, *cftmp; 2580 2581 spin_lock_bh(&adapter->cloud_filter_list_lock); 2582 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, 2583 list) { 2584 list_del(&cf->list); 2585 kfree(cf); 2586 adapter->num_cloud_filters--; 2587 } 2588 spin_unlock_bh(&adapter->cloud_filter_list_lock); 2589 } 2590 2591 /** 2592 * __iavf_setup_tc - configure multiple traffic classes 2593 * @netdev: network interface device structure 2594 * @type_data: tc offload data 2595 * 2596 * This function processes the config information provided by the 2597 * user to configure traffic classes/queue channels and packages the 2598 * information to request the PF to setup traffic classes. 2599 * 2600 * Returns 0 on success. 2601 **/ 2602 static int __iavf_setup_tc(struct net_device *netdev, void *type_data) 2603 { 2604 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 2605 struct iavf_adapter *adapter = netdev_priv(netdev); 2606 struct virtchnl_vf_resource *vfres = adapter->vf_res; 2607 u8 num_tc = 0, total_qps = 0; 2608 int ret = 0, netdev_tc = 0; 2609 u64 max_tx_rate; 2610 u16 mode; 2611 int i; 2612 2613 num_tc = mqprio_qopt->qopt.num_tc; 2614 mode = mqprio_qopt->mode; 2615 2616 /* delete queue_channel */ 2617 if (!mqprio_qopt->qopt.hw) { 2618 if (adapter->ch_config.state == __IAVF_TC_RUNNING) { 2619 /* reset the tc configuration */ 2620 netdev_reset_tc(netdev); 2621 adapter->num_tc = 0; 2622 netif_tx_stop_all_queues(netdev); 2623 netif_tx_disable(netdev); 2624 iavf_del_all_cloud_filters(adapter); 2625 adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS; 2626 goto exit; 2627 } else { 2628 return -EINVAL; 2629 } 2630 } 2631 2632 /* add queue channel */ 2633 if (mode == TC_MQPRIO_MODE_CHANNEL) { 2634 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) { 2635 dev_err(&adapter->pdev->dev, "ADq not supported\n"); 2636 return -EOPNOTSUPP; 2637 } 2638 if (adapter->ch_config.state != __IAVF_TC_INVALID) { 2639 dev_err(&adapter->pdev->dev, "TC configuration already exists\n"); 2640 return -EINVAL; 2641 } 2642 2643 ret = iavf_validate_ch_config(adapter, mqprio_qopt); 2644 if (ret) 2645 return ret; 2646 /* Return if same TC config is requested */ 2647 if (adapter->num_tc == num_tc) 2648 return 0; 2649 adapter->num_tc = num_tc; 2650 2651 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) { 2652 if (i < num_tc) { 2653 adapter->ch_config.ch_info[i].count = 2654 mqprio_qopt->qopt.count[i]; 2655 adapter->ch_config.ch_info[i].offset = 2656 mqprio_qopt->qopt.offset[i]; 2657 total_qps += mqprio_qopt->qopt.count[i]; 2658 max_tx_rate = mqprio_qopt->max_rate[i]; 2659 /* convert to Mbps */ 2660 max_tx_rate = div_u64(max_tx_rate, 2661 IAVF_MBPS_DIVISOR); 2662 adapter->ch_config.ch_info[i].max_tx_rate = 2663 max_tx_rate; 2664 } else { 2665 adapter->ch_config.ch_info[i].count = 1; 2666 adapter->ch_config.ch_info[i].offset = 0; 2667 } 2668 } 2669 adapter->ch_config.total_qps = total_qps; 2670 netif_tx_stop_all_queues(netdev); 2671 netif_tx_disable(netdev); 2672 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS; 2673 netdev_reset_tc(netdev); 2674 /* Report the tc mapping up the stack */ 2675 netdev_set_num_tc(adapter->netdev, num_tc); 2676 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) { 2677 u16 qcount = mqprio_qopt->qopt.count[i]; 2678 u16 qoffset = mqprio_qopt->qopt.offset[i]; 2679 2680 if (i < num_tc) 2681 netdev_set_tc_queue(netdev, netdev_tc++, qcount, 2682 qoffset); 2683 } 2684 } 2685 exit: 2686 return ret; 2687 } 2688 2689 /** 2690 * iavf_parse_cls_flower - Parse tc flower filters provided by kernel 2691 * @adapter: board private structure 2692 * @f: pointer to struct flow_cls_offload 2693 * @filter: pointer to cloud filter structure 2694 */ 2695 static int iavf_parse_cls_flower(struct iavf_adapter *adapter, 2696 struct flow_cls_offload *f, 2697 struct iavf_cloud_filter *filter) 2698 { 2699 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 2700 struct flow_dissector *dissector = rule->match.dissector; 2701 u16 n_proto_mask = 0; 2702 u16 n_proto_key = 0; 2703 u8 field_flags = 0; 2704 u16 addr_type = 0; 2705 u16 n_proto = 0; 2706 int i = 0; 2707 struct virtchnl_filter *vf = &filter->f; 2708 2709 if (dissector->used_keys & 2710 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 2711 BIT(FLOW_DISSECTOR_KEY_BASIC) | 2712 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 2713 BIT(FLOW_DISSECTOR_KEY_VLAN) | 2714 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 2715 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 2716 BIT(FLOW_DISSECTOR_KEY_PORTS) | 2717 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 2718 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%x\n", 2719 dissector->used_keys); 2720 return -EOPNOTSUPP; 2721 } 2722 2723 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 2724 struct flow_match_enc_keyid match; 2725 2726 flow_rule_match_enc_keyid(rule, &match); 2727 if (match.mask->keyid != 0) 2728 field_flags |= IAVF_CLOUD_FIELD_TEN_ID; 2729 } 2730 2731 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 2732 struct flow_match_basic match; 2733 2734 flow_rule_match_basic(rule, &match); 2735 n_proto_key = ntohs(match.key->n_proto); 2736 n_proto_mask = ntohs(match.mask->n_proto); 2737 2738 if (n_proto_key == ETH_P_ALL) { 2739 n_proto_key = 0; 2740 n_proto_mask = 0; 2741 } 2742 n_proto = n_proto_key & n_proto_mask; 2743 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6) 2744 return -EINVAL; 2745 if (n_proto == ETH_P_IPV6) { 2746 /* specify flow type as TCP IPv6 */ 2747 vf->flow_type = VIRTCHNL_TCP_V6_FLOW; 2748 } 2749 2750 if (match.key->ip_proto != IPPROTO_TCP) { 2751 dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n"); 2752 return -EINVAL; 2753 } 2754 } 2755 2756 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 2757 struct flow_match_eth_addrs match; 2758 2759 flow_rule_match_eth_addrs(rule, &match); 2760 2761 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 2762 if (!is_zero_ether_addr(match.mask->dst)) { 2763 if (is_broadcast_ether_addr(match.mask->dst)) { 2764 field_flags |= IAVF_CLOUD_FIELD_OMAC; 2765 } else { 2766 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n", 2767 match.mask->dst); 2768 return IAVF_ERR_CONFIG; 2769 } 2770 } 2771 2772 if (!is_zero_ether_addr(match.mask->src)) { 2773 if (is_broadcast_ether_addr(match.mask->src)) { 2774 field_flags |= IAVF_CLOUD_FIELD_IMAC; 2775 } else { 2776 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n", 2777 match.mask->src); 2778 return IAVF_ERR_CONFIG; 2779 } 2780 } 2781 2782 if (!is_zero_ether_addr(match.key->dst)) 2783 if (is_valid_ether_addr(match.key->dst) || 2784 is_multicast_ether_addr(match.key->dst)) { 2785 /* set the mask if a valid dst_mac address */ 2786 for (i = 0; i < ETH_ALEN; i++) 2787 vf->mask.tcp_spec.dst_mac[i] |= 0xff; 2788 ether_addr_copy(vf->data.tcp_spec.dst_mac, 2789 match.key->dst); 2790 } 2791 2792 if (!is_zero_ether_addr(match.key->src)) 2793 if (is_valid_ether_addr(match.key->src) || 2794 is_multicast_ether_addr(match.key->src)) { 2795 /* set the mask if a valid dst_mac address */ 2796 for (i = 0; i < ETH_ALEN; i++) 2797 vf->mask.tcp_spec.src_mac[i] |= 0xff; 2798 ether_addr_copy(vf->data.tcp_spec.src_mac, 2799 match.key->src); 2800 } 2801 } 2802 2803 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 2804 struct flow_match_vlan match; 2805 2806 flow_rule_match_vlan(rule, &match); 2807 if (match.mask->vlan_id) { 2808 if (match.mask->vlan_id == VLAN_VID_MASK) { 2809 field_flags |= IAVF_CLOUD_FIELD_IVLAN; 2810 } else { 2811 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n", 2812 match.mask->vlan_id); 2813 return IAVF_ERR_CONFIG; 2814 } 2815 } 2816 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff); 2817 vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id); 2818 } 2819 2820 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 2821 struct flow_match_control match; 2822 2823 flow_rule_match_control(rule, &match); 2824 addr_type = match.key->addr_type; 2825 } 2826 2827 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 2828 struct flow_match_ipv4_addrs match; 2829 2830 flow_rule_match_ipv4_addrs(rule, &match); 2831 if (match.mask->dst) { 2832 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 2833 field_flags |= IAVF_CLOUD_FIELD_IIP; 2834 } else { 2835 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n", 2836 be32_to_cpu(match.mask->dst)); 2837 return IAVF_ERR_CONFIG; 2838 } 2839 } 2840 2841 if (match.mask->src) { 2842 if (match.mask->src == cpu_to_be32(0xffffffff)) { 2843 field_flags |= IAVF_CLOUD_FIELD_IIP; 2844 } else { 2845 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n", 2846 be32_to_cpu(match.mask->dst)); 2847 return IAVF_ERR_CONFIG; 2848 } 2849 } 2850 2851 if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) { 2852 dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n"); 2853 return IAVF_ERR_CONFIG; 2854 } 2855 if (match.key->dst) { 2856 vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff); 2857 vf->data.tcp_spec.dst_ip[0] = match.key->dst; 2858 } 2859 if (match.key->src) { 2860 vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff); 2861 vf->data.tcp_spec.src_ip[0] = match.key->src; 2862 } 2863 } 2864 2865 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 2866 struct flow_match_ipv6_addrs match; 2867 2868 flow_rule_match_ipv6_addrs(rule, &match); 2869 2870 /* validate mask, make sure it is not IPV6_ADDR_ANY */ 2871 if (ipv6_addr_any(&match.mask->dst)) { 2872 dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n", 2873 IPV6_ADDR_ANY); 2874 return IAVF_ERR_CONFIG; 2875 } 2876 2877 /* src and dest IPv6 address should not be LOOPBACK 2878 * (0:0:0:0:0:0:0:1) which can be represented as ::1 2879 */ 2880 if (ipv6_addr_loopback(&match.key->dst) || 2881 ipv6_addr_loopback(&match.key->src)) { 2882 dev_err(&adapter->pdev->dev, 2883 "ipv6 addr should not be loopback\n"); 2884 return IAVF_ERR_CONFIG; 2885 } 2886 if (!ipv6_addr_any(&match.mask->dst) || 2887 !ipv6_addr_any(&match.mask->src)) 2888 field_flags |= IAVF_CLOUD_FIELD_IIP; 2889 2890 for (i = 0; i < 4; i++) 2891 vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff); 2892 memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32, 2893 sizeof(vf->data.tcp_spec.dst_ip)); 2894 for (i = 0; i < 4; i++) 2895 vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff); 2896 memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32, 2897 sizeof(vf->data.tcp_spec.src_ip)); 2898 } 2899 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 2900 struct flow_match_ports match; 2901 2902 flow_rule_match_ports(rule, &match); 2903 if (match.mask->src) { 2904 if (match.mask->src == cpu_to_be16(0xffff)) { 2905 field_flags |= IAVF_CLOUD_FIELD_IIP; 2906 } else { 2907 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n", 2908 be16_to_cpu(match.mask->src)); 2909 return IAVF_ERR_CONFIG; 2910 } 2911 } 2912 2913 if (match.mask->dst) { 2914 if (match.mask->dst == cpu_to_be16(0xffff)) { 2915 field_flags |= IAVF_CLOUD_FIELD_IIP; 2916 } else { 2917 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n", 2918 be16_to_cpu(match.mask->dst)); 2919 return IAVF_ERR_CONFIG; 2920 } 2921 } 2922 if (match.key->dst) { 2923 vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff); 2924 vf->data.tcp_spec.dst_port = match.key->dst; 2925 } 2926 2927 if (match.key->src) { 2928 vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff); 2929 vf->data.tcp_spec.src_port = match.key->src; 2930 } 2931 } 2932 vf->field_flags = field_flags; 2933 2934 return 0; 2935 } 2936 2937 /** 2938 * iavf_handle_tclass - Forward to a traffic class on the device 2939 * @adapter: board private structure 2940 * @tc: traffic class index on the device 2941 * @filter: pointer to cloud filter structure 2942 */ 2943 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc, 2944 struct iavf_cloud_filter *filter) 2945 { 2946 if (tc == 0) 2947 return 0; 2948 if (tc < adapter->num_tc) { 2949 if (!filter->f.data.tcp_spec.dst_port) { 2950 dev_err(&adapter->pdev->dev, 2951 "Specify destination port to redirect to traffic class other than TC0\n"); 2952 return -EINVAL; 2953 } 2954 } 2955 /* redirect to a traffic class on the same device */ 2956 filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT; 2957 filter->f.action_meta = tc; 2958 return 0; 2959 } 2960 2961 /** 2962 * iavf_configure_clsflower - Add tc flower filters 2963 * @adapter: board private structure 2964 * @cls_flower: Pointer to struct flow_cls_offload 2965 */ 2966 static int iavf_configure_clsflower(struct iavf_adapter *adapter, 2967 struct flow_cls_offload *cls_flower) 2968 { 2969 int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid); 2970 struct iavf_cloud_filter *filter = NULL; 2971 int err = -EINVAL, count = 50; 2972 2973 if (tc < 0) { 2974 dev_err(&adapter->pdev->dev, "Invalid traffic class\n"); 2975 return -EINVAL; 2976 } 2977 2978 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 2979 if (!filter) 2980 return -ENOMEM; 2981 2982 while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, 2983 &adapter->crit_section)) { 2984 if (--count == 0) 2985 goto err; 2986 udelay(1); 2987 } 2988 2989 filter->cookie = cls_flower->cookie; 2990 2991 /* set the mask to all zeroes to begin with */ 2992 memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec)); 2993 /* start out with flow type and eth type IPv4 to begin with */ 2994 filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW; 2995 err = iavf_parse_cls_flower(adapter, cls_flower, filter); 2996 if (err < 0) 2997 goto err; 2998 2999 err = iavf_handle_tclass(adapter, tc, filter); 3000 if (err < 0) 3001 goto err; 3002 3003 /* add filter to the list */ 3004 spin_lock_bh(&adapter->cloud_filter_list_lock); 3005 list_add_tail(&filter->list, &adapter->cloud_filter_list); 3006 adapter->num_cloud_filters++; 3007 filter->add = true; 3008 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; 3009 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3010 err: 3011 if (err) 3012 kfree(filter); 3013 3014 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 3015 return err; 3016 } 3017 3018 /* iavf_find_cf - Find the cloud filter in the list 3019 * @adapter: Board private structure 3020 * @cookie: filter specific cookie 3021 * 3022 * Returns ptr to the filter object or NULL. Must be called while holding the 3023 * cloud_filter_list_lock. 3024 */ 3025 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter, 3026 unsigned long *cookie) 3027 { 3028 struct iavf_cloud_filter *filter = NULL; 3029 3030 if (!cookie) 3031 return NULL; 3032 3033 list_for_each_entry(filter, &adapter->cloud_filter_list, list) { 3034 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 3035 return filter; 3036 } 3037 return NULL; 3038 } 3039 3040 /** 3041 * iavf_delete_clsflower - Remove tc flower filters 3042 * @adapter: board private structure 3043 * @cls_flower: Pointer to struct flow_cls_offload 3044 */ 3045 static int iavf_delete_clsflower(struct iavf_adapter *adapter, 3046 struct flow_cls_offload *cls_flower) 3047 { 3048 struct iavf_cloud_filter *filter = NULL; 3049 int err = 0; 3050 3051 spin_lock_bh(&adapter->cloud_filter_list_lock); 3052 filter = iavf_find_cf(adapter, &cls_flower->cookie); 3053 if (filter) { 3054 filter->del = true; 3055 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; 3056 } else { 3057 err = -EINVAL; 3058 } 3059 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3060 3061 return err; 3062 } 3063 3064 /** 3065 * iavf_setup_tc_cls_flower - flower classifier offloads 3066 * @adapter: board private structure 3067 * @cls_flower: pointer to flow_cls_offload struct with flow info 3068 */ 3069 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter, 3070 struct flow_cls_offload *cls_flower) 3071 { 3072 switch (cls_flower->command) { 3073 case FLOW_CLS_REPLACE: 3074 return iavf_configure_clsflower(adapter, cls_flower); 3075 case FLOW_CLS_DESTROY: 3076 return iavf_delete_clsflower(adapter, cls_flower); 3077 case FLOW_CLS_STATS: 3078 return -EOPNOTSUPP; 3079 default: 3080 return -EOPNOTSUPP; 3081 } 3082 } 3083 3084 /** 3085 * iavf_setup_tc_block_cb - block callback for tc 3086 * @type: type of offload 3087 * @type_data: offload data 3088 * @cb_priv: 3089 * 3090 * This function is the block callback for traffic classes 3091 **/ 3092 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 3093 void *cb_priv) 3094 { 3095 struct iavf_adapter *adapter = cb_priv; 3096 3097 if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data)) 3098 return -EOPNOTSUPP; 3099 3100 switch (type) { 3101 case TC_SETUP_CLSFLOWER: 3102 return iavf_setup_tc_cls_flower(cb_priv, type_data); 3103 default: 3104 return -EOPNOTSUPP; 3105 } 3106 } 3107 3108 static LIST_HEAD(iavf_block_cb_list); 3109 3110 /** 3111 * iavf_setup_tc - configure multiple traffic classes 3112 * @netdev: network interface device structure 3113 * @type: type of offload 3114 * @type_data: tc offload data 3115 * 3116 * This function is the callback to ndo_setup_tc in the 3117 * netdev_ops. 3118 * 3119 * Returns 0 on success 3120 **/ 3121 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type, 3122 void *type_data) 3123 { 3124 struct iavf_adapter *adapter = netdev_priv(netdev); 3125 3126 switch (type) { 3127 case TC_SETUP_QDISC_MQPRIO: 3128 return __iavf_setup_tc(netdev, type_data); 3129 case TC_SETUP_BLOCK: 3130 return flow_block_cb_setup_simple(type_data, 3131 &iavf_block_cb_list, 3132 iavf_setup_tc_block_cb, 3133 adapter, adapter, true); 3134 default: 3135 return -EOPNOTSUPP; 3136 } 3137 } 3138 3139 /** 3140 * iavf_open - Called when a network interface is made active 3141 * @netdev: network interface device structure 3142 * 3143 * Returns 0 on success, negative value on failure 3144 * 3145 * The open entry point is called when a network interface is made 3146 * active by the system (IFF_UP). At this point all resources needed 3147 * for transmit and receive operations are allocated, the interrupt 3148 * handler is registered with the OS, the watchdog is started, 3149 * and the stack is notified that the interface is ready. 3150 **/ 3151 static int iavf_open(struct net_device *netdev) 3152 { 3153 struct iavf_adapter *adapter = netdev_priv(netdev); 3154 int err; 3155 3156 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) { 3157 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n"); 3158 return -EIO; 3159 } 3160 3161 while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, 3162 &adapter->crit_section)) 3163 usleep_range(500, 1000); 3164 3165 if (adapter->state != __IAVF_DOWN) { 3166 err = -EBUSY; 3167 goto err_unlock; 3168 } 3169 3170 /* allocate transmit descriptors */ 3171 err = iavf_setup_all_tx_resources(adapter); 3172 if (err) 3173 goto err_setup_tx; 3174 3175 /* allocate receive descriptors */ 3176 err = iavf_setup_all_rx_resources(adapter); 3177 if (err) 3178 goto err_setup_rx; 3179 3180 /* clear any pending interrupts, may auto mask */ 3181 err = iavf_request_traffic_irqs(adapter, netdev->name); 3182 if (err) 3183 goto err_req_irq; 3184 3185 spin_lock_bh(&adapter->mac_vlan_list_lock); 3186 3187 iavf_add_filter(adapter, adapter->hw.mac.addr); 3188 3189 spin_unlock_bh(&adapter->mac_vlan_list_lock); 3190 3191 iavf_configure(adapter); 3192 3193 iavf_up_complete(adapter); 3194 3195 iavf_irq_enable(adapter, true); 3196 3197 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 3198 3199 return 0; 3200 3201 err_req_irq: 3202 iavf_down(adapter); 3203 iavf_free_traffic_irqs(adapter); 3204 err_setup_rx: 3205 iavf_free_all_rx_resources(adapter); 3206 err_setup_tx: 3207 iavf_free_all_tx_resources(adapter); 3208 err_unlock: 3209 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 3210 3211 return err; 3212 } 3213 3214 /** 3215 * iavf_close - Disables a network interface 3216 * @netdev: network interface device structure 3217 * 3218 * Returns 0, this is not allowed to fail 3219 * 3220 * The close entry point is called when an interface is de-activated 3221 * by the OS. The hardware is still under the drivers control, but 3222 * needs to be disabled. All IRQs except vector 0 (reserved for admin queue) 3223 * are freed, along with all transmit and receive resources. 3224 **/ 3225 static int iavf_close(struct net_device *netdev) 3226 { 3227 struct iavf_adapter *adapter = netdev_priv(netdev); 3228 int status; 3229 3230 if (adapter->state <= __IAVF_DOWN_PENDING) 3231 return 0; 3232 3233 while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, 3234 &adapter->crit_section)) 3235 usleep_range(500, 1000); 3236 3237 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 3238 if (CLIENT_ENABLED(adapter)) 3239 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE; 3240 3241 iavf_down(adapter); 3242 adapter->state = __IAVF_DOWN_PENDING; 3243 iavf_free_traffic_irqs(adapter); 3244 3245 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 3246 3247 /* We explicitly don't free resources here because the hardware is 3248 * still active and can DMA into memory. Resources are cleared in 3249 * iavf_virtchnl_completion() after we get confirmation from the PF 3250 * driver that the rings have been stopped. 3251 * 3252 * Also, we wait for state to transition to __IAVF_DOWN before 3253 * returning. State change occurs in iavf_virtchnl_completion() after 3254 * VF resources are released (which occurs after PF driver processes and 3255 * responds to admin queue commands). 3256 */ 3257 3258 status = wait_event_timeout(adapter->down_waitqueue, 3259 adapter->state == __IAVF_DOWN, 3260 msecs_to_jiffies(500)); 3261 if (!status) 3262 netdev_warn(netdev, "Device resources not yet released\n"); 3263 return 0; 3264 } 3265 3266 /** 3267 * iavf_change_mtu - Change the Maximum Transfer Unit 3268 * @netdev: network interface device structure 3269 * @new_mtu: new value for maximum frame size 3270 * 3271 * Returns 0 on success, negative on failure 3272 **/ 3273 static int iavf_change_mtu(struct net_device *netdev, int new_mtu) 3274 { 3275 struct iavf_adapter *adapter = netdev_priv(netdev); 3276 3277 netdev->mtu = new_mtu; 3278 if (CLIENT_ENABLED(adapter)) { 3279 iavf_notify_client_l2_params(&adapter->vsi); 3280 adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED; 3281 } 3282 adapter->flags |= IAVF_FLAG_RESET_NEEDED; 3283 queue_work(iavf_wq, &adapter->reset_task); 3284 3285 return 0; 3286 } 3287 3288 /** 3289 * iavf_set_features - set the netdev feature flags 3290 * @netdev: ptr to the netdev being adjusted 3291 * @features: the feature set that the stack is suggesting 3292 * Note: expects to be called while under rtnl_lock() 3293 **/ 3294 static int iavf_set_features(struct net_device *netdev, 3295 netdev_features_t features) 3296 { 3297 struct iavf_adapter *adapter = netdev_priv(netdev); 3298 3299 /* Don't allow changing VLAN_RX flag when adapter is not capable 3300 * of VLAN offload 3301 */ 3302 if (!VLAN_ALLOWED(adapter)) { 3303 if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX) 3304 return -EINVAL; 3305 } else if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX) { 3306 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3307 adapter->aq_required |= 3308 IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING; 3309 else 3310 adapter->aq_required |= 3311 IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING; 3312 } 3313 3314 return 0; 3315 } 3316 3317 /** 3318 * iavf_features_check - Validate encapsulated packet conforms to limits 3319 * @skb: skb buff 3320 * @dev: This physical port's netdev 3321 * @features: Offload features that the stack believes apply 3322 **/ 3323 static netdev_features_t iavf_features_check(struct sk_buff *skb, 3324 struct net_device *dev, 3325 netdev_features_t features) 3326 { 3327 size_t len; 3328 3329 /* No point in doing any of this if neither checksum nor GSO are 3330 * being requested for this frame. We can rule out both by just 3331 * checking for CHECKSUM_PARTIAL 3332 */ 3333 if (skb->ip_summed != CHECKSUM_PARTIAL) 3334 return features; 3335 3336 /* We cannot support GSO if the MSS is going to be less than 3337 * 64 bytes. If it is then we need to drop support for GSO. 3338 */ 3339 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 3340 features &= ~NETIF_F_GSO_MASK; 3341 3342 /* MACLEN can support at most 63 words */ 3343 len = skb_network_header(skb) - skb->data; 3344 if (len & ~(63 * 2)) 3345 goto out_err; 3346 3347 /* IPLEN and EIPLEN can support at most 127 dwords */ 3348 len = skb_transport_header(skb) - skb_network_header(skb); 3349 if (len & ~(127 * 4)) 3350 goto out_err; 3351 3352 if (skb->encapsulation) { 3353 /* L4TUNLEN can support 127 words */ 3354 len = skb_inner_network_header(skb) - skb_transport_header(skb); 3355 if (len & ~(127 * 2)) 3356 goto out_err; 3357 3358 /* IPLEN can support at most 127 dwords */ 3359 len = skb_inner_transport_header(skb) - 3360 skb_inner_network_header(skb); 3361 if (len & ~(127 * 4)) 3362 goto out_err; 3363 } 3364 3365 /* No need to validate L4LEN as TCP is the only protocol with a 3366 * a flexible value and we support all possible values supported 3367 * by TCP, which is at most 15 dwords 3368 */ 3369 3370 return features; 3371 out_err: 3372 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 3373 } 3374 3375 /** 3376 * iavf_fix_features - fix up the netdev feature bits 3377 * @netdev: our net device 3378 * @features: desired feature bits 3379 * 3380 * Returns fixed-up features bits 3381 **/ 3382 static netdev_features_t iavf_fix_features(struct net_device *netdev, 3383 netdev_features_t features) 3384 { 3385 struct iavf_adapter *adapter = netdev_priv(netdev); 3386 3387 if (!(adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 3388 features &= ~(NETIF_F_HW_VLAN_CTAG_TX | 3389 NETIF_F_HW_VLAN_CTAG_RX | 3390 NETIF_F_HW_VLAN_CTAG_FILTER); 3391 3392 return features; 3393 } 3394 3395 static const struct net_device_ops iavf_netdev_ops = { 3396 .ndo_open = iavf_open, 3397 .ndo_stop = iavf_close, 3398 .ndo_start_xmit = iavf_xmit_frame, 3399 .ndo_set_rx_mode = iavf_set_rx_mode, 3400 .ndo_validate_addr = eth_validate_addr, 3401 .ndo_set_mac_address = iavf_set_mac, 3402 .ndo_change_mtu = iavf_change_mtu, 3403 .ndo_tx_timeout = iavf_tx_timeout, 3404 .ndo_vlan_rx_add_vid = iavf_vlan_rx_add_vid, 3405 .ndo_vlan_rx_kill_vid = iavf_vlan_rx_kill_vid, 3406 .ndo_features_check = iavf_features_check, 3407 .ndo_fix_features = iavf_fix_features, 3408 .ndo_set_features = iavf_set_features, 3409 .ndo_setup_tc = iavf_setup_tc, 3410 }; 3411 3412 /** 3413 * iavf_check_reset_complete - check that VF reset is complete 3414 * @hw: pointer to hw struct 3415 * 3416 * Returns 0 if device is ready to use, or -EBUSY if it's in reset. 3417 **/ 3418 static int iavf_check_reset_complete(struct iavf_hw *hw) 3419 { 3420 u32 rstat; 3421 int i; 3422 3423 for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) { 3424 rstat = rd32(hw, IAVF_VFGEN_RSTAT) & 3425 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 3426 if ((rstat == VIRTCHNL_VFR_VFACTIVE) || 3427 (rstat == VIRTCHNL_VFR_COMPLETED)) 3428 return 0; 3429 usleep_range(10, 20); 3430 } 3431 return -EBUSY; 3432 } 3433 3434 /** 3435 * iavf_process_config - Process the config information we got from the PF 3436 * @adapter: board private structure 3437 * 3438 * Verify that we have a valid config struct, and set up our netdev features 3439 * and our VSI struct. 3440 **/ 3441 int iavf_process_config(struct iavf_adapter *adapter) 3442 { 3443 struct virtchnl_vf_resource *vfres = adapter->vf_res; 3444 int i, num_req_queues = adapter->num_req_queues; 3445 struct net_device *netdev = adapter->netdev; 3446 struct iavf_vsi *vsi = &adapter->vsi; 3447 netdev_features_t hw_enc_features; 3448 netdev_features_t hw_features; 3449 3450 /* got VF config message back from PF, now we can parse it */ 3451 for (i = 0; i < vfres->num_vsis; i++) { 3452 if (vfres->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) 3453 adapter->vsi_res = &vfres->vsi_res[i]; 3454 } 3455 if (!adapter->vsi_res) { 3456 dev_err(&adapter->pdev->dev, "No LAN VSI found\n"); 3457 return -ENODEV; 3458 } 3459 3460 if (num_req_queues && 3461 num_req_queues > adapter->vsi_res->num_queue_pairs) { 3462 /* Problem. The PF gave us fewer queues than what we had 3463 * negotiated in our request. Need a reset to see if we can't 3464 * get back to a working state. 3465 */ 3466 dev_err(&adapter->pdev->dev, 3467 "Requested %d queues, but PF only gave us %d.\n", 3468 num_req_queues, 3469 adapter->vsi_res->num_queue_pairs); 3470 adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED; 3471 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs; 3472 iavf_schedule_reset(adapter); 3473 return -ENODEV; 3474 } 3475 adapter->num_req_queues = 0; 3476 3477 hw_enc_features = NETIF_F_SG | 3478 NETIF_F_IP_CSUM | 3479 NETIF_F_IPV6_CSUM | 3480 NETIF_F_HIGHDMA | 3481 NETIF_F_SOFT_FEATURES | 3482 NETIF_F_TSO | 3483 NETIF_F_TSO_ECN | 3484 NETIF_F_TSO6 | 3485 NETIF_F_SCTP_CRC | 3486 NETIF_F_RXHASH | 3487 NETIF_F_RXCSUM | 3488 0; 3489 3490 /* advertise to stack only if offloads for encapsulated packets is 3491 * supported 3492 */ 3493 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) { 3494 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 3495 NETIF_F_GSO_GRE | 3496 NETIF_F_GSO_GRE_CSUM | 3497 NETIF_F_GSO_IPXIP4 | 3498 NETIF_F_GSO_IPXIP6 | 3499 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3500 NETIF_F_GSO_PARTIAL | 3501 0; 3502 3503 if (!(vfres->vf_cap_flags & 3504 VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM)) 3505 netdev->gso_partial_features |= 3506 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3507 3508 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 3509 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 3510 netdev->hw_enc_features |= hw_enc_features; 3511 } 3512 /* record features VLANs can make use of */ 3513 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 3514 3515 /* Write features and hw_features separately to avoid polluting 3516 * with, or dropping, features that are set when we registered. 3517 */ 3518 hw_features = hw_enc_features; 3519 3520 /* Enable VLAN features if supported */ 3521 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) 3522 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX | 3523 NETIF_F_HW_VLAN_CTAG_RX); 3524 /* Enable cloud filter if ADQ is supported */ 3525 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) 3526 hw_features |= NETIF_F_HW_TC; 3527 3528 netdev->hw_features |= hw_features; 3529 3530 netdev->features |= hw_features; 3531 3532 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) 3533 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 3534 3535 netdev->priv_flags |= IFF_UNICAST_FLT; 3536 3537 /* Do not turn on offloads when they are requested to be turned off. 3538 * TSO needs minimum 576 bytes to work correctly. 3539 */ 3540 if (netdev->wanted_features) { 3541 if (!(netdev->wanted_features & NETIF_F_TSO) || 3542 netdev->mtu < 576) 3543 netdev->features &= ~NETIF_F_TSO; 3544 if (!(netdev->wanted_features & NETIF_F_TSO6) || 3545 netdev->mtu < 576) 3546 netdev->features &= ~NETIF_F_TSO6; 3547 if (!(netdev->wanted_features & NETIF_F_TSO_ECN)) 3548 netdev->features &= ~NETIF_F_TSO_ECN; 3549 if (!(netdev->wanted_features & NETIF_F_GRO)) 3550 netdev->features &= ~NETIF_F_GRO; 3551 if (!(netdev->wanted_features & NETIF_F_GSO)) 3552 netdev->features &= ~NETIF_F_GSO; 3553 } 3554 3555 adapter->vsi.id = adapter->vsi_res->vsi_id; 3556 3557 adapter->vsi.back = adapter; 3558 adapter->vsi.base_vector = 1; 3559 adapter->vsi.work_limit = IAVF_DEFAULT_IRQ_WORK; 3560 vsi->netdev = adapter->netdev; 3561 vsi->qs_handle = adapter->vsi_res->qset_handle; 3562 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 3563 adapter->rss_key_size = vfres->rss_key_size; 3564 adapter->rss_lut_size = vfres->rss_lut_size; 3565 } else { 3566 adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE; 3567 adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE; 3568 } 3569 3570 return 0; 3571 } 3572 3573 /** 3574 * iavf_init_task - worker thread to perform delayed initialization 3575 * @work: pointer to work_struct containing our data 3576 * 3577 * This task completes the work that was begun in probe. Due to the nature 3578 * of VF-PF communications, we may need to wait tens of milliseconds to get 3579 * responses back from the PF. Rather than busy-wait in probe and bog down the 3580 * whole system, we'll do it in a task so we can sleep. 3581 * This task only runs during driver init. Once we've established 3582 * communications with the PF driver and set up our netdev, the watchdog 3583 * takes over. 3584 **/ 3585 static void iavf_init_task(struct work_struct *work) 3586 { 3587 struct iavf_adapter *adapter = container_of(work, 3588 struct iavf_adapter, 3589 init_task.work); 3590 struct iavf_hw *hw = &adapter->hw; 3591 3592 switch (adapter->state) { 3593 case __IAVF_STARTUP: 3594 if (iavf_startup(adapter) < 0) 3595 goto init_failed; 3596 break; 3597 case __IAVF_INIT_VERSION_CHECK: 3598 if (iavf_init_version_check(adapter) < 0) 3599 goto init_failed; 3600 break; 3601 case __IAVF_INIT_GET_RESOURCES: 3602 if (iavf_init_get_resources(adapter) < 0) 3603 goto init_failed; 3604 return; 3605 default: 3606 goto init_failed; 3607 } 3608 3609 queue_delayed_work(iavf_wq, &adapter->init_task, 3610 msecs_to_jiffies(30)); 3611 return; 3612 init_failed: 3613 if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) { 3614 dev_err(&adapter->pdev->dev, 3615 "Failed to communicate with PF; waiting before retry\n"); 3616 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED; 3617 iavf_shutdown_adminq(hw); 3618 adapter->state = __IAVF_STARTUP; 3619 queue_delayed_work(iavf_wq, &adapter->init_task, HZ * 5); 3620 return; 3621 } 3622 queue_delayed_work(iavf_wq, &adapter->init_task, HZ); 3623 } 3624 3625 /** 3626 * iavf_shutdown - Shutdown the device in preparation for a reboot 3627 * @pdev: pci device structure 3628 **/ 3629 static void iavf_shutdown(struct pci_dev *pdev) 3630 { 3631 struct net_device *netdev = pci_get_drvdata(pdev); 3632 struct iavf_adapter *adapter = netdev_priv(netdev); 3633 3634 netif_device_detach(netdev); 3635 3636 if (netif_running(netdev)) 3637 iavf_close(netdev); 3638 3639 /* Prevent the watchdog from running. */ 3640 adapter->state = __IAVF_REMOVE; 3641 adapter->aq_required = 0; 3642 3643 #ifdef CONFIG_PM 3644 pci_save_state(pdev); 3645 3646 #endif 3647 pci_disable_device(pdev); 3648 } 3649 3650 /** 3651 * iavf_probe - Device Initialization Routine 3652 * @pdev: PCI device information struct 3653 * @ent: entry in iavf_pci_tbl 3654 * 3655 * Returns 0 on success, negative on failure 3656 * 3657 * iavf_probe initializes an adapter identified by a pci_dev structure. 3658 * The OS initialization, configuring of the adapter private structure, 3659 * and a hardware reset occur. 3660 **/ 3661 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3662 { 3663 struct net_device *netdev; 3664 struct iavf_adapter *adapter = NULL; 3665 struct iavf_hw *hw = NULL; 3666 int err; 3667 3668 err = pci_enable_device(pdev); 3669 if (err) 3670 return err; 3671 3672 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3673 if (err) { 3674 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 3675 if (err) { 3676 dev_err(&pdev->dev, 3677 "DMA configuration failed: 0x%x\n", err); 3678 goto err_dma; 3679 } 3680 } 3681 3682 err = pci_request_regions(pdev, iavf_driver_name); 3683 if (err) { 3684 dev_err(&pdev->dev, 3685 "pci_request_regions failed 0x%x\n", err); 3686 goto err_pci_reg; 3687 } 3688 3689 pci_enable_pcie_error_reporting(pdev); 3690 3691 pci_set_master(pdev); 3692 3693 netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter), 3694 IAVF_MAX_REQ_QUEUES); 3695 if (!netdev) { 3696 err = -ENOMEM; 3697 goto err_alloc_etherdev; 3698 } 3699 3700 SET_NETDEV_DEV(netdev, &pdev->dev); 3701 3702 pci_set_drvdata(pdev, netdev); 3703 adapter = netdev_priv(netdev); 3704 3705 adapter->netdev = netdev; 3706 adapter->pdev = pdev; 3707 3708 hw = &adapter->hw; 3709 hw->back = adapter; 3710 3711 adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1; 3712 adapter->state = __IAVF_STARTUP; 3713 3714 /* Call save state here because it relies on the adapter struct. */ 3715 pci_save_state(pdev); 3716 3717 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3718 pci_resource_len(pdev, 0)); 3719 if (!hw->hw_addr) { 3720 err = -EIO; 3721 goto err_ioremap; 3722 } 3723 hw->vendor_id = pdev->vendor; 3724 hw->device_id = pdev->device; 3725 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 3726 hw->subsystem_vendor_id = pdev->subsystem_vendor; 3727 hw->subsystem_device_id = pdev->subsystem_device; 3728 hw->bus.device = PCI_SLOT(pdev->devfn); 3729 hw->bus.func = PCI_FUNC(pdev->devfn); 3730 hw->bus.bus_id = pdev->bus->number; 3731 3732 /* set up the locks for the AQ, do this only once in probe 3733 * and destroy them only once in remove 3734 */ 3735 mutex_init(&hw->aq.asq_mutex); 3736 mutex_init(&hw->aq.arq_mutex); 3737 3738 spin_lock_init(&adapter->mac_vlan_list_lock); 3739 spin_lock_init(&adapter->cloud_filter_list_lock); 3740 3741 INIT_LIST_HEAD(&adapter->mac_filter_list); 3742 INIT_LIST_HEAD(&adapter->vlan_filter_list); 3743 INIT_LIST_HEAD(&adapter->cloud_filter_list); 3744 3745 INIT_WORK(&adapter->reset_task, iavf_reset_task); 3746 INIT_WORK(&adapter->adminq_task, iavf_adminq_task); 3747 INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task); 3748 INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task); 3749 INIT_DELAYED_WORK(&adapter->init_task, iavf_init_task); 3750 queue_delayed_work(iavf_wq, &adapter->init_task, 3751 msecs_to_jiffies(5 * (pdev->devfn & 0x07))); 3752 3753 /* Setup the wait queue for indicating transition to down status */ 3754 init_waitqueue_head(&adapter->down_waitqueue); 3755 3756 return 0; 3757 3758 err_ioremap: 3759 free_netdev(netdev); 3760 err_alloc_etherdev: 3761 pci_release_regions(pdev); 3762 err_pci_reg: 3763 err_dma: 3764 pci_disable_device(pdev); 3765 return err; 3766 } 3767 3768 /** 3769 * iavf_suspend - Power management suspend routine 3770 * @dev_d: device info pointer 3771 * 3772 * Called when the system (VM) is entering sleep/suspend. 3773 **/ 3774 static int __maybe_unused iavf_suspend(struct device *dev_d) 3775 { 3776 struct net_device *netdev = dev_get_drvdata(dev_d); 3777 struct iavf_adapter *adapter = netdev_priv(netdev); 3778 3779 netif_device_detach(netdev); 3780 3781 while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, 3782 &adapter->crit_section)) 3783 usleep_range(500, 1000); 3784 3785 if (netif_running(netdev)) { 3786 rtnl_lock(); 3787 iavf_down(adapter); 3788 rtnl_unlock(); 3789 } 3790 iavf_free_misc_irq(adapter); 3791 iavf_reset_interrupt_capability(adapter); 3792 3793 clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); 3794 3795 return 0; 3796 } 3797 3798 /** 3799 * iavf_resume - Power management resume routine 3800 * @dev_d: device info pointer 3801 * 3802 * Called when the system (VM) is resumed from sleep/suspend. 3803 **/ 3804 static int __maybe_unused iavf_resume(struct device *dev_d) 3805 { 3806 struct pci_dev *pdev = to_pci_dev(dev_d); 3807 struct net_device *netdev = pci_get_drvdata(pdev); 3808 struct iavf_adapter *adapter = netdev_priv(netdev); 3809 u32 err; 3810 3811 pci_set_master(pdev); 3812 3813 rtnl_lock(); 3814 err = iavf_set_interrupt_capability(adapter); 3815 if (err) { 3816 rtnl_unlock(); 3817 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n"); 3818 return err; 3819 } 3820 err = iavf_request_misc_irq(adapter); 3821 rtnl_unlock(); 3822 if (err) { 3823 dev_err(&pdev->dev, "Cannot get interrupt vector.\n"); 3824 return err; 3825 } 3826 3827 queue_work(iavf_wq, &adapter->reset_task); 3828 3829 netif_device_attach(netdev); 3830 3831 return err; 3832 } 3833 3834 /** 3835 * iavf_remove - Device Removal Routine 3836 * @pdev: PCI device information struct 3837 * 3838 * iavf_remove is called by the PCI subsystem to alert the driver 3839 * that it should release a PCI device. The could be caused by a 3840 * Hot-Plug event, or because the driver is going to be removed from 3841 * memory. 3842 **/ 3843 static void iavf_remove(struct pci_dev *pdev) 3844 { 3845 struct net_device *netdev = pci_get_drvdata(pdev); 3846 struct iavf_adapter *adapter = netdev_priv(netdev); 3847 struct iavf_vlan_filter *vlf, *vlftmp; 3848 struct iavf_mac_filter *f, *ftmp; 3849 struct iavf_cloud_filter *cf, *cftmp; 3850 struct iavf_hw *hw = &adapter->hw; 3851 int err; 3852 /* Indicate we are in remove and not to run reset_task */ 3853 set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section); 3854 cancel_delayed_work_sync(&adapter->init_task); 3855 cancel_work_sync(&adapter->reset_task); 3856 cancel_delayed_work_sync(&adapter->client_task); 3857 if (adapter->netdev_registered) { 3858 unregister_netdev(netdev); 3859 adapter->netdev_registered = false; 3860 } 3861 if (CLIENT_ALLOWED(adapter)) { 3862 err = iavf_lan_del_device(adapter); 3863 if (err) 3864 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 3865 err); 3866 } 3867 3868 /* Shut down all the garbage mashers on the detention level */ 3869 adapter->state = __IAVF_REMOVE; 3870 adapter->aq_required = 0; 3871 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; 3872 iavf_request_reset(adapter); 3873 msleep(50); 3874 /* If the FW isn't responding, kick it once, but only once. */ 3875 if (!iavf_asq_done(hw)) { 3876 iavf_request_reset(adapter); 3877 msleep(50); 3878 } 3879 iavf_free_all_tx_resources(adapter); 3880 iavf_free_all_rx_resources(adapter); 3881 iavf_misc_irq_disable(adapter); 3882 iavf_free_misc_irq(adapter); 3883 iavf_reset_interrupt_capability(adapter); 3884 iavf_free_q_vectors(adapter); 3885 3886 cancel_delayed_work_sync(&adapter->watchdog_task); 3887 3888 cancel_work_sync(&adapter->adminq_task); 3889 3890 iavf_free_rss(adapter); 3891 3892 if (hw->aq.asq.count) 3893 iavf_shutdown_adminq(hw); 3894 3895 /* destroy the locks only once, here */ 3896 mutex_destroy(&hw->aq.arq_mutex); 3897 mutex_destroy(&hw->aq.asq_mutex); 3898 3899 iounmap(hw->hw_addr); 3900 pci_release_regions(pdev); 3901 iavf_free_all_tx_resources(adapter); 3902 iavf_free_all_rx_resources(adapter); 3903 iavf_free_queues(adapter); 3904 kfree(adapter->vf_res); 3905 spin_lock_bh(&adapter->mac_vlan_list_lock); 3906 /* If we got removed before an up/down sequence, we've got a filter 3907 * hanging out there that we need to get rid of. 3908 */ 3909 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 3910 list_del(&f->list); 3911 kfree(f); 3912 } 3913 list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list, 3914 list) { 3915 list_del(&vlf->list); 3916 kfree(vlf); 3917 } 3918 3919 spin_unlock_bh(&adapter->mac_vlan_list_lock); 3920 3921 spin_lock_bh(&adapter->cloud_filter_list_lock); 3922 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { 3923 list_del(&cf->list); 3924 kfree(cf); 3925 } 3926 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3927 3928 free_netdev(netdev); 3929 3930 pci_disable_pcie_error_reporting(pdev); 3931 3932 pci_disable_device(pdev); 3933 } 3934 3935 static SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume); 3936 3937 static struct pci_driver iavf_driver = { 3938 .name = iavf_driver_name, 3939 .id_table = iavf_pci_tbl, 3940 .probe = iavf_probe, 3941 .remove = iavf_remove, 3942 .driver.pm = &iavf_pm_ops, 3943 .shutdown = iavf_shutdown, 3944 }; 3945 3946 /** 3947 * iavf_init_module - Driver Registration Routine 3948 * 3949 * iavf_init_module is the first routine called when the driver is 3950 * loaded. All it does is register with the PCI subsystem. 3951 **/ 3952 static int __init iavf_init_module(void) 3953 { 3954 int ret; 3955 3956 pr_info("iavf: %s\n", iavf_driver_string); 3957 3958 pr_info("%s\n", iavf_copyright); 3959 3960 iavf_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1, 3961 iavf_driver_name); 3962 if (!iavf_wq) { 3963 pr_err("%s: Failed to create workqueue\n", iavf_driver_name); 3964 return -ENOMEM; 3965 } 3966 ret = pci_register_driver(&iavf_driver); 3967 return ret; 3968 } 3969 3970 module_init(iavf_init_module); 3971 3972 /** 3973 * iavf_exit_module - Driver Exit Cleanup Routine 3974 * 3975 * iavf_exit_module is called just before the driver is removed 3976 * from memory. 3977 **/ 3978 static void __exit iavf_exit_module(void) 3979 { 3980 pci_unregister_driver(&iavf_driver); 3981 destroy_workqueue(iavf_wq); 3982 } 3983 3984 module_exit(iavf_exit_module); 3985 3986 /* iavf_main.c */ 3987